From 476014bfe2ba9babef33bae8fd1feeb56c887928 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 01/43] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- Makefile | 1 + app/demo_adapter.py | 210 ++++++++++++++++++++++- app/main.py | 33 +++- app/routers/account/account.py | 8 + app/routers/compute/compute.py | 4 +- app/routers/error_handlers.py | 6 + app/routers/facility/__init__.py | 0 app/routers/facility/facility.py | 77 +++++++++ app/routers/facility/facility_adapter.py | 65 +++++++ app/routers/facility/models.py | 58 +++++++ app/routers/iri_router.py | 36 +++- 11 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 app/routers/facility/__init__.py create mode 100644 app/routers/facility/facility.py create mode 100644 app/routers/facility/facility_adapter.py create mode 100644 app/routers/facility/models.py diff --git a/Makefile b/Makefile index 5bd9034..ba7552a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ dev : .venv @source ./.venv/bin/activate && \ + IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9cbc796..daf266d 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -13,6 +13,7 @@ from pydantic import BaseModel from typing import Any, Tuple from fastapi import HTTPException +from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter from .routers.compute import models as compute_models, facility_adapter as compute_adapter @@ -40,9 +41,13 @@ def get_base_temp_dir(cls): return cls._base_temp_dir +def demo_uuid(kind: str, name: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, - task_adapter.FacilityAdapter): + task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): def __init__(self): self.resources = [] self.incidents = [] @@ -52,11 +57,83 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - + self.locations = [] + self.facility = {} + self.sites = [] self._init_state() def _init_state(self): + now = datetime.datetime.now(datetime.timezone.utc) + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + + facility = facility_models.Facility( + id=demo_uuid("facility", "demo_facility"), + name="Demo Facility", + description="A demo facility for testing the IRI Facility API", + last_modified=now, + short_name="DEMO", + organization_name="Demo Organization", + support_uri="https://support.demo.example", + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], + ) + + self.facility = facility + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + + day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -182,6 +259,135 @@ def _init_state(self): d += datetime.timedelta(minutes=int(random.random() * 15 + 1)) + # ---------------------------- + # Facility API + # ---------------------------- + + async def get_facility( + self: "DemoAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility: + return self.facility + + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + # ---------------------------- + # Status API + # ---------------------------- async def get_resources( self : "DemoAdapter", diff --git a/app/main.py b/app/main.py index be5feaa..080645e 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,11 @@ """Main API application""" import logging from fastapi import FastAPI +from fastapi import Request +from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers +from app.routers.facility import facility from app.routers.status import status from app.routers.account import account from app.routers.compute import compute @@ -19,11 +22,39 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" +@APP.get(api_prefix) +async def api_discovery(request: Request): + base = str(request.base_url).rstrip("/") + items = [] + for route in APP.router.routes: + if not isinstance(route, APIRoute): + continue + # skip docs & openapi + if route.path.startswith("/docs") or route.path.startswith("/openapi"): + continue + for method in route.methods: + if method == "HEAD" or method == "OPTIONS": + continue + items.append({ + "id": route.name or f"{method}_{route.path}", + "method": method, + "path": route.path, + "_links": [ + { + "rel": "self", + "href": f"{base.rstrip('/')}{route.path}", + "type": "application/json" + } + ] + }) + return items + # Attach routers under the prefix +APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) APP.include_router(account.router, prefix=api_prefix) APP.include_router(compute.router, prefix=api_prefix) APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") +logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 951fc9b..508d556 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -21,6 +21,7 @@ ) async def get_capabilities( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -35,6 +36,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -53,6 +55,7 @@ async def get_capability( ) async def get_projects( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -71,6 +74,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -93,6 +97,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -116,6 +121,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -141,6 +147,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -169,6 +176,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index e7481ac..72a8169 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -154,8 +154,8 @@ async def get_job_status( async def get_job_statuses( resource_id : str, request : Request, - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=10000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, historical : bool = False, include_spec: bool = False, diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 09769ec..337b5fc 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -65,6 +65,12 @@ async def validation_error_handler(request: Request, exc: RequestValidationError @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 304: + return JSONResponse( + status_code=304, + content=None, + headers=exc.headers or {}) + if exc.status_code == 401: return problem_response( request=request, diff --git a/app/routers/facility/__init__.py b/app/routers/facility/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py new file mode 100644 index 0000000..9964925 --- /dev/null +++ b/app/routers/facility/facility.py @@ -0,0 +1,77 @@ +from fastapi import Request, HTTPException, Depends, Query +from .. import iri_router +from ..error_handlers import DEFAULT_RESPONSES +from .import models, facility_adapter + + +router = iri_router.IriRouter( + facility_adapter.FacilityAdapter, + prefix="/facility", + tags=["facility"], +) + +@router.get("", responses=DEFAULT_RESPONSES) +async def get_facility( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + ) -> models.Facility: + """Get facility information""" + return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES) +async def list_sites( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +async def get_site( + request: Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +async def get_site_location( + request : Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES) +async def list_locations( + request : Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +async def get_location( + request : Request, + location_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py new file mode 100644 index 0000000..d316674 --- /dev/null +++ b/app/routers/facility/facility_adapter.py @@ -0,0 +1,65 @@ +from abc import abstractmethod +from . import models as facility_models +from ..iri_router import AuthenticatedAdapter + + +class FacilityAdapter(AuthenticatedAdapter): + """ + Facility-specific code is handled by the implementation of this interface. + Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) + to install your facility adapter before the API starts. + """ + + @abstractmethod + async def get_facility( + self: "FacilityAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility | None: + pass + + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py new file mode 100644 index 0000000..0c04cc4 --- /dev/null +++ b/app/routers/facility/models.py @@ -0,0 +1,58 @@ +from datetime import datetime +from uuid import UUID +from typing import List, Optional +from pydantic import BaseModel, Field, HttpUrl, computed_field +from .. import iri_router +from ... import config + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") + country_name: Optional[str] = Field(None, description="Country name of the Location.") + locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") + state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") + street_address: Optional[str] = Field(None, description="Street address of the Location.") + unlocode: Optional[str] = Field(None, description="United Nations trade and transport location code.") + altitude: Optional[float] = Field(None, description="Altitude of the Location.") + latitude: Optional[float] = Field(None, description="Latitude of the Location.") + longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + +class Facility(NamedObject): + def _self_path(self) -> str: + return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index dafb970..2de7e69 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -3,11 +3,13 @@ import logging import importlib import datetime +from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from pydantic_core import core_schema from .account.models import User + bearer_token = APIKeyHeader(name="Authorization") @@ -128,17 +130,33 @@ async def get_user( def forbidExtraQueryParams(*allowedParams: str): - """Dependency to forbid extra query parameters not in allowedParams.""" - - async def checker(_req: Request): + async def checker(req: Request): if "*" in allowedParams: - return # Permit anything - incoming = set(_req.query_params.keys()) + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + allowed = set(allowedParams) - unknown = incoming - allowed - if unknown: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", "loc": ["query", param], "msg": f"Unexpected query parameter: {param}"} for param in unknown]) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) return checker class StrictDateTime: From 9985b7d23538b8b6bd179c45c1a8e452a79fe716 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 07:50:47 -0600 Subject: [PATCH 02/43] Make NamedObject reusable --- app/routers/facility/models.py | 22 ++------- app/routers/models.py | 46 +++++++++++++++++++ app/routers/status/models.py | 82 +++++++++------------------------- 3 files changed, 69 insertions(+), 81 deletions(-) create mode 100644 app/routers/models.py diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 0c04cc4..2bea62f 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,23 +1,7 @@ -from datetime import datetime -from uuid import UUID +"""Facility-related models.""" from typing import List, Optional -from pydantic import BaseModel, Field, HttpUrl, computed_field -from .. import iri_router -from ... import config - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - +from pydantic import Field, HttpUrl +from ..models import NamedObject class Site(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/models.py b/app/routers/models.py new file mode 100644 index 0000000..f9a1c00 --- /dev/null +++ b/app/routers/models.py @@ -0,0 +1,46 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from pydantic import BaseModel, Field, computed_field +from . import iri_router +from .. import config + + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index b1f3a8b..2c7dc76 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,6 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config +from ..models import NamedObject class Link(BaseModel): rel : str @@ -15,38 +16,6 @@ class Status(enum.Enum): unknown = "unknown" -class NamedResource(BaseModel): - id : str - name : str - description : str - last_modified : datetime.datetime - - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a - - class ResourceType(enum.Enum): website = "website" service = "service" @@ -57,28 +26,24 @@ class ResourceType(enum.Enum): unknown = "unknown" -class Resource(NamedResource): +class Resource(NamedObject): + + def _self_path(self) -> str: + return f"/status/resources/{self.id}" + capability_ids: list[str] = Field(exclude=True) group: str | None current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] - @staticmethod def find(resources, name, description, group, modified_since, resource_type): - a = NamedResource.find(resources, name, description, modified_since) + a = NamedObject.find(resources, name, description, modified_since) if group: a = [aa for aa in a if aa.group == group] if resource_type: @@ -86,25 +51,21 @@ def find(resources, name, description, group, modified_since, resource_type): return a -class Event(NamedResource): +class Event(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.incident_id}/events/{self.id}" + occurred_at : datetime.datetime status : Status resource_id : str = Field(exclude=True) incident_id : str | None = Field(exclude=True, default=None) - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}/events/{self.id}" - - @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" - @computed_field(description="The event's incident") @property def incident_uri(self) -> str|None: @@ -123,7 +84,7 @@ def find( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, ) -> list: - events = NamedResource.find(events, name, description, modified_since) + events = NamedObject.find(events, name, description, modified_since) if resource_id: events = [e for e in events if e.resource_id == resource_id] if status: @@ -150,7 +111,11 @@ class Resolution(enum.Enum): pending = "pending" -class Incident(NamedResource): +class Incident(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.id}" + status : Status resource_ids : list[str] = Field(exclude=True) event_ids : list[str] = Field(exclude=True) @@ -159,25 +124,18 @@ class Incident(NamedResource): type : IncidentType resolution : Resolution - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] - @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] def find( + self, incidents : list, name : str | None = None, description : str | None = None, @@ -189,7 +147,7 @@ def find( modified_since : datetime.datetime | None = None, resource_id : str | None = None, ) -> list: - incidents = NamedResource.find(incidents, name, description, modified_since) + incidents = NamedObject.find(incidents, name, description, modified_since) if resource_id: incidents = [e for e in incidents if resource_id in e.resource_ids] if status: From bf82bccf71a696df5bd8513e58c44662ef4a0eaa Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 08:16:21 -0600 Subject: [PATCH 03/43] Include operation_id for facility (similar to pull request #21) --- app/routers/facility/facility.py | 12 ++++++------ app/routers/status/models.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 9964925..0e6f960 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -10,7 +10,7 @@ tags=["facility"], ) -@router.get("", responses=DEFAULT_RESPONSES) +@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -19,7 +19,7 @@ async def get_facility( """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) -@router.get("/sites", responses=DEFAULT_RESPONSES) +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") async def list_sites( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -32,7 +32,7 @@ async def list_sites( """List sites""" return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") async def get_site( request: Request, site_id: str, @@ -42,7 +42,7 @@ async def get_site( """Get site by ID""" return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") async def get_site_location( request : Request, site_id: str, @@ -52,7 +52,7 @@ async def get_site_location( """Get site location by site ID""" return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) -@router.get("/locations", responses=DEFAULT_RESPONSES) +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") async def list_locations( request : Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -66,7 +66,7 @@ async def list_locations( """List locations""" return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") async def get_location( request : Request, location_id: str, diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 2c7dc76..bb1f87b 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -134,8 +134,8 @@ def event_uris(self) -> list[str]: def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] + @staticmethod def find( - self, incidents : list, name : str | None = None, description : str | None = None, From 75945e0ba66794cd0eeb9806f296a391239f9901 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Thu, 15 Jan 2026 11:48:45 -0800 Subject: [PATCH 04/43] simplified facility endpoint proposal --- app/demo_adapter.py | 184 +---------------------- app/routers/facility/facility.py | 57 ------- app/routers/facility/facility_adapter.py | 47 ------ app/routers/facility/models.py | 33 +--- 4 files changed, 14 insertions(+), 307 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index daf266d..2d2ca30 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -65,50 +65,7 @@ def __init__(self): def _init_state(self): now = datetime.datetime.now(datetime.timezone.utc) - loc1 = facility_models.Location( - id=demo_uuid("location", "demo_location_1"), - name="Demo Location 1", - description="The first demo location", - last_modified=now, - short_name="DL1", - country_name="USA", - locality_name="Demo City", - state_or_province_name="DC", - latitude=36.173357, - longitude=-234.51452) - - loc2 = facility_models.Location( - id=demo_uuid("location", "demo_location_2"), - name="Demo Location 2", - description="The second demo location", - last_modified=now, - short_name="DL2", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - latitude=38.410558, - longitude=-286.36999) - - site1 = facility_models.Site( - id=demo_uuid("site", "demo_site_1"), - name="Demo Site 1", - description="The first demo site", - last_modified=now, - short_name="DS1", - operating_organization="Demo Org", - location_uri=loc1.self_uri, - resource_uris=[]) - site2 = facility_models.Site( - id=demo_uuid("site", "demo_site_2"), - name="Demo Site 2", - description="The second demo site", - last_modified=now, - short_name="DS2", - operating_organization="Demo Org", - location_uri=loc2.self_uri, - resource_uris=[]) - - facility = facility_models.Facility( + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", description="A demo facility for testing the IRI Facility API", @@ -116,24 +73,15 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri], - location_uris=[loc1.self_uri, loc2.self_uri], - resource_uris=[], - event_uris=[], - incident_uris=[], - capability_uris=[], - project_uris=[], - project_allocation_uris=[], - user_allocation_uris=[], + facility_uri="https://www.demo.example", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + street_address="1 main st", + latitude=38.410558, + longitude=-286.36999 ) - self.facility = facility - loc1.site_uris.append(site1.self_uri) - loc2.site_uris.append(site2.self_uri) - self.locations = [loc1, loc2] - self.sites = [site1, site2] - - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -269,122 +217,6 @@ async def get_facility( ) -> facility_models.Facility: return self.facility - - async def list_sites( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - - sites = self.sites - - if name: - sites = [s for s in sites if name.lower() in s.name.lower()] - - if short_name: - sites = [s for s in sites if s.short_name == short_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - sites = [s for s in sites if s.last_modified > ms] - - o = offset or 0 - l = limit or len(sites) - return sites[o:o+l] - - - async def get_site( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site: - - site = next((s for s in self.sites if s.id == site_id), None) - if not site: - raise HTTPException(status_code=404, detail="Site not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if site.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) - - return site - - - async def get_site_location( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - site = await self.get_site(site_id) - - if not site.location_uri: - raise HTTPException(status_code=404, detail="Site has no location") - - location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - - return location - - - async def list_locations( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - - locs = self.locations - - if name: - locs = [l for l in locs if name.lower() in l.name.lower()] - - if short_name: - locs = [l for l in locs if l.short_name == short_name] - - if country_name: - locs = [l for l in locs if l.country_name == country_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - locs = [l for l in locs if l.last_modified > ms] - - o = offset or 0 - l = limit or len(locs) - return locs[o:o+l] - - - async def get_location( - self: "DemoAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - location = next((l for l in self.locations if l.id == location_id), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - return location - # ---------------------------- # Status API # ---------------------------- diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 0e6f960..9b7545b 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,60 +18,3 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") -async def list_sites( - request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), - )-> list[models.Site]: - """List sites""" - return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) - -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") -async def get_site( - request: Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Site: - """Get site by ID""" - return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) - -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") -async def get_site_location( - request : Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get site location by site ID""" - return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) - -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") -async def list_locations( - request : Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - country_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), - )-> list[models.Location]: - """List locations""" - return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) - -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") -async def get_location( - request : Request, - location_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get location by ID""" - return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674..bccdcfe 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,50 +16,3 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass - - @abstractmethod - async def list_sites( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - pass - - @abstractmethod - async def get_site( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site | None: - pass - - @abstractmethod - async def get_site_location( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass - - @abstractmethod - async def list_locations( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - pass - - @abstractmethod - async def get_location( - self: "FacilityAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 2bea62f..efd93dd 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,20 +1,13 @@ """Facility-related models.""" -from typing import List, Optional +from typing import Optional from pydantic import Field, HttpUrl from ..models import NamedObject -class Site(NamedObject): - def _self_path(self) -> str: - return f"/facility/sites/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Site.") - operating_organization: str = Field(..., description="Organization operating the Site.") - location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - -class Location(NamedObject): - def _self_path(self) -> str: - return f"/facility/locations/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Location.") +class Facility(NamedObject): + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization's name.") + facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -23,20 +16,6 @@ def _self_path(self) -> str: altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") -class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization’s name.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") - location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") - event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") - incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") - capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") - project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") - project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") - user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") From 52567f26d8abae17400f382824e4f9c5a016c740 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 05/43] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- app/demo_adapter.py | 2 -- app/routers/facility/facility.py | 1 + app/routers/facility/facility_adapter.py | 1 + app/routers/facility/models.py | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 2d2ca30..0d58103 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -57,9 +57,7 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - self.locations = [] self.facility = {} - self.sites = [] self._init_state() diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 9b7545b..fdba124 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,3 +18,4 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index bccdcfe..cbee951 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,3 +16,4 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass + diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index efd93dd..5b21d8a 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -19,3 +19,4 @@ class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + From 0dbb6d8eaad88ae80a40e02bdb23c47c52c6e7bc Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 15:27:00 -0600 Subject: [PATCH 06/43] Remove /api/v1 --- app/main.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/app/main.py b/app/main.py index 080645e..7864143 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,6 @@ """Main API application""" import logging from fastapi import FastAPI -from fastapi import Request -from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -22,33 +20,6 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" -@APP.get(api_prefix) -async def api_discovery(request: Request): - base = str(request.base_url).rstrip("/") - items = [] - for route in APP.router.routes: - if not isinstance(route, APIRoute): - continue - # skip docs & openapi - if route.path.startswith("/docs") or route.path.startswith("/openapi"): - continue - for method in route.methods: - if method == "HEAD" or method == "OPTIONS": - continue - items.append({ - "id": route.name or f"{method}_{route.path}", - "method": method, - "path": route.path, - "_links": [ - { - "rel": "self", - "href": f"{base.rstrip('/')}{route.path}", - "type": "application/json" - } - ] - }) - return items - # Attach routers under the prefix APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) @@ -57,4 +28,4 @@ async def api_discovery(request: Request): APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file +logging.getLogger().info(f"API path: {api_prefix}") From 65a4e46849160eea8e2b7e415e81d1eec170b9c0 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:14:23 -0600 Subject: [PATCH 07/43] Refactor shared validators & models to fix import loading issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move common validators and base models into routers/dependencies and update imports across routers and schemas to use the new shared location. This keeps API behavior unchanged. No functional API changes — purely structural and validation hygiene: --- app/routers/account/account.py | 17 +- app/routers/account/models.py | 15 +- app/routers/compute/compute.py | 88 ++++++----- app/routers/compute/models.py | 60 ++++--- app/routers/dependencies.py | 176 +++++++++++++++++++++ app/routers/facility/facility.py | 8 +- app/routers/facility/models.py | 2 +- app/routers/filesystem/facility_adapter.py | 4 +- app/routers/filesystem/filesystem.py | 42 +---- app/routers/filesystem/models.py | 9 +- app/routers/iri_router.py | 77 --------- app/routers/models.py | 46 ------ app/routers/status/models.py | 2 +- app/routers/status/status.py | 25 +-- app/routers/task/models.py | 19 ++- 15 files changed, 311 insertions(+), 279 deletions(-) create mode 100644 app/routers/dependencies.py delete mode 100644 app/routers/models.py diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 508d556..b2c41f4 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import forbidExtraQueryParams router = iri_router.IriRouter( @@ -21,7 +22,7 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -36,7 +37,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -55,7 +56,7 @@ async def get_capability( ) async def get_projects( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -74,7 +75,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -97,7 +98,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -121,7 +122,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -147,7 +148,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -176,7 +177,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 6ed69ea..c012ee3 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, computed_field, Field +from pydantic import computed_field, Field import enum from ... import config +from ..dependencies import IRIBaseModel class AllocationUnit(enum.Enum): @@ -9,7 +10,7 @@ class AllocationUnit(enum.Enum): inodes = "inodes" -class Capability(BaseModel): +class Capability(IRIBaseModel): """ An aspect of a resource that can have an allocation. For example, Perlmutter nodes with GPUs @@ -22,7 +23,7 @@ class Capability(BaseModel): units: list[AllocationUnit] -class User(BaseModel): +class User(IRIBaseModel): """A user of the facility""" id: str name: str @@ -31,7 +32,7 @@ class User(BaseModel): # we could expose more fields here (eg. email) but it might be against policy -class Project(BaseModel): +class Project(IRIBaseModel): """A project and its users at a facility""" id: str name: str @@ -39,14 +40,14 @@ class Project(BaseModel): user_ids: list[str] -class AllocationEntry(BaseModel): +class AllocationEntry(IRIBaseModel): """Base class for allocations.""" allocation: float # how much this allocation can spend usage: float # how much this allocation has spent unit: AllocationUnit -class ProjectAllocation(BaseModel): +class ProjectAllocation(IRIBaseModel): """ A project's allocation for a capability. (aka. repo) This allocation is a piece of the total allocation for the capability. (eg. 5% of the total node hours of Perlmutter GPU nodes) @@ -71,7 +72,7 @@ def capability_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{self.capability_id}" -class UserAllocation(BaseModel): +class UserAllocation(IRIBaseModel): """ A user's allcation in a project. This allocation is a piece of the project's allocation. diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 72a8169..7a2db23 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,9 +1,10 @@ -from typing import List, Annotated -from fastapi import HTTPException, Request, Depends, status, Form, Query +from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router + from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router +from ..dependencies import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -24,6 +25,7 @@ async def submit_job( resource_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Submit a job on a compute resource @@ -45,39 +47,41 @@ async def submit_job( return await router.adapter.submit_job(resource, user, job_spec) -@router.post( - "/job/script/{resource_id:str}", - dependencies=[Depends(router.current_user)], - response_model=models.Job, - response_model_exclude_unset=True, - responses=DEFAULT_RESPONSES, - operation_id="launchJobScript", -) -async def submit_job_path( - resource_id: str, - job_script_path : str, - request : Request, - args : Annotated[List[str], Form()] = [], - ): - """ - Submit a job on a compute resource - - - **resource**: the name of the compute resource to use - - **job_script_path**: path to the job script on the compute resource - - **args**: optional arguments to the job script - - This command will attempt to submit a job and return its id. - """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - - # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) - - # the handler can use whatever means it wants to submit the job and then fill in its id - # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job_script(resource, user, job_script_path, args) +# TODO: this conflicts with PUT commented out while we finalize the API design +#@router.post( +# "/job/script/{resource_id:str}", +# dependencies=[Depends(router.current_user)], +# response_model=models.Job, +# response_model_exclude_unset=True, +# responses=DEFAULT_RESPONSES, +# operation_id="launchJobScript", +#) +#async def submit_job_path( +# resource_id: str, +# job_script_path : str, +# request : Request, +# args : Annotated[List[str], Form()] = [], +# _forbid = Depends(iri_router.forbidExtraQueryParams("job_script_path")), +# ): +# """ +# Submit a job on a compute resource +# +# - **resource**: the name of the compute resource to use +# - **job_script_path**: path to the job script on the compute resource +# - **args**: optional arguments to the job script +# +# This command will attempt to submit a job and return its id. +# """ +# user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) +# if not user: +# raise HTTPException(status_code=404, detail="User not found") +# +# # look up the resource (todo: maybe ensure it's available) +# resource = await status_router.adapter.get_resource(resource_id) +# +# # the handler can use whatever means it wants to submit the job and then fill in its id +# # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs +# return await router.adapter.submit_job_script(resource, user, job_script_path, args) @router.put( @@ -93,6 +97,7 @@ async def update_job( job_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Update a previously submitted job for a resource. @@ -126,8 +131,9 @@ async def get_job_status( resource_id : str, job_id : str, request : Request, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -149,7 +155,7 @@ async def get_job_status( response_model=list[models.Job], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, - operation_id="getJobs", + operation_id="getAllJobs", ) async def get_job_statuses( resource_id : str, @@ -157,8 +163,9 @@ async def get_job_statuses( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -187,6 +194,7 @@ async def cancel_job( resource_id : str, job_id : str, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """Cancel a job""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 35d34ef..006aabc 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,46 +1,45 @@ -from typing import Annotated -from pydantic import BaseModel, field_serializer, ConfigDict, Field -import datetime from enum import IntEnum +from pydantic import field_serializer, ConfigDict, StrictBool, Field +from ..dependencies import IRIBaseModel -class ResourceSpec(BaseModel): - node_count: int | None = None - process_count: int | None = None - processes_per_node: int | None = None - cpu_cores_per_process: int | None = None - gpu_cores_per_process: int | None = None - exclusive_node_use: bool = True - memory: int | None = None +class ResourceSpec(IRIBaseModel): + node_count: int = Field(default=None, ge=1, description="Number of nodes") + process_count: int = Field(default=None, ge=1, description="Number of processes") + processes_per_node: int = Field(default=None, ge=1, description="Number of processes per node") + cpu_cores_per_process: int = Field(default=None, ge=1, description="Number of CPU cores per process") + gpu_cores_per_process: int = Field(default=None, ge=1, description="Number of GPU cores per process") + exclusive_node_use: StrictBool = True + memory: int = Field(default=None, ge=1, description="Amount of memory in megabytes") -class JobAttributes(BaseModel): - duration: Annotated[int | None, Field(description="Duration in seconds", ge=0, examples=[30, 60, 120])] = None - queue_name: str | None = None - account: str | None = None - reservation_id: str | None = None +class JobAttributes(IRIBaseModel): + duration: int = Field(default=None, ge=1, description="Duration in seconds", examples=[30, 60, 120]) + queue_name: str = Field(default=None, min_length=1, description="Name of the queue/partition to use") + account: str = Field(default=None, min_length=1, description="Account/Project name to charge") + reservation_id: str = Field(default=None, min_length=1, description="Reservation ID to use") custom_attributes: dict[str, str] = {} -class JobSpec(BaseModel): +class JobSpec(IRIBaseModel): model_config = ConfigDict(extra="forbid") - executable : str | None = None + executable : str = Field(min_length=1, description="The executable to run") arguments: list[str] = [] - directory: str | None = None - name: str | None = None - inherit_environment: bool = True + directory: str = Field(default=None, min_length=1, description="The working directory for the job") + name: str = Field(default=None, min_length=1, description="The name of the job") + inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment") environment: dict[str, str] = {} - stdin_path: str | None = None - stdout_path: str | None = None - stderr_path: str | None = None + stdin_path: str = Field(default=None, min_length=1, description="Path to the standard input file") + stdout_path: str = Field(default=None, min_length=1, description="Path to the standard output file") + stderr_path: str = Field(default=None, min_length=1, description="Path to the standard error file") resources: ResourceSpec | None = None attributes: JobAttributes | None = None - pre_launch: str | None = None - post_launch: str | None = None - launcher: str | None = None + pre_launch: str = Field(default=None, min_length=1, description="Command to run before launching the job") + post_launch: str = Field(default=None, min_length=1, description="Command to run after launching the job") + launcher: str = Field(default=None, min_length=1, description="Launcher to use for the job") -class CommandResult(BaseModel): +class CommandResult(IRIBaseModel): status : str result : str | None = None @@ -80,20 +79,19 @@ class JobState(IntEnum): """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" -class JobStatus(BaseModel): +class JobStatus(IRIBaseModel): state : JobState time : float | None = None message : str | None = None exit_code : int | None = None meta_data : dict[str, object] | None = None - @field_serializer('state') def serialize_state(self, state: JobState): return state.name -class Job(BaseModel): +class Job(IRIBaseModel): id : str status : JobStatus | None = None job_spec : JobSpec | None = None diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py new file mode 100644 index 0000000..17b4f09 --- /dev/null +++ b/app/routers/dependencies.py @@ -0,0 +1,176 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from urllib.parse import parse_qs + +from pydantic_core import core_schema +from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer +from fastapi import Request, HTTPException + +from .. import config + + +# These are Pydantic custom types for strict validation +# that are not implmented in Pydantic by default. +# ----------------------------------------------------------------------- +# StrictBool: a strict boolean type +class StrictBool: + """Strict boolean: + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + v = value.strip().lower() + if v == "true": + return True + if v == "false": + return False + raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") + raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "boolean", + "description": "Strict boolean. Only true/false allowed (bool or string)." + } + +# ----------------------------------------------------------------------- +# StrictDateTime: a strict ISO8601 datetime type + +class StrictDateTime: + """ + Strict ISO8601 datetime: + - Accepts datetime objects + - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 + - Converts 'Z' → UTC + - Converts naive datetimes → UTC + - Rejects integers ("0"), null, garbage strings, etc. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + if isinstance(value, datetime.datetime): + return StrictDateTime._normalize(value) + if not isinstance(value, str): + raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") + v = value.strip() + if v.endswith("Z"): + v = v[:-1] + "+00:00" + try: + dt = datetime.datetime.fromisoformat(v) + except Exception as ex: + raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex + + return StrictDateTime._normalize(dt) + + @staticmethod + def _normalize(dt: datetime.datetime) -> datetime.datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "string", + "format": "date-time", + "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." + } + + +def forbidExtraQueryParams(*allowedParams: str): + async def checker(req: Request): + if "*" in allowedParams: + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + + allowed = set(allowedParams) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) + return checker + + +class IRIBaseModel(BaseModel): + """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _hide_extra(self, handler): + data = handler(self) + extra = getattr(self, "__pydantic_extra__", {}) or {} + for k in extra: + data.pop(k, None) + return data + +class NamedObject(IRIBaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index fdba124..1c38918 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,7 +1,8 @@ -from fastapi import Request, HTTPException, Depends, Query +from fastapi import Request, Depends, Query from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( @@ -13,9 +14,8 @@ @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5b21d8a..5db7a18 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..models import NamedObject +from ..dependencies import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 2c08a3c..a70efb0 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -1,15 +1,15 @@ import os from abc import abstractmethod +from typing import Any, Tuple from ..status import models as status_models from ..account import models as account_models from . import models as filesystem_models from ..iri_router import AuthenticatedAdapter -from typing import Any, Tuple def to_int(name, default_value): try: - return os.environ.get(name) or default_value + return int(os.environ.get(name) or default_value) except: return default_value diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index a111484..d583c64 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -354,40 +354,13 @@ async def get_view( resource_id: str, request : Request, path: Annotated[str, Query(description="File path")], - size: Annotated[ - int, - Query( - alias="size", - description="Value, in bytes, of the size of data to be retrieved from the file.", - ), - ] = facility_adapter.OPS_SIZE_LIMIT, - offset: Annotated[ - int, - Query( - alias="offset", - description="Value in bytes of the offset.", - ), - ] = 0, -) -> str: - user, resource = await _user_resource(resource_id, request) - if offset < 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`offset` value must be an integer value equal or greater than 0", - ) - - if size <= 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`size` value must be an integer value greater than 0", - ) + size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", + ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, - if size > facility_adapter.OPS_SIZE_LIMIT: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"`size` value must be less than {facility_adapter.OPS_SIZE_LIMIT} bytes", - ) + offset: Annotated[int, Query( description="Value in bytes of the offset.", ge=0)] = 0 + ) -> str: + user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user, @@ -397,9 +370,8 @@ async def get_view( command="view", args={ "path": path, - "size": size or facility_adapter.OPS_SIZE_LIMIT, - "offset": offset or 0, - + "size": size, + "offset": offset, } ) ) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index b24c908..d32ad94 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -12,11 +12,10 @@ class CompressionType(str, Enum): - none = "none" - bzip2 = "bzip2" - gzip = "gzip" - xz = "xz" - + none = "none" + bzip2 = "bzip2" + gzip = "gzip" + xz = "xz" class ContentUnit(str, Enum): lines = "lines" diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 2de7e69..d3fc9e1 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -2,11 +2,9 @@ import os import logging import importlib -import datetime from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader -from pydantic_core import core_schema from .account.models import User @@ -127,78 +125,3 @@ async def get_user( Retrieve additional user information (name, email, etc.) for the given user_id. """ pass - - -def forbidExtraQueryParams(*allowedParams: str): - async def checker(req: Request): - if "*" in allowedParams: - return - - raw_qs = req.scope.get("query_string", b"") - parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) - - allowed = set(allowedParams) - - for key, values in parsed.items(): - if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) - - if len(values) > 1: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) - return checker - -class StrictDateTime: - """ - Strict ISO8601 datetime: - ✔ Accepts datetime objects - ✔ Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 - ✔ Converts 'Z' → UTC - ✔ Converts naive datetimes → UTC - ✘ Rejects integers ("0"), null, garbage strings, etc. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - if isinstance(value, datetime.datetime): - return StrictDateTime._normalize(value) - if not isinstance(value, str): - raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") - v = value.strip() - if v.endswith("Z"): - v = v[:-1] + "+00:00" - try: - dt = datetime.datetime.fromisoformat(v) - except Exception as ex: - raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex - - return StrictDateTime._normalize(dt) - - @staticmethod - def _normalize(dt: datetime.datetime) -> datetime.datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } diff --git a/app/routers/models.py b/app/routers/models.py deleted file mode 100644 index f9a1c00..0000000 --- a/app/routers/models.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Default models used by multiple routers.""" -import datetime -from typing import Optional -from pydantic import BaseModel, Field, computed_field -from . import iri_router -from .. import config - - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index bb1f87b..ff5d4e4 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..models import NamedObject +from ..dependencies import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 7bb0fd0..2daf30e 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -23,9 +24,9 @@ async def get_resources( group : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - modified_since: iri_router.StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), ) -> list[models.Resource]: return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) @@ -60,14 +61,14 @@ async def get_incidents( description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), type_: models.IncidentType = Query(alias="type", default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), ) -> list[models.Incident]: return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) @@ -104,13 +105,13 @@ async def get_events( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) diff --git a/app/routers/task/models.py b/app/routers/task/models.py index da3c5cc..cea9787 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,19 +1,18 @@ -from pydantic import BaseModel import enum +from pydantic import BaseModel class TaskStatus(str, enum.Enum): - pending = "pending" - active = "active" - completed = "completed" - failed = "failed" - canceled = "canceled" - + pending = "pending" + active = "active" + completed = "completed" + failed = "failed" + canceled = "canceled" class TaskCommand(BaseModel): - router: str - command: str - args: dict + router: str + command: str + args: dict class Task(BaseModel): From f4adcb66f90f4ff99af0c7f8bbc25cd6cea6ad8a Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:17:00 -0600 Subject: [PATCH 08/43] Github Action to validate api --- .github/workflows/api-validation.yml | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/api-validation.yml diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml new file mode 100644 index 0000000..624e84e --- /dev/null +++ b/.github/workflows/api-validation.yml @@ -0,0 +1,133 @@ +name: API Validation with Schemathesis + +on: + pull_request: + push: + branches: [ main ] + +jobs: + schemathesis: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged + - name: Checkout schema validator repository + uses: actions/checkout@v4 + with: + repository: juztas/iri-facility-api-docs + ref: schemavalidator + path: schema-validator + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: pip install uv + + - name: Build an image + run: docker build --platform=linux/amd64 -t iri-facility-api-base . + + - name: Run Facility API container + run: | + docker run -d \ + -p 8000:8000 \ + --platform=linux/amd64 \ + --name iri-facility-api-base \ + -e IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + -e API_URL_ROOT=http://127.0.0.1:8000 \ + -e IRI_API_TOKEN=12345 \ + iri-facility-api-base + + - name: Wait for API to be ready + run: | + for i in {1..60}; do + if curl -fs http://127.0.0.1:8000/openapi.json; then + echo "API ready" + exit 0 + fi + sleep 2 + done + echo "API did not start" + exit 1 + + - name: Create venv & install validator dependencies + run: | + uv venv + source .venv/bin/activate + uv pip install -r schema-validator/verification/requirements.txt + + - name: Run Schemathesis validation (local spec) + id: schemathesis_local + env: + IRI_API_TOKEN: "12345" # This is dummy token for testing (mock adapter) + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://127.0.0.1:8000 \ + --report-name schemathesis-local + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Run Schemathesis validation (official spec) + id: schemathesis_official + env: + IRI_API_TOKEN: "12345" + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://localhost:8000 \ + --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --report-name schemathesis-official + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Fail if any Schemathesis run failed + if: always() + run: | + if [ "${{ steps.schemathesis_local.outputs.exitcode }}" != "0" ] || \ + [ "${{ steps.schemathesis_official.outputs.exitcode }}" != "0" ]; then + echo "One or more Schemathesis validations failed" + exit 1 + else + echo "Both Schemathesis validations passed" + fi + + - name: Upload Schemathesis report # This only works on git actions + if: always() && env.ACT != 'true' + uses: actions/upload-artifact@v4 + with: + if-no-files-found: warn + name: schemathesis-report + path: | + schemathesis-local.html + schemathesis-local.xml + schemathesis-official.html + schemathesis-official.xml + + - name: Save Schemathesis reports locally # This only works if run locally with act + if: always() && env.ACT == 'true' + run: | + mkdir -p artifacts + cp schemathesis-local.html schemathesis-local.xml artifacts/ || true + cp schemathesis-official.html schemathesis-official.xml artifacts/ || true + + - name: Dump API logs + if: always() + run: docker logs iri-facility-api-base || true + + - name: Stop container + if: always() + run: docker stop iri-facility-api-base || true From b3e96f7fb52720c514d1f1f4e646cc1e9ed99bb8 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:41:35 -0600 Subject: [PATCH 09/43] Add custom get extra function --- app/routers/dependencies.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py index 17b4f09..7bfaf0b 100644 --- a/app/routers/dependencies.py +++ b/app/routers/dependencies.py @@ -136,6 +136,10 @@ def _hide_extra(self, handler): data.pop(k, None) return data + def get_extra(self, key, default=None): + return getattr(self, "__pydantic_extra__", {}).get(key, default) + + class NamedObject(IRIBaseModel): id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") def _self_path(self) -> str: From 5b4d7e2cb7e56c367c24d94f64bc5c287e8cb7da Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 13:06:36 -0600 Subject: [PATCH 10/43] Implement opentelemetry. Use UTC in demo adapter --- Makefile | 1 + README.md | 2 ++ VALIDATION.MD | 23 +++++++++++++++++++++++ app/config.py | 5 +++++ app/demo_adapter.py | 31 +++++++++++++++++++++---------- app/main.py | 32 ++++++++++++++++++++++++++++++++ pyproject.toml | 8 ++++++-- 7 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 VALIDATION.MD diff --git a/Makefile b/Makefile index ba7552a..abd87e4 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ dev : .venv IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + OPENTELEMETRY_ENABLED=true \ API_URL_ROOT='http://127.0.0.1:8000' fastapi dev diff --git a/README.md b/README.md index 8160cc4..e87a916 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ If using docker (see next section), your dockerfile could extend this reference - `API_URL_ROOT`: the base url when constructing links returned by the api (eg.: https://iri.myfacility.com) - `API_PREFIX`: the path prefix where the api is hosted. Defaults to `/`. (eg.: `/api`) - `API_URL`: the path to the api itself. Defaults to `api/v1`. +- `OPENTELEMETRY_ENABLED`: Enables OpenTelemetry. If enabled, the application will use OpenTelemetry SDKs and emit traces, metrics, and logs. Default to false +- `OTLP_ENDPOINT`: OpenTelemetry Protocol collector endpoint to export telemetry data. If empty or not set, telemetry data is logged locally to log file. Default: "" Links to data, created by this api, will concatenate these values producing links, eg: `https://iri.myfacility.com/my_api_prefix/my_api_url/projects/123` diff --git a/VALIDATION.MD b/VALIDATION.MD new file mode 100644 index 0000000..26f77ef --- /dev/null +++ b/VALIDATION.MD @@ -0,0 +1,23 @@ +# API Validation with Schemathesis + +On every pull request or push to `main` branch, Github Actions run the following steps below that validates an IRI Facility API implementation against OpenAPI spec using Schemathesis. + +1. Builds the Facility API Docker image from Dockerfile. +2. Runs the API container with demo adapter. +3. Waits for `/openapi.json` to become available on localhost:8000. +4. Runs Schemathesis validation twice: + - Against Facilities API’s OpenAPI spec. (http://localhost:8000/openapi.json) + - Against the official IRI Facility API OpenAPI spec. (https://github.com/doe-iri/iri-facility-api-docs/blob/main/specification/openapi/openapi_iri_facility_api_v1.json) +5. Fails the workflow if either validation fails. +6. Saves Schemathesis HTML/XML reports as artifacts (or saves it locally when run with `act`). +7. Dumps API container logs and do clean up to stop container. + +## Running locally + +```bash +act -W .github/workflows/api-validator.yml -s GITHUB_TOKEN= +``` + +## Known issues + +Python implementation not fully aligns with the official Specification. Running against Official Spec will continue to fail, until Spec or Py implementation is fixed. diff --git a/app/config.py b/app/config.py index 078b6a5..71a7c20 100644 --- a/app/config.py +++ b/app/config.py @@ -40,3 +40,8 @@ API_URL_ROOT = os.environ.get("API_URL_ROOT", "https://api.iri.nersc.gov") API_PREFIX = os.environ.get("API_PREFIX", "/") API_URL = os.environ.get("API_URL", "api/v1") + +OPENTELEMETRY_ENABLED = os.environ.get("OPENTELEMETRY_ENABLED", "false").lower() == "true" +OPENTELEMETRY_DEBUG = os.environ.get("OPENTELEMETRY_DEBUG", "false").lower() == "true" +OTLP_ENDPOINT = os.environ.get("OTLP_ENDPOINT", "") +OTEL_SAMPLE_RATE = float(os.environ.get("OTEL_SAMPLE_RATE", "0.2")) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 0d58103..e63c720 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,7 +1,6 @@ import datetime import random import uuid -import time import os import stat import pwd @@ -45,6 +44,16 @@ def demo_uuid(kind: str, name: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) +def utc_now() -> datetime.datetime: + """Return current UTC datetime timestamp""" + return datetime.datetime.now(datetime.timezone.utc) + + +def utc_timestamp() -> int: + """Return current UTC datetime timestamp as integer""" + return int(utc_now().timestamp()) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): @@ -62,7 +71,8 @@ def __init__(self): def _init_state(self): - now = datetime.datetime.now(datetime.timezone.utc) + now = utc_now() + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -80,7 +90,8 @@ def _init_state(self): longitude=-286.36999 ) - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) + + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -352,7 +363,7 @@ async def submit_job( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -371,7 +382,7 @@ async def submit_job_script( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -390,7 +401,7 @@ async def update_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.ACTIVE, - time=time.time(), + time=utc_timestamp(), message="job updated", exit_code=None, meta_data={ "account": "account1" }, @@ -410,7 +421,7 @@ async def get_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.COMPLETED, - time=time.time(), + time=utc_timestamp(), message="job completed successfully", exit_code=0, meta_data={ "account": "account1" }, @@ -432,7 +443,7 @@ async def get_jobs( id=f"job_{i}", status=compute_models.JobStatus( state=random.choice([s for s in compute_models.JobState]), - time=time.time() - (random.random() * 100), + time=utc_timestamp() - int(random.random() * 100), message="", exit_code=random.choice([0, 0, 0, 0, 0, 1, 1, 128, 127]), meta_data={ "account": "account1" }, @@ -900,7 +911,7 @@ class DemoTaskQueue: @staticmethod async def _process_tasks(da: DemoAdapter): - now = time.time() + now = utc_timestamp() _tasks = [] for t in DemoTaskQueue.tasks: if now - t.start > 5 * 60 and t.status in [task_models.TaskStatus.completed, task_models.TaskStatus.canceled, task_models.TaskStatus.failed]: @@ -921,5 +932,5 @@ async def _process_tasks(da: DemoAdapter): @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" - DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=time.time())) + DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) return task_id diff --git a/app/main.py b/app/main.py index 7864143..fa3f1ed 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,13 @@ """Main API application""" import logging from fastapi import FastAPI +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, BatchSpanProcessor, SimpleSpanProcessor +from opentelemetry.sdk.trace.sampling import TraceIdRatioBased, ParentBased +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -13,9 +20,34 @@ from . import config +# ------------------------------------------------------------------ +# OpenTelemetry Tracing Configuration +# ------------------------------------------------------------------ +if config.OPENTELEMETRY_ENABLED: + resource = Resource.create({ + "service.name": "iri-facility-api", + "service.version": config.API_VERSION, + "service.endpoint": config.API_URL_ROOT}) + + samplerate = "1.0" if config.OPENTELEMETRY_DEBUG else config.OTEL_SAMPLE_RATE + provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(samplerate))) + trace.set_tracer_provider(provider) + + if config.OTLP_ENDPOINT: + exporter = OTLPSpanExporter(endpoint=config.OTLP_ENDPOINT, insecure=True) + span_processor = BatchSpanProcessor(exporter) + else: + exporter = ConsoleSpanExporter() + span_processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(span_processor) + tracer = trace.get_tracer(__name__) +# ------------------------------------------------------------------ APP = FastAPI(**config.API_CONFIG) +if config.OPENTELEMETRY_ENABLED: + FastAPIInstrumentor.instrument_app(APP) + install_error_handlers(APP) api_prefix = f"{config.API_PREFIX}{config.API_URL}" diff --git a/pyproject.toml b/pyproject.toml index 03858a4..1f5c0d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,9 @@ requires-python = ">=3.12" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", - "humps>=0.2.2" -] + "humps>=0.2.2", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-exporter-otlp" +] \ No newline at end of file From b697d186629e6eadd62436fdffedb536c5a1953f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 14:28:41 -0600 Subject: [PATCH 11/43] Rename dependencies to common --- app/routers/account/account.py | 2 +- app/routers/account/models.py | 2 +- app/routers/{dependencies.py => common.py} | 0 app/routers/compute/compute.py | 2 +- app/routers/compute/models.py | 2 +- app/routers/facility/facility.py | 2 +- app/routers/facility/models.py | 2 +- app/routers/status/models.py | 2 +- app/routers/status/status.py | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename app/routers/{dependencies.py => common.py} (100%) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index b2c41f4..fe16b8b 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import forbidExtraQueryParams +from ..common import forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/account/models.py b/app/routers/account/models.py index c012ee3..2158575 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,7 +1,7 @@ from pydantic import computed_field, Field import enum from ... import config -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class AllocationUnit(enum.Enum): diff --git a/app/routers/dependencies.py b/app/routers/common.py similarity index 100% rename from app/routers/dependencies.py rename to app/routers/common.py diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 7a2db23..20081ee 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -4,7 +4,7 @@ from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router -from ..dependencies import forbidExtraQueryParams, StrictBool +from ..common import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 006aabc..1b876ca 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,6 +1,6 @@ from enum import IntEnum from pydantic import field_serializer, ConfigDict, StrictBool, Field -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class ResourceSpec(IRIBaseModel): diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 1c38918..c0d3e80 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -2,7 +2,7 @@ from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5db7a18..508dbcf 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..dependencies import NamedObject +from ..common import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/status/models.py b/app/routers/status/models.py index ff5d4e4..d338e2c 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..dependencies import NamedObject +from ..common import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 2daf30e..d599866 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, From db5878cec3d4062ae257719d2e7d40001b95b626 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 09:26:32 -0600 Subject: [PATCH 12/43] Do not swallow exceptions --- app/routers/compute/compute.py | 9 ++++----- app/routers/iri_router.py | 6 ++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 20081ee..cca45be 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,3 +1,4 @@ +"""Compute resource API router""" from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router @@ -6,13 +7,13 @@ from ..status.status import router as status_router from ..common import forbidExtraQueryParams, StrictBool + router = iri_router.IriRouter( facility_adapter.FacilityAdapter, prefix="/compute", tags=["compute"], ) - @router.post( "/job/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -204,8 +205,6 @@ async def cancel_job( # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) - try: - await router.adapter.cancel_job(resource, user, job_id) - except Exception as exc: - raise HTTPException(status_code=400, detail=f"Unable to cancel job: {str(exc)}") from exc + await router.adapter.cancel_job(resource, user, job_id) + return None diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index d3fc9e1..f0b5b49 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod +import traceback import os import logging import importlib -from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from .account.models import User @@ -12,9 +12,6 @@ def get_client_ip(request : Request) -> str|None: - # logging.debug("Request headers=%s" % request.headers) - # logging.debug("client=%s" % request.client.host) - forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: return forwarded_for.split(",")[0].strip() @@ -91,6 +88,7 @@ async def current_user( user_id = await self.adapter.get_current_user(api_key, get_client_ip(request)) except Exception as exc: logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") + traceback.print_exc() raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") From df38b39fad33e5c9e37bc87fdc64c339b2c6567c Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 18:58:36 -0600 Subject: [PATCH 13/43] Ensure that computed fields are included in output --- app/routers/common.py | 14 ++++++++++++-- app/routers/status/models.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index 7bfaf0b..f46c888 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -129,13 +129,23 @@ class IRIBaseModel(BaseModel): model_config = ConfigDict(extra="allow") @model_serializer(mode="wrap") - def _hide_extra(self, handler): + def _hide_extra(self, handler, info): data = handler(self) + + model_fields = set(self.model_fields or {}) + computed_fields = set(self.model_computed_fields or {}) + print(model_fields) + print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: - data.pop(k, None) + if k not in model_fields and k not in computed_fields: + data.pop(k, None) + return data + + + def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) diff --git a/app/routers/status/models.py b/app/routers/status/models.py index d338e2c..448a7e2 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -36,7 +36,7 @@ def _self_path(self) -> str: current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - @computed_field(description="The list of past events in this incident") + @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] From a634c63d25c3a3cd415cbbfb718d8904b2b2fe92 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 09:57:21 -0600 Subject: [PATCH 14/43] Code base compliant with the official Spec --- app/demo_adapter.py | 203 +++++++++++++++++++++-- app/routers/account/account.py | 17 +- app/routers/account/facility_adapter.py | 3 +- app/routers/account/models.py | 22 +-- app/routers/common.py | 48 ++++-- app/routers/facility/facility.py | 57 +++++++ app/routers/facility/facility_adapter.py | 46 +++++ app/routers/facility/models.py | 35 +++- app/routers/status/facility_adapter.py | 6 +- app/routers/status/models.py | 2 + app/routers/status/status.py | 32 ++-- 11 files changed, 398 insertions(+), 73 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e63c720..e68635e 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -9,9 +9,10 @@ import subprocess import pathlib import base64 -from pydantic import BaseModel from typing import Any, Tuple +from pydantic import BaseModel from fastapi import HTTPException +from .routers.common import AllocationUnit, Capability from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter @@ -35,7 +36,7 @@ def get_base_temp_dir(cls): os.makedirs(cls._base_temp_dir, exist_ok=True) # create a test file - with open(f"{cls._base_temp_dir}/test.txt", "w") as f: + with open(f"{cls._base_temp_dir}/test.txt", encoding="utf-8", mode="w") as f: f.write("hello world") return cls._base_temp_dir @@ -67,12 +68,57 @@ def __init__(self): self.project_allocations = [] self.user_allocations = [] self.facility = {} + self.locations = [] + self.sites = [] self._init_state() def _init_state(self): now = utc_now() + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -81,22 +127,29 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - facility_uri="https://www.demo.example", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - street_address="1 main st", - latitude=38.410558, - longitude=-286.36999 + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], ) + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { - "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "hpss": account_models.Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), - "gpfs": account_models.Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), + "cpu": Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[AllocationUnit.node_hours]), + "gpu": Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[AllocationUnit.node_hours]), + "hpss": Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), + "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } pm = status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", capability_ids=[ @@ -226,6 +279,125 @@ async def get_facility( ) -> facility_models.Facility: return self.facility + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + + + # ---------------------------- # Status API # ---------------------------- @@ -239,6 +411,8 @@ async def get_resources( group : str | None = None, modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, + current_status : status_models.Status | None = None, + capability: Capability | None = None ) -> list[status_models.Resource]: return status_models.Resource.find(self.resources, name, description, group, modified_since, resource_type)[offset:offset + limit] @@ -288,6 +462,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: return status_models.Incident.find(self.incidents, name, description, status, type, from_, to, time_, modified_since, resource_id)[offset:offset + limit] @@ -301,7 +476,7 @@ async def get_incident( async def get_capabilities( self : "DemoAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: return self.capabilities.values() diff --git a/app/routers/account/account.py b/app/routers/account/account.py index fe16b8b..f856312 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -1,8 +1,8 @@ -from fastapi import HTTPException, Request, Depends +from fastapi import HTTPException, Request, Depends, Query from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import forbidExtraQueryParams +from ..common import forbidExtraQueryParams, StrictDateTime, Capability router = iri_router.IriRouter( @@ -22,8 +22,12 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.Capability]: + name : str = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), + ) -> list[Capability]: return await router.adapter.get_capabilities() @@ -37,8 +41,9 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.Capability: + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + ) -> Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) if not cc: diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 78b622f..235a2f7 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,5 +1,6 @@ from abc import abstractmethod from . import models as account_models +from ..common import Capability from ..iri_router import AuthenticatedAdapter @@ -13,7 +14,7 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_capabilities( self : "FacilityAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: pass diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 2158575..1a9333d 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,26 +1,6 @@ from pydantic import computed_field, Field -import enum from ... import config -from ..common import IRIBaseModel - - -class AllocationUnit(enum.Enum): - node_hours = "node_hours" - bytes = "bytes" - inodes = "inodes" - - -class Capability(IRIBaseModel): - """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) - """ - id: str - name: str - units: list[AllocationUnit] +from ..common import IRIBaseModel, AllocationUnit class User(IRIBaseModel): diff --git a/app/routers/common.py b/app/routers/common.py index f46c888..d580565 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -1,5 +1,6 @@ """Default models used by multiple routers.""" import datetime +import enum from typing import Optional from urllib.parse import parse_qs @@ -93,7 +94,11 @@ def __get_pydantic_json_schema__(cls, schema, handler): } -def forbidExtraQueryParams(*allowedParams: str): +def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): + multiParams = multiParams or set() + + print(allowedParams, multiParams) + async def checker(req: Request): if "*" in allowedParams: return @@ -110,20 +115,26 @@ async def checker(req: Request): detail=[{ "type": "extra_forbidden", "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) + "msg": f"Unexpected query parameter: {key}", + }], + ) - if len(values) > 1: + if len(values) > 1 and key not in multiParams: raise HTTPException( status_code=422, detail=[{ "type": "duplicate_forbidden", "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) + "msg": f"Duplicate query parameter: {key}", + }], + ) + return checker + + + class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") @@ -134,18 +145,12 @@ def _hide_extra(self, handler, info): model_fields = set(self.model_fields or {}) computed_fields = set(self.model_computed_fields or {}) - print(model_fields) - print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: if k not in model_fields and k not in computed_fields: data.pop(k, None) - return data - - - def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) @@ -188,3 +193,22 @@ def normalize(dt: datetime) -> datetime: modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] return a + + +class AllocationUnit(enum.Enum): + node_hours = "node_hours" + bytes = "bytes" + inodes = "inodes" + + +class Capability(IRIBaseModel): + """ + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + """ + id: str + name: str + units: list[AllocationUnit] \ No newline at end of file diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index c0d3e80..7c685e1 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -19,3 +19,60 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") +async def list_sites( + request: Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") +async def get_site( + request: Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") +async def get_site_location( + request : Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") +async def list_locations( + request : Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") +async def get_location( + request : Request, + location_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index cbee951..d316674 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -17,3 +17,49 @@ async def get_facility( ) -> facility_models.Facility | None: pass + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 508dbcf..fd164fa 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,13 +1,22 @@ """Facility-related models.""" -from typing import Optional +from typing import Optional, List from pydantic import Field, HttpUrl from ..common import NamedObject -class Facility(NamedObject): - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization's name.") - facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -16,7 +25,21 @@ class Facility(NamedObject): altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") +class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 6753a47..d7358c5 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -2,6 +2,7 @@ import datetime from fastapi import Query from . import models as status_models +from ..common import Capability class FacilityAdapter(ABC): @@ -21,7 +22,9 @@ async def get_resources( description : str | None = None, group : str | None = None, modified_since : datetime.datetime | None = None, - resource_type: status_models.ResourceType = Query(default=None) + resource_type: status_models.ResourceType = Query(default=None), + current_status: status_models.Status = Query(default=None), + capability: Capability | None = None, ) -> list[status_models.Resource]: pass @@ -75,6 +78,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 448a7e2..3650451 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -101,6 +101,7 @@ def find( class IncidentType(enum.Enum): planned = "planned" unplanned = "unplanned" + reservation = "reservation" class Resolution(enum.Enum): @@ -111,6 +112,7 @@ class Resolution(enum.Enum): pending = "pending" + class Incident(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/status/status.py b/app/routers/status/status.py index d599866..5290a8a 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,8 +1,9 @@ +from typing import Optional, List, Annotated from fastapi import HTTPException, Request, Query, Depends from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams, AllocationUnit router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -22,13 +23,16 @@ async def get_resources( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), group : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + current_status: models.Status = Query(default=None), + #event_uris: Optional[List[str]] = Query(default=None, min_length=1), + capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) @router.get( @@ -48,7 +52,7 @@ async def get_resource( return item -@router.get( +@router.get( "/incidents", summary="Get all incidents without their events", description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", @@ -66,11 +70,15 @@ async def get_incidents( to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), - _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + resolution : models.Resolution = Query(default=None), + resource_uris: Optional[List[str]] = Query(default=None, min_length=1), + event_uris: Optional[List[str]] = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", + "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) + return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) @router.get( @@ -109,8 +117,8 @@ async def get_events( time_ : StrictDateTime = Query(alias="time", default=None), to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) From bca4945eafa8bcdda363559501aea12c20aac2f7 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:08:21 -0600 Subject: [PATCH 15/43] Fully compliant with official spec --- app/routers/common.py | 28 +++++++++------------------- app/routers/status/status.py | 5 ++--- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index d580565..fd2882f 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -97,8 +97,6 @@ def __get_pydantic_json_schema__(cls, schema, handler): def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): multiParams = multiParams or set() - print(allowedParams, multiParams) - async def checker(req: Request): if "*" in allowedParams: return @@ -110,31 +108,23 @@ async def checker(req: Request): for key, values in parsed.items(): if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}"}]) + if len(values) > 1 and key not in multiParams: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}"}]) return checker - class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 5290a8a..6a0e948 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -28,11 +28,10 @@ async def get_resources( modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), current_status: models.Status = Query(default=None), - #event_uris: Optional[List[str]] = Query(default=None, min_length=1), - capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status, capability) @router.get( From a6bc9af87fa386dd9677d9508037dea0e8f99d62 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:45:23 -0600 Subject: [PATCH 16/43] Enforce Py 3.14 as used release for everything --- .github/workflows/api-validation.yml | 11 ++++++----- Dockerfile | 2 +- pyproject.toml | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index 624e84e..e4d688b 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -15,19 +15,18 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged - name: Checkout schema validator repository uses: actions/checkout@v4 with: - repository: juztas/iri-facility-api-docs - ref: schemavalidator + repository: doe-iri/iri-facility-api-docs + ref: main path: schema-validator token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" - name: Install uv run: pip install uv @@ -81,6 +80,8 @@ jobs: --report-name schemathesis-local echo "exitcode=$?" >> $GITHUB_OUTPUT + # TODO: Change back to https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json + # Once https://github.com/doe-iri/iri-facility-api-docs/pull/12 merged. - name: Run Schemathesis validation (official spec) id: schemathesis_official env: @@ -90,7 +91,7 @@ jobs: source .venv/bin/activate python schema-validator/verification/api-validator.py \ --baseurl http://localhost:8000 \ - --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --schema-url https://raw.githubusercontent.com/juztas/iri-facility-api-docs/refs/heads/newspec/specification/openapi/openapi_iri_facility_api_v1.json \ --report-name schemathesis-official echo "exitcode=$?" >> $GITHUB_OUTPUT diff --git a/Dockerfile b/Dockerfile index f3ad071..93c80d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.14 RUN mkdir /app COPY . /app diff --git a/pyproject.toml b/pyproject.toml index 1f5c0d6..6dbf14b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12" +requires-python = ">=3.12,<3.13" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", @@ -10,4 +10,4 @@ dependencies = [ "opentelemetry-sdk", "opentelemetry-instrumentation-fastapi", "opentelemetry-exporter-otlp" -] \ No newline at end of file +] From 8d830e0110e7f7a1c5ac58778fb56d52fee2c408 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 17:10:38 -0600 Subject: [PATCH 17/43] Enable deepsource scanning --- .deepsource.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 0000000..01a1066 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,9 @@ +version = 1 + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" + max_line_length = 200 From f4914e87ec34380479a9534581d3c0db21606a16 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sun, 25 Jan 2026 08:08:41 -0600 Subject: [PATCH 18/43] Enforce consistent package versions (pin major/minor version) --- pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6dbf14b..63f6c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12,<3.13" +requires-python = ">=3.14,<3.15" dependencies = [ - "fastapi[standard]>=0.100.0", - "uvicorn[standard]>=0.22.0", - "humps>=0.2.2", - "opentelemetry-api", - "opentelemetry-sdk", - "opentelemetry-instrumentation-fastapi", - "opentelemetry-exporter-otlp" -] + "fastapi[standard]>=0.128.0,<0.129.0", + "uvicorn[standard]>=0.40.0,<0.41.0", + "humps>=0.2.2,<0.3.0", + "opentelemetry-api>=1.39.1,<1.40.0", + "opentelemetry-sdk>=1.39.1,<1.40.0", + "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", + "opentelemetry-exporter-otlp>=1.39.1,<1.40.0" +] \ No newline at end of file From cba88c2c672f505f9e759253ae36afbb0c79a09b Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 19/43] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- Makefile | 1 + app/demo_adapter.py | 210 ++++++++++++++++++++++- app/main.py | 33 +++- app/routers/account/account.py | 8 + app/routers/compute/compute.py | 4 +- app/routers/error_handlers.py | 6 + app/routers/facility/__init__.py | 0 app/routers/facility/facility.py | 77 +++++++++ app/routers/facility/facility_adapter.py | 65 +++++++ app/routers/facility/models.py | 58 +++++++ app/routers/iri_router.py | 36 +++- 11 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 app/routers/facility/__init__.py create mode 100644 app/routers/facility/facility.py create mode 100644 app/routers/facility/facility_adapter.py create mode 100644 app/routers/facility/models.py diff --git a/Makefile b/Makefile index 5bd9034..ba7552a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ dev : .venv @source ./.venv/bin/activate && \ + IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9cbc796..daf266d 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -13,6 +13,7 @@ from pydantic import BaseModel from typing import Any, Tuple from fastapi import HTTPException +from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter from .routers.compute import models as compute_models, facility_adapter as compute_adapter @@ -40,9 +41,13 @@ def get_base_temp_dir(cls): return cls._base_temp_dir +def demo_uuid(kind: str, name: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, - task_adapter.FacilityAdapter): + task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): def __init__(self): self.resources = [] self.incidents = [] @@ -52,11 +57,83 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - + self.locations = [] + self.facility = {} + self.sites = [] self._init_state() def _init_state(self): + now = datetime.datetime.now(datetime.timezone.utc) + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + + facility = facility_models.Facility( + id=demo_uuid("facility", "demo_facility"), + name="Demo Facility", + description="A demo facility for testing the IRI Facility API", + last_modified=now, + short_name="DEMO", + organization_name="Demo Organization", + support_uri="https://support.demo.example", + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], + ) + + self.facility = facility + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + + day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -182,6 +259,135 @@ def _init_state(self): d += datetime.timedelta(minutes=int(random.random() * 15 + 1)) + # ---------------------------- + # Facility API + # ---------------------------- + + async def get_facility( + self: "DemoAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility: + return self.facility + + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + # ---------------------------- + # Status API + # ---------------------------- async def get_resources( self : "DemoAdapter", diff --git a/app/main.py b/app/main.py index be5feaa..080645e 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,11 @@ """Main API application""" import logging from fastapi import FastAPI +from fastapi import Request +from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers +from app.routers.facility import facility from app.routers.status import status from app.routers.account import account from app.routers.compute import compute @@ -19,11 +22,39 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" +@APP.get(api_prefix) +async def api_discovery(request: Request): + base = str(request.base_url).rstrip("/") + items = [] + for route in APP.router.routes: + if not isinstance(route, APIRoute): + continue + # skip docs & openapi + if route.path.startswith("/docs") or route.path.startswith("/openapi"): + continue + for method in route.methods: + if method == "HEAD" or method == "OPTIONS": + continue + items.append({ + "id": route.name or f"{method}_{route.path}", + "method": method, + "path": route.path, + "_links": [ + { + "rel": "self", + "href": f"{base.rstrip('/')}{route.path}", + "type": "application/json" + } + ] + }) + return items + # Attach routers under the prefix +APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) APP.include_router(account.router, prefix=api_prefix) APP.include_router(compute.router, prefix=api_prefix) APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") +logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 951fc9b..508d556 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -21,6 +21,7 @@ ) async def get_capabilities( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -35,6 +36,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -53,6 +55,7 @@ async def get_capability( ) async def get_projects( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -71,6 +74,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -93,6 +97,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -116,6 +121,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -141,6 +147,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -169,6 +176,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index e7481ac..72a8169 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -154,8 +154,8 @@ async def get_job_status( async def get_job_statuses( resource_id : str, request : Request, - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=10000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, historical : bool = False, include_spec: bool = False, diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 09769ec..337b5fc 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -65,6 +65,12 @@ async def validation_error_handler(request: Request, exc: RequestValidationError @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 304: + return JSONResponse( + status_code=304, + content=None, + headers=exc.headers or {}) + if exc.status_code == 401: return problem_response( request=request, diff --git a/app/routers/facility/__init__.py b/app/routers/facility/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py new file mode 100644 index 0000000..9964925 --- /dev/null +++ b/app/routers/facility/facility.py @@ -0,0 +1,77 @@ +from fastapi import Request, HTTPException, Depends, Query +from .. import iri_router +from ..error_handlers import DEFAULT_RESPONSES +from .import models, facility_adapter + + +router = iri_router.IriRouter( + facility_adapter.FacilityAdapter, + prefix="/facility", + tags=["facility"], +) + +@router.get("", responses=DEFAULT_RESPONSES) +async def get_facility( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + ) -> models.Facility: + """Get facility information""" + return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES) +async def list_sites( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +async def get_site( + request: Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +async def get_site_location( + request : Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES) +async def list_locations( + request : Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +async def get_location( + request : Request, + location_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py new file mode 100644 index 0000000..d316674 --- /dev/null +++ b/app/routers/facility/facility_adapter.py @@ -0,0 +1,65 @@ +from abc import abstractmethod +from . import models as facility_models +from ..iri_router import AuthenticatedAdapter + + +class FacilityAdapter(AuthenticatedAdapter): + """ + Facility-specific code is handled by the implementation of this interface. + Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) + to install your facility adapter before the API starts. + """ + + @abstractmethod + async def get_facility( + self: "FacilityAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility | None: + pass + + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py new file mode 100644 index 0000000..0c04cc4 --- /dev/null +++ b/app/routers/facility/models.py @@ -0,0 +1,58 @@ +from datetime import datetime +from uuid import UUID +from typing import List, Optional +from pydantic import BaseModel, Field, HttpUrl, computed_field +from .. import iri_router +from ... import config + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") + country_name: Optional[str] = Field(None, description="Country name of the Location.") + locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") + state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") + street_address: Optional[str] = Field(None, description="Street address of the Location.") + unlocode: Optional[str] = Field(None, description="United Nations trade and transport location code.") + altitude: Optional[float] = Field(None, description="Altitude of the Location.") + latitude: Optional[float] = Field(None, description="Latitude of the Location.") + longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + +class Facility(NamedObject): + def _self_path(self) -> str: + return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index dafb970..2de7e69 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -3,11 +3,13 @@ import logging import importlib import datetime +from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from pydantic_core import core_schema from .account.models import User + bearer_token = APIKeyHeader(name="Authorization") @@ -128,17 +130,33 @@ async def get_user( def forbidExtraQueryParams(*allowedParams: str): - """Dependency to forbid extra query parameters not in allowedParams.""" - - async def checker(_req: Request): + async def checker(req: Request): if "*" in allowedParams: - return # Permit anything - incoming = set(_req.query_params.keys()) + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + allowed = set(allowedParams) - unknown = incoming - allowed - if unknown: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", "loc": ["query", param], "msg": f"Unexpected query parameter: {param}"} for param in unknown]) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) return checker class StrictDateTime: From 6f44f6742409c85de32acfac714145f070c39db0 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 07:50:47 -0600 Subject: [PATCH 20/43] Make NamedObject reusable --- app/routers/facility/models.py | 22 ++------- app/routers/models.py | 46 +++++++++++++++++++ app/routers/status/models.py | 82 +++++++++------------------------- 3 files changed, 69 insertions(+), 81 deletions(-) create mode 100644 app/routers/models.py diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 0c04cc4..2bea62f 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,23 +1,7 @@ -from datetime import datetime -from uuid import UUID +"""Facility-related models.""" from typing import List, Optional -from pydantic import BaseModel, Field, HttpUrl, computed_field -from .. import iri_router -from ... import config - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - +from pydantic import Field, HttpUrl +from ..models import NamedObject class Site(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/models.py b/app/routers/models.py new file mode 100644 index 0000000..f9a1c00 --- /dev/null +++ b/app/routers/models.py @@ -0,0 +1,46 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from pydantic import BaseModel, Field, computed_field +from . import iri_router +from .. import config + + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index b1f3a8b..2c7dc76 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,6 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config +from ..models import NamedObject class Link(BaseModel): rel : str @@ -15,38 +16,6 @@ class Status(enum.Enum): unknown = "unknown" -class NamedResource(BaseModel): - id : str - name : str - description : str - last_modified : datetime.datetime - - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a - - class ResourceType(enum.Enum): website = "website" service = "service" @@ -57,28 +26,24 @@ class ResourceType(enum.Enum): unknown = "unknown" -class Resource(NamedResource): +class Resource(NamedObject): + + def _self_path(self) -> str: + return f"/status/resources/{self.id}" + capability_ids: list[str] = Field(exclude=True) group: str | None current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] - @staticmethod def find(resources, name, description, group, modified_since, resource_type): - a = NamedResource.find(resources, name, description, modified_since) + a = NamedObject.find(resources, name, description, modified_since) if group: a = [aa for aa in a if aa.group == group] if resource_type: @@ -86,25 +51,21 @@ def find(resources, name, description, group, modified_since, resource_type): return a -class Event(NamedResource): +class Event(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.incident_id}/events/{self.id}" + occurred_at : datetime.datetime status : Status resource_id : str = Field(exclude=True) incident_id : str | None = Field(exclude=True, default=None) - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}/events/{self.id}" - - @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" - @computed_field(description="The event's incident") @property def incident_uri(self) -> str|None: @@ -123,7 +84,7 @@ def find( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, ) -> list: - events = NamedResource.find(events, name, description, modified_since) + events = NamedObject.find(events, name, description, modified_since) if resource_id: events = [e for e in events if e.resource_id == resource_id] if status: @@ -150,7 +111,11 @@ class Resolution(enum.Enum): pending = "pending" -class Incident(NamedResource): +class Incident(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.id}" + status : Status resource_ids : list[str] = Field(exclude=True) event_ids : list[str] = Field(exclude=True) @@ -159,25 +124,18 @@ class Incident(NamedResource): type : IncidentType resolution : Resolution - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] - @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] def find( + self, incidents : list, name : str | None = None, description : str | None = None, @@ -189,7 +147,7 @@ def find( modified_since : datetime.datetime | None = None, resource_id : str | None = None, ) -> list: - incidents = NamedResource.find(incidents, name, description, modified_since) + incidents = NamedObject.find(incidents, name, description, modified_since) if resource_id: incidents = [e for e in incidents if resource_id in e.resource_ids] if status: From fbab525d7fa52cfdf617c3a83a49a6e29a01ca4e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 08:16:21 -0600 Subject: [PATCH 21/43] Include operation_id for facility (similar to pull request #21) --- app/routers/facility/facility.py | 12 ++++++------ app/routers/status/models.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 9964925..0e6f960 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -10,7 +10,7 @@ tags=["facility"], ) -@router.get("", responses=DEFAULT_RESPONSES) +@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -19,7 +19,7 @@ async def get_facility( """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) -@router.get("/sites", responses=DEFAULT_RESPONSES) +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") async def list_sites( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -32,7 +32,7 @@ async def list_sites( """List sites""" return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") async def get_site( request: Request, site_id: str, @@ -42,7 +42,7 @@ async def get_site( """Get site by ID""" return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") async def get_site_location( request : Request, site_id: str, @@ -52,7 +52,7 @@ async def get_site_location( """Get site location by site ID""" return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) -@router.get("/locations", responses=DEFAULT_RESPONSES) +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") async def list_locations( request : Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -66,7 +66,7 @@ async def list_locations( """List locations""" return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") async def get_location( request : Request, location_id: str, diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 2c7dc76..bb1f87b 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -134,8 +134,8 @@ def event_uris(self) -> list[str]: def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] + @staticmethod def find( - self, incidents : list, name : str | None = None, description : str | None = None, From 7ca9b7565c7fc84a7f310fe126d7133523f60a11 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Thu, 15 Jan 2026 11:48:45 -0800 Subject: [PATCH 22/43] simplified facility endpoint proposal --- app/demo_adapter.py | 184 +---------------------- app/routers/facility/facility.py | 57 ------- app/routers/facility/facility_adapter.py | 47 ------ app/routers/facility/models.py | 33 +--- 4 files changed, 14 insertions(+), 307 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index daf266d..2d2ca30 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -65,50 +65,7 @@ def __init__(self): def _init_state(self): now = datetime.datetime.now(datetime.timezone.utc) - loc1 = facility_models.Location( - id=demo_uuid("location", "demo_location_1"), - name="Demo Location 1", - description="The first demo location", - last_modified=now, - short_name="DL1", - country_name="USA", - locality_name="Demo City", - state_or_province_name="DC", - latitude=36.173357, - longitude=-234.51452) - - loc2 = facility_models.Location( - id=demo_uuid("location", "demo_location_2"), - name="Demo Location 2", - description="The second demo location", - last_modified=now, - short_name="DL2", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - latitude=38.410558, - longitude=-286.36999) - - site1 = facility_models.Site( - id=demo_uuid("site", "demo_site_1"), - name="Demo Site 1", - description="The first demo site", - last_modified=now, - short_name="DS1", - operating_organization="Demo Org", - location_uri=loc1.self_uri, - resource_uris=[]) - site2 = facility_models.Site( - id=demo_uuid("site", "demo_site_2"), - name="Demo Site 2", - description="The second demo site", - last_modified=now, - short_name="DS2", - operating_organization="Demo Org", - location_uri=loc2.self_uri, - resource_uris=[]) - - facility = facility_models.Facility( + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", description="A demo facility for testing the IRI Facility API", @@ -116,24 +73,15 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri], - location_uris=[loc1.self_uri, loc2.self_uri], - resource_uris=[], - event_uris=[], - incident_uris=[], - capability_uris=[], - project_uris=[], - project_allocation_uris=[], - user_allocation_uris=[], + facility_uri="https://www.demo.example", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + street_address="1 main st", + latitude=38.410558, + longitude=-286.36999 ) - self.facility = facility - loc1.site_uris.append(site1.self_uri) - loc2.site_uris.append(site2.self_uri) - self.locations = [loc1, loc2] - self.sites = [site1, site2] - - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -269,122 +217,6 @@ async def get_facility( ) -> facility_models.Facility: return self.facility - - async def list_sites( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - - sites = self.sites - - if name: - sites = [s for s in sites if name.lower() in s.name.lower()] - - if short_name: - sites = [s for s in sites if s.short_name == short_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - sites = [s for s in sites if s.last_modified > ms] - - o = offset or 0 - l = limit or len(sites) - return sites[o:o+l] - - - async def get_site( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site: - - site = next((s for s in self.sites if s.id == site_id), None) - if not site: - raise HTTPException(status_code=404, detail="Site not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if site.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) - - return site - - - async def get_site_location( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - site = await self.get_site(site_id) - - if not site.location_uri: - raise HTTPException(status_code=404, detail="Site has no location") - - location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - - return location - - - async def list_locations( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - - locs = self.locations - - if name: - locs = [l for l in locs if name.lower() in l.name.lower()] - - if short_name: - locs = [l for l in locs if l.short_name == short_name] - - if country_name: - locs = [l for l in locs if l.country_name == country_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - locs = [l for l in locs if l.last_modified > ms] - - o = offset or 0 - l = limit or len(locs) - return locs[o:o+l] - - - async def get_location( - self: "DemoAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - location = next((l for l in self.locations if l.id == location_id), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - return location - # ---------------------------- # Status API # ---------------------------- diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 0e6f960..9b7545b 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,60 +18,3 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") -async def list_sites( - request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), - )-> list[models.Site]: - """List sites""" - return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) - -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") -async def get_site( - request: Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Site: - """Get site by ID""" - return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) - -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") -async def get_site_location( - request : Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get site location by site ID""" - return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) - -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") -async def list_locations( - request : Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - country_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), - )-> list[models.Location]: - """List locations""" - return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) - -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") -async def get_location( - request : Request, - location_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get location by ID""" - return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674..bccdcfe 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,50 +16,3 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass - - @abstractmethod - async def list_sites( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - pass - - @abstractmethod - async def get_site( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site | None: - pass - - @abstractmethod - async def get_site_location( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass - - @abstractmethod - async def list_locations( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - pass - - @abstractmethod - async def get_location( - self: "FacilityAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 2bea62f..efd93dd 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,20 +1,13 @@ """Facility-related models.""" -from typing import List, Optional +from typing import Optional from pydantic import Field, HttpUrl from ..models import NamedObject -class Site(NamedObject): - def _self_path(self) -> str: - return f"/facility/sites/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Site.") - operating_organization: str = Field(..., description="Organization operating the Site.") - location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - -class Location(NamedObject): - def _self_path(self) -> str: - return f"/facility/locations/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Location.") +class Facility(NamedObject): + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization's name.") + facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -23,20 +16,6 @@ def _self_path(self) -> str: altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") -class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization’s name.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") - location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") - event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") - incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") - capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") - project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") - project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") - user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") From 936cf356ced3327ad858c33092eadd635ad66542 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 23/43] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- app/demo_adapter.py | 2 -- app/routers/facility/facility.py | 1 + app/routers/facility/facility_adapter.py | 1 + app/routers/facility/models.py | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 2d2ca30..0d58103 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -57,9 +57,7 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - self.locations = [] self.facility = {} - self.sites = [] self._init_state() diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 9b7545b..fdba124 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,3 +18,4 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index bccdcfe..cbee951 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,3 +16,4 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass + diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index efd93dd..5b21d8a 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -19,3 +19,4 @@ class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + From 58d560298850998473658a24cf972ff4f9b57eb9 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 15:27:00 -0600 Subject: [PATCH 24/43] Remove /api/v1 --- app/main.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/app/main.py b/app/main.py index 080645e..7864143 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,6 @@ """Main API application""" import logging from fastapi import FastAPI -from fastapi import Request -from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -22,33 +20,6 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" -@APP.get(api_prefix) -async def api_discovery(request: Request): - base = str(request.base_url).rstrip("/") - items = [] - for route in APP.router.routes: - if not isinstance(route, APIRoute): - continue - # skip docs & openapi - if route.path.startswith("/docs") or route.path.startswith("/openapi"): - continue - for method in route.methods: - if method == "HEAD" or method == "OPTIONS": - continue - items.append({ - "id": route.name or f"{method}_{route.path}", - "method": method, - "path": route.path, - "_links": [ - { - "rel": "self", - "href": f"{base.rstrip('/')}{route.path}", - "type": "application/json" - } - ] - }) - return items - # Attach routers under the prefix APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) @@ -57,4 +28,4 @@ async def api_discovery(request: Request): APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file +logging.getLogger().info(f"API path: {api_prefix}") From 630cd619cb64709e2490bba33f1c07f909a010c5 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:14:23 -0600 Subject: [PATCH 25/43] Refactor shared validators & models to fix import loading issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move common validators and base models into routers/dependencies and update imports across routers and schemas to use the new shared location. This keeps API behavior unchanged. No functional API changes — purely structural and validation hygiene: --- app/routers/account/account.py | 17 +- app/routers/account/models.py | 15 +- app/routers/compute/compute.py | 88 ++++++----- app/routers/compute/models.py | 70 ++++---- app/routers/dependencies.py | 176 +++++++++++++++++++++ app/routers/facility/facility.py | 8 +- app/routers/facility/models.py | 2 +- app/routers/filesystem/facility_adapter.py | 4 +- app/routers/filesystem/filesystem.py | 42 +---- app/routers/filesystem/models.py | 9 +- app/routers/iri_router.py | 77 --------- app/routers/models.py | 46 ------ app/routers/status/models.py | 2 +- app/routers/status/status.py | 25 +-- app/routers/task/models.py | 19 ++- 15 files changed, 317 insertions(+), 283 deletions(-) create mode 100644 app/routers/dependencies.py delete mode 100644 app/routers/models.py diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 508d556..b2c41f4 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import forbidExtraQueryParams router = iri_router.IriRouter( @@ -21,7 +22,7 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -36,7 +37,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -55,7 +56,7 @@ async def get_capability( ) async def get_projects( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -74,7 +75,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -97,7 +98,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -121,7 +122,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -147,7 +148,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -176,7 +177,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 6ed69ea..c012ee3 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, computed_field, Field +from pydantic import computed_field, Field import enum from ... import config +from ..dependencies import IRIBaseModel class AllocationUnit(enum.Enum): @@ -9,7 +10,7 @@ class AllocationUnit(enum.Enum): inodes = "inodes" -class Capability(BaseModel): +class Capability(IRIBaseModel): """ An aspect of a resource that can have an allocation. For example, Perlmutter nodes with GPUs @@ -22,7 +23,7 @@ class Capability(BaseModel): units: list[AllocationUnit] -class User(BaseModel): +class User(IRIBaseModel): """A user of the facility""" id: str name: str @@ -31,7 +32,7 @@ class User(BaseModel): # we could expose more fields here (eg. email) but it might be against policy -class Project(BaseModel): +class Project(IRIBaseModel): """A project and its users at a facility""" id: str name: str @@ -39,14 +40,14 @@ class Project(BaseModel): user_ids: list[str] -class AllocationEntry(BaseModel): +class AllocationEntry(IRIBaseModel): """Base class for allocations.""" allocation: float # how much this allocation can spend usage: float # how much this allocation has spent unit: AllocationUnit -class ProjectAllocation(BaseModel): +class ProjectAllocation(IRIBaseModel): """ A project's allocation for a capability. (aka. repo) This allocation is a piece of the total allocation for the capability. (eg. 5% of the total node hours of Perlmutter GPU nodes) @@ -71,7 +72,7 @@ def capability_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{self.capability_id}" -class UserAllocation(BaseModel): +class UserAllocation(IRIBaseModel): """ A user's allcation in a project. This allocation is a piece of the project's allocation. diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 72a8169..7a2db23 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,9 +1,10 @@ -from typing import List, Annotated -from fastapi import HTTPException, Request, Depends, status, Form, Query +from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router + from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router +from ..dependencies import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -24,6 +25,7 @@ async def submit_job( resource_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Submit a job on a compute resource @@ -45,39 +47,41 @@ async def submit_job( return await router.adapter.submit_job(resource, user, job_spec) -@router.post( - "/job/script/{resource_id:str}", - dependencies=[Depends(router.current_user)], - response_model=models.Job, - response_model_exclude_unset=True, - responses=DEFAULT_RESPONSES, - operation_id="launchJobScript", -) -async def submit_job_path( - resource_id: str, - job_script_path : str, - request : Request, - args : Annotated[List[str], Form()] = [], - ): - """ - Submit a job on a compute resource - - - **resource**: the name of the compute resource to use - - **job_script_path**: path to the job script on the compute resource - - **args**: optional arguments to the job script - - This command will attempt to submit a job and return its id. - """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - - # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) - - # the handler can use whatever means it wants to submit the job and then fill in its id - # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job_script(resource, user, job_script_path, args) +# TODO: this conflicts with PUT commented out while we finalize the API design +#@router.post( +# "/job/script/{resource_id:str}", +# dependencies=[Depends(router.current_user)], +# response_model=models.Job, +# response_model_exclude_unset=True, +# responses=DEFAULT_RESPONSES, +# operation_id="launchJobScript", +#) +#async def submit_job_path( +# resource_id: str, +# job_script_path : str, +# request : Request, +# args : Annotated[List[str], Form()] = [], +# _forbid = Depends(iri_router.forbidExtraQueryParams("job_script_path")), +# ): +# """ +# Submit a job on a compute resource +# +# - **resource**: the name of the compute resource to use +# - **job_script_path**: path to the job script on the compute resource +# - **args**: optional arguments to the job script +# +# This command will attempt to submit a job and return its id. +# """ +# user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) +# if not user: +# raise HTTPException(status_code=404, detail="User not found") +# +# # look up the resource (todo: maybe ensure it's available) +# resource = await status_router.adapter.get_resource(resource_id) +# +# # the handler can use whatever means it wants to submit the job and then fill in its id +# # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs +# return await router.adapter.submit_job_script(resource, user, job_script_path, args) @router.put( @@ -93,6 +97,7 @@ async def update_job( job_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Update a previously submitted job for a resource. @@ -126,8 +131,9 @@ async def get_job_status( resource_id : str, job_id : str, request : Request, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -149,7 +155,7 @@ async def get_job_status( response_model=list[models.Job], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, - operation_id="getJobs", + operation_id="getAllJobs", ) async def get_job_statuses( resource_id : str, @@ -157,8 +163,9 @@ async def get_job_statuses( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -187,6 +194,7 @@ async def cancel_job( resource_id : str, job_id : str, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """Cancel a job""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 167643e..a56d4fe 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,41 +1,42 @@ from typing import Annotated -from pydantic import BaseModel, field_serializer, ConfigDict, Field from enum import IntEnum +from pydantic import field_serializer, ConfigDict, StrictBool, Field +from ..common import IRIBaseModel -class ResourceSpec(BaseModel): +class ResourceSpec(IRIBaseModel): """ Specification of computational resources required for a job. """ - node_count: Annotated[int | None, Field(description="Number of compute nodes to allocate")] = None - process_count: Annotated[int | None, Field(description="Total number of processes to launch")] = None - processes_per_node: Annotated[int | None, Field(description="Number of processes to launch per node")] = None - cpu_cores_per_process: Annotated[int | None, Field(description="Number of CPU cores to allocate per process")] = None - gpu_cores_per_process: Annotated[int | None, Field(description="Number of GPU cores to allocate per process")] = None - exclusive_node_use: Annotated[bool, Field(description="Whether to request exclusive use of allocated nodes")] = True - memory: Annotated[int | None, Field(description="Amount of memory to allocate in bytes")] = None + node_count: Annotated[int | None, Field(ge=1, description="Number of compute nodes to allocate")] = None + process_count: Annotated[int | None, Field(ge=1, description="Total number of processes to launch")] = None + processes_per_node: Annotated[int | None, Field(ge=1, description="Number of processes to launch per node")] = None + cpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of CPU cores to allocate per process")] = None + gpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of GPU cores to allocate per process")] = None + exclusive_node_use: Annotated[StrictBool, Field(description="Whether to request exclusive use of allocated nodes")] = True + memory: Annotated[int | None, Field(ge=1,description="Amount of memory to allocate in bytes")] = None -class JobAttributes(BaseModel): +class JobAttributes(IRIBaseModel): """ Additional attributes and scheduling parameters for a job. """ - duration: Annotated[int | None, Field(description="Duration in seconds", ge=0, examples=[30, 60, 120])] = None - queue_name: Annotated[str | None, Field(description="Name of the queue or partition to submit the job to")] = None - account: Annotated[str | None, Field(description="Account or project to charge for resource usage")] = None - reservation_id: Annotated[str | None, Field(description="ID of a reservation to use for the job")] = None + duration: Annotated[int | None, Field(description="Duration in seconds", ge=1, examples=[30, 60, 120])] = None + queue_name: Annotated[str | None, Field(min_length=1, description="Name of the queue or partition to submit the job to")] = None + account: Annotated[str | None, Field(min_length=1, description="Account or project to charge for resource usage")] = None + reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} -class VolumeMount(BaseModel): +class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ - source: Annotated[str, Field(description="The source path on the host system to mount")] - target: Annotated[str, Field(description="The target path inside the container where the volume will be mounted")] - read_only: Annotated[bool, Field(description="Whether the mount should be read-only")] = True + source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] + target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] + read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True -class Container(BaseModel): +class Container(IRIBaseModel): """ Represents a container specification for job execution. @@ -44,33 +45,33 @@ class Container(BaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ - image: Annotated[str, Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] + image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] -class JobSpec(BaseModel): +class JobSpec(IRIBaseModel): """ Specification for job. """ model_config = ConfigDict(extra="forbid") - executable: Annotated[str | None, Field(description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None + executable: Annotated[str | None, Field(min_length=1, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None container: Annotated[Container | None, Field(description="Container specification for containerized execution")] = None arguments: Annotated[list[str], Field(description="Command-line arguments to pass to the executable or container")] = [] - directory: Annotated[str | None, Field(description="Working directory for the job")] = None - name: Annotated[str | None, Field(description="Name of the job")] = None - inherit_environment: Annotated[bool, Field(description="Whether to inherit the environment variables from the submission environment")] = True + directory: Annotated[str | None, Field(min_length=1, description="Working directory for the job")] = None + name: Annotated[str | None, Field(min_length=1, description="Name of the job")] = None + inherit_environment: Annotated[StrictBool, Field(description="Whether to inherit the environment variables from the submission environment")] = True environment: Annotated[dict[str, str], Field(description="Environment variables to set for the job. If container is specified, these will be set inside the container.")] = {} - stdin_path: Annotated[str | None, Field(description="Path to file to use as standard input")] = None - stdout_path: Annotated[str | None, Field(description="Path to file to write standard output")] = None - stderr_path: Annotated[str | None, Field(description="Path to file to write standard error")] = None + stdin_path: Annotated[str | None, Field(min_length=1, description="Path to file to use as standard input")] = None + stdout_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard output")] = None + stderr_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard error")] = None resources: Annotated[ResourceSpec | None, Field(description="Resource requirements for the job")] = None attributes: Annotated[JobAttributes | None, Field(description="Additional job attributes such as duration, queue, and account")] = None - pre_launch: Annotated[str | None, Field(description="Script or commands to run before launching the job")] = None - post_launch: Annotated[str | None, Field(description="Script or commands to run after the job completes")] = None - launcher: Annotated[str | None, Field(description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None + pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None + post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None + launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None -class CommandResult(BaseModel): +class CommandResult(IRIBaseModel): status : str result : str | None = None @@ -110,20 +111,19 @@ class JobState(IntEnum): """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" -class JobStatus(BaseModel): +class JobStatus(IRIBaseModel): state : JobState time : float | None = None message : str | None = None exit_code : int | None = None meta_data : dict[str, object] | None = None - @field_serializer('state') def serialize_state(self, state: JobState): return state.name -class Job(BaseModel): +class Job(IRIBaseModel): id : str status : JobStatus | None = None job_spec : JobSpec | None = None diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py new file mode 100644 index 0000000..17b4f09 --- /dev/null +++ b/app/routers/dependencies.py @@ -0,0 +1,176 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from urllib.parse import parse_qs + +from pydantic_core import core_schema +from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer +from fastapi import Request, HTTPException + +from .. import config + + +# These are Pydantic custom types for strict validation +# that are not implmented in Pydantic by default. +# ----------------------------------------------------------------------- +# StrictBool: a strict boolean type +class StrictBool: + """Strict boolean: + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + v = value.strip().lower() + if v == "true": + return True + if v == "false": + return False + raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") + raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "boolean", + "description": "Strict boolean. Only true/false allowed (bool or string)." + } + +# ----------------------------------------------------------------------- +# StrictDateTime: a strict ISO8601 datetime type + +class StrictDateTime: + """ + Strict ISO8601 datetime: + - Accepts datetime objects + - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 + - Converts 'Z' → UTC + - Converts naive datetimes → UTC + - Rejects integers ("0"), null, garbage strings, etc. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + if isinstance(value, datetime.datetime): + return StrictDateTime._normalize(value) + if not isinstance(value, str): + raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") + v = value.strip() + if v.endswith("Z"): + v = v[:-1] + "+00:00" + try: + dt = datetime.datetime.fromisoformat(v) + except Exception as ex: + raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex + + return StrictDateTime._normalize(dt) + + @staticmethod + def _normalize(dt: datetime.datetime) -> datetime.datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "string", + "format": "date-time", + "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." + } + + +def forbidExtraQueryParams(*allowedParams: str): + async def checker(req: Request): + if "*" in allowedParams: + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + + allowed = set(allowedParams) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) + return checker + + +class IRIBaseModel(BaseModel): + """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _hide_extra(self, handler): + data = handler(self) + extra = getattr(self, "__pydantic_extra__", {}) or {} + for k in extra: + data.pop(k, None) + return data + +class NamedObject(IRIBaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index fdba124..1c38918 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,7 +1,8 @@ -from fastapi import Request, HTTPException, Depends, Query +from fastapi import Request, Depends, Query from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( @@ -13,9 +14,8 @@ @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5b21d8a..5db7a18 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..models import NamedObject +from ..dependencies import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 2c08a3c..a70efb0 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -1,15 +1,15 @@ import os from abc import abstractmethod +from typing import Any, Tuple from ..status import models as status_models from ..account import models as account_models from . import models as filesystem_models from ..iri_router import AuthenticatedAdapter -from typing import Any, Tuple def to_int(name, default_value): try: - return os.environ.get(name) or default_value + return int(os.environ.get(name) or default_value) except: return default_value diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index a111484..d583c64 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -354,40 +354,13 @@ async def get_view( resource_id: str, request : Request, path: Annotated[str, Query(description="File path")], - size: Annotated[ - int, - Query( - alias="size", - description="Value, in bytes, of the size of data to be retrieved from the file.", - ), - ] = facility_adapter.OPS_SIZE_LIMIT, - offset: Annotated[ - int, - Query( - alias="offset", - description="Value in bytes of the offset.", - ), - ] = 0, -) -> str: - user, resource = await _user_resource(resource_id, request) - if offset < 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`offset` value must be an integer value equal or greater than 0", - ) - - if size <= 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`size` value must be an integer value greater than 0", - ) + size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", + ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, - if size > facility_adapter.OPS_SIZE_LIMIT: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"`size` value must be less than {facility_adapter.OPS_SIZE_LIMIT} bytes", - ) + offset: Annotated[int, Query( description="Value in bytes of the offset.", ge=0)] = 0 + ) -> str: + user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user, @@ -397,9 +370,8 @@ async def get_view( command="view", args={ "path": path, - "size": size or facility_adapter.OPS_SIZE_LIMIT, - "offset": offset or 0, - + "size": size, + "offset": offset, } ) ) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index b24c908..d32ad94 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -12,11 +12,10 @@ class CompressionType(str, Enum): - none = "none" - bzip2 = "bzip2" - gzip = "gzip" - xz = "xz" - + none = "none" + bzip2 = "bzip2" + gzip = "gzip" + xz = "xz" class ContentUnit(str, Enum): lines = "lines" diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 2de7e69..d3fc9e1 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -2,11 +2,9 @@ import os import logging import importlib -import datetime from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader -from pydantic_core import core_schema from .account.models import User @@ -127,78 +125,3 @@ async def get_user( Retrieve additional user information (name, email, etc.) for the given user_id. """ pass - - -def forbidExtraQueryParams(*allowedParams: str): - async def checker(req: Request): - if "*" in allowedParams: - return - - raw_qs = req.scope.get("query_string", b"") - parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) - - allowed = set(allowedParams) - - for key, values in parsed.items(): - if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) - - if len(values) > 1: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) - return checker - -class StrictDateTime: - """ - Strict ISO8601 datetime: - ✔ Accepts datetime objects - ✔ Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 - ✔ Converts 'Z' → UTC - ✔ Converts naive datetimes → UTC - ✘ Rejects integers ("0"), null, garbage strings, etc. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - if isinstance(value, datetime.datetime): - return StrictDateTime._normalize(value) - if not isinstance(value, str): - raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") - v = value.strip() - if v.endswith("Z"): - v = v[:-1] + "+00:00" - try: - dt = datetime.datetime.fromisoformat(v) - except Exception as ex: - raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex - - return StrictDateTime._normalize(dt) - - @staticmethod - def _normalize(dt: datetime.datetime) -> datetime.datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } diff --git a/app/routers/models.py b/app/routers/models.py deleted file mode 100644 index f9a1c00..0000000 --- a/app/routers/models.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Default models used by multiple routers.""" -import datetime -from typing import Optional -from pydantic import BaseModel, Field, computed_field -from . import iri_router -from .. import config - - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index bb1f87b..ff5d4e4 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..models import NamedObject +from ..dependencies import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 7bb0fd0..2daf30e 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -23,9 +24,9 @@ async def get_resources( group : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - modified_since: iri_router.StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), ) -> list[models.Resource]: return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) @@ -60,14 +61,14 @@ async def get_incidents( description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), type_: models.IncidentType = Query(alias="type", default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), ) -> list[models.Incident]: return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) @@ -104,13 +105,13 @@ async def get_events( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) diff --git a/app/routers/task/models.py b/app/routers/task/models.py index da3c5cc..cea9787 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,19 +1,18 @@ -from pydantic import BaseModel import enum +from pydantic import BaseModel class TaskStatus(str, enum.Enum): - pending = "pending" - active = "active" - completed = "completed" - failed = "failed" - canceled = "canceled" - + pending = "pending" + active = "active" + completed = "completed" + failed = "failed" + canceled = "canceled" class TaskCommand(BaseModel): - router: str - command: str - args: dict + router: str + command: str + args: dict class Task(BaseModel): From 2cf089b78105abfb2d8b185e3df46cf0769e7b3f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:17:00 -0600 Subject: [PATCH 26/43] Github Action to validate api --- .github/workflows/api-validation.yml | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/api-validation.yml diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml new file mode 100644 index 0000000..624e84e --- /dev/null +++ b/.github/workflows/api-validation.yml @@ -0,0 +1,133 @@ +name: API Validation with Schemathesis + +on: + pull_request: + push: + branches: [ main ] + +jobs: + schemathesis: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged + - name: Checkout schema validator repository + uses: actions/checkout@v4 + with: + repository: juztas/iri-facility-api-docs + ref: schemavalidator + path: schema-validator + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: pip install uv + + - name: Build an image + run: docker build --platform=linux/amd64 -t iri-facility-api-base . + + - name: Run Facility API container + run: | + docker run -d \ + -p 8000:8000 \ + --platform=linux/amd64 \ + --name iri-facility-api-base \ + -e IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + -e API_URL_ROOT=http://127.0.0.1:8000 \ + -e IRI_API_TOKEN=12345 \ + iri-facility-api-base + + - name: Wait for API to be ready + run: | + for i in {1..60}; do + if curl -fs http://127.0.0.1:8000/openapi.json; then + echo "API ready" + exit 0 + fi + sleep 2 + done + echo "API did not start" + exit 1 + + - name: Create venv & install validator dependencies + run: | + uv venv + source .venv/bin/activate + uv pip install -r schema-validator/verification/requirements.txt + + - name: Run Schemathesis validation (local spec) + id: schemathesis_local + env: + IRI_API_TOKEN: "12345" # This is dummy token for testing (mock adapter) + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://127.0.0.1:8000 \ + --report-name schemathesis-local + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Run Schemathesis validation (official spec) + id: schemathesis_official + env: + IRI_API_TOKEN: "12345" + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://localhost:8000 \ + --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --report-name schemathesis-official + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Fail if any Schemathesis run failed + if: always() + run: | + if [ "${{ steps.schemathesis_local.outputs.exitcode }}" != "0" ] || \ + [ "${{ steps.schemathesis_official.outputs.exitcode }}" != "0" ]; then + echo "One or more Schemathesis validations failed" + exit 1 + else + echo "Both Schemathesis validations passed" + fi + + - name: Upload Schemathesis report # This only works on git actions + if: always() && env.ACT != 'true' + uses: actions/upload-artifact@v4 + with: + if-no-files-found: warn + name: schemathesis-report + path: | + schemathesis-local.html + schemathesis-local.xml + schemathesis-official.html + schemathesis-official.xml + + - name: Save Schemathesis reports locally # This only works if run locally with act + if: always() && env.ACT == 'true' + run: | + mkdir -p artifacts + cp schemathesis-local.html schemathesis-local.xml artifacts/ || true + cp schemathesis-official.html schemathesis-official.xml artifacts/ || true + + - name: Dump API logs + if: always() + run: docker logs iri-facility-api-base || true + + - name: Stop container + if: always() + run: docker stop iri-facility-api-base || true From ea013e2c9fdb5e8961ce225cd1607dfc2ce50d13 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:41:35 -0600 Subject: [PATCH 27/43] Add custom get extra function --- app/routers/dependencies.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py index 17b4f09..7bfaf0b 100644 --- a/app/routers/dependencies.py +++ b/app/routers/dependencies.py @@ -136,6 +136,10 @@ def _hide_extra(self, handler): data.pop(k, None) return data + def get_extra(self, key, default=None): + return getattr(self, "__pydantic_extra__", {}).get(key, default) + + class NamedObject(IRIBaseModel): id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") def _self_path(self) -> str: From 9eb22d92638ad2115bdde4dfe48962de073c5fcd Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 13:06:36 -0600 Subject: [PATCH 28/43] Implement opentelemetry. Use UTC in demo adapter --- Makefile | 1 + README.md | 2 ++ VALIDATION.MD | 23 +++++++++++++++++++++++ app/config.py | 5 +++++ app/demo_adapter.py | 31 +++++++++++++++++++++---------- app/main.py | 32 ++++++++++++++++++++++++++++++++ pyproject.toml | 8 ++++++-- 7 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 VALIDATION.MD diff --git a/Makefile b/Makefile index ba7552a..abd87e4 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ dev : .venv IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + OPENTELEMETRY_ENABLED=true \ API_URL_ROOT='http://127.0.0.1:8000' fastapi dev diff --git a/README.md b/README.md index 5ef41a6..02b796c 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ If using docker (see next section), your dockerfile could extend this reference - `API_URL_ROOT`: the base url when constructing links returned by the api (eg.: https://iri.myfacility.com) - `API_PREFIX`: the path prefix where the api is hosted. Defaults to `/`. (eg.: `/api`) - `API_URL`: the path to the api itself. Defaults to `api/v1`. +- `OPENTELEMETRY_ENABLED`: Enables OpenTelemetry. If enabled, the application will use OpenTelemetry SDKs and emit traces, metrics, and logs. Default to false +- `OTLP_ENDPOINT`: OpenTelemetry Protocol collector endpoint to export telemetry data. If empty or not set, telemetry data is logged locally to log file. Default: "" Links to data, created by this api, will concatenate these values producing links, eg: `https://iri.myfacility.com/my_api_prefix/my_api_url/projects/123` diff --git a/VALIDATION.MD b/VALIDATION.MD new file mode 100644 index 0000000..26f77ef --- /dev/null +++ b/VALIDATION.MD @@ -0,0 +1,23 @@ +# API Validation with Schemathesis + +On every pull request or push to `main` branch, Github Actions run the following steps below that validates an IRI Facility API implementation against OpenAPI spec using Schemathesis. + +1. Builds the Facility API Docker image from Dockerfile. +2. Runs the API container with demo adapter. +3. Waits for `/openapi.json` to become available on localhost:8000. +4. Runs Schemathesis validation twice: + - Against Facilities API’s OpenAPI spec. (http://localhost:8000/openapi.json) + - Against the official IRI Facility API OpenAPI spec. (https://github.com/doe-iri/iri-facility-api-docs/blob/main/specification/openapi/openapi_iri_facility_api_v1.json) +5. Fails the workflow if either validation fails. +6. Saves Schemathesis HTML/XML reports as artifacts (or saves it locally when run with `act`). +7. Dumps API container logs and do clean up to stop container. + +## Running locally + +```bash +act -W .github/workflows/api-validator.yml -s GITHUB_TOKEN= +``` + +## Known issues + +Python implementation not fully aligns with the official Specification. Running against Official Spec will continue to fail, until Spec or Py implementation is fixed. diff --git a/app/config.py b/app/config.py index 078b6a5..71a7c20 100644 --- a/app/config.py +++ b/app/config.py @@ -40,3 +40,8 @@ API_URL_ROOT = os.environ.get("API_URL_ROOT", "https://api.iri.nersc.gov") API_PREFIX = os.environ.get("API_PREFIX", "/") API_URL = os.environ.get("API_URL", "api/v1") + +OPENTELEMETRY_ENABLED = os.environ.get("OPENTELEMETRY_ENABLED", "false").lower() == "true" +OPENTELEMETRY_DEBUG = os.environ.get("OPENTELEMETRY_DEBUG", "false").lower() == "true" +OTLP_ENDPOINT = os.environ.get("OTLP_ENDPOINT", "") +OTEL_SAMPLE_RATE = float(os.environ.get("OTEL_SAMPLE_RATE", "0.2")) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 0d58103..e63c720 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,7 +1,6 @@ import datetime import random import uuid -import time import os import stat import pwd @@ -45,6 +44,16 @@ def demo_uuid(kind: str, name: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) +def utc_now() -> datetime.datetime: + """Return current UTC datetime timestamp""" + return datetime.datetime.now(datetime.timezone.utc) + + +def utc_timestamp() -> int: + """Return current UTC datetime timestamp as integer""" + return int(utc_now().timestamp()) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): @@ -62,7 +71,8 @@ def __init__(self): def _init_state(self): - now = datetime.datetime.now(datetime.timezone.utc) + now = utc_now() + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -80,7 +90,8 @@ def _init_state(self): longitude=-286.36999 ) - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) + + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -352,7 +363,7 @@ async def submit_job( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -371,7 +382,7 @@ async def submit_job_script( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -390,7 +401,7 @@ async def update_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.ACTIVE, - time=time.time(), + time=utc_timestamp(), message="job updated", exit_code=None, meta_data={ "account": "account1" }, @@ -410,7 +421,7 @@ async def get_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.COMPLETED, - time=time.time(), + time=utc_timestamp(), message="job completed successfully", exit_code=0, meta_data={ "account": "account1" }, @@ -432,7 +443,7 @@ async def get_jobs( id=f"job_{i}", status=compute_models.JobStatus( state=random.choice([s for s in compute_models.JobState]), - time=time.time() - (random.random() * 100), + time=utc_timestamp() - int(random.random() * 100), message="", exit_code=random.choice([0, 0, 0, 0, 0, 1, 1, 128, 127]), meta_data={ "account": "account1" }, @@ -900,7 +911,7 @@ class DemoTaskQueue: @staticmethod async def _process_tasks(da: DemoAdapter): - now = time.time() + now = utc_timestamp() _tasks = [] for t in DemoTaskQueue.tasks: if now - t.start > 5 * 60 and t.status in [task_models.TaskStatus.completed, task_models.TaskStatus.canceled, task_models.TaskStatus.failed]: @@ -921,5 +932,5 @@ async def _process_tasks(da: DemoAdapter): @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" - DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=time.time())) + DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) return task_id diff --git a/app/main.py b/app/main.py index 7864143..fa3f1ed 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,13 @@ """Main API application""" import logging from fastapi import FastAPI +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, BatchSpanProcessor, SimpleSpanProcessor +from opentelemetry.sdk.trace.sampling import TraceIdRatioBased, ParentBased +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -13,9 +20,34 @@ from . import config +# ------------------------------------------------------------------ +# OpenTelemetry Tracing Configuration +# ------------------------------------------------------------------ +if config.OPENTELEMETRY_ENABLED: + resource = Resource.create({ + "service.name": "iri-facility-api", + "service.version": config.API_VERSION, + "service.endpoint": config.API_URL_ROOT}) + + samplerate = "1.0" if config.OPENTELEMETRY_DEBUG else config.OTEL_SAMPLE_RATE + provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(samplerate))) + trace.set_tracer_provider(provider) + + if config.OTLP_ENDPOINT: + exporter = OTLPSpanExporter(endpoint=config.OTLP_ENDPOINT, insecure=True) + span_processor = BatchSpanProcessor(exporter) + else: + exporter = ConsoleSpanExporter() + span_processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(span_processor) + tracer = trace.get_tracer(__name__) +# ------------------------------------------------------------------ APP = FastAPI(**config.API_CONFIG) +if config.OPENTELEMETRY_ENABLED: + FastAPIInstrumentor.instrument_app(APP) + install_error_handlers(APP) api_prefix = f"{config.API_PREFIX}{config.API_URL}" diff --git a/pyproject.toml b/pyproject.toml index 03858a4..1f5c0d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,9 @@ requires-python = ">=3.12" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", - "humps>=0.2.2" -] + "humps>=0.2.2", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-exporter-otlp" +] \ No newline at end of file From a23407e3a0c9a56f31382c35c320966c944dcdc1 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 14:28:41 -0600 Subject: [PATCH 29/43] Rename dependencies to common --- app/routers/account/account.py | 2 +- app/routers/account/models.py | 2 +- app/routers/{dependencies.py => common.py} | 0 app/routers/compute/compute.py | 2 +- app/routers/facility/facility.py | 2 +- app/routers/facility/models.py | 2 +- app/routers/status/models.py | 2 +- app/routers/status/status.py | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename app/routers/{dependencies.py => common.py} (100%) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index b2c41f4..fe16b8b 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import forbidExtraQueryParams +from ..common import forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/account/models.py b/app/routers/account/models.py index c012ee3..2158575 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,7 +1,7 @@ from pydantic import computed_field, Field import enum from ... import config -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class AllocationUnit(enum.Enum): diff --git a/app/routers/dependencies.py b/app/routers/common.py similarity index 100% rename from app/routers/dependencies.py rename to app/routers/common.py diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 7a2db23..20081ee 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -4,7 +4,7 @@ from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router -from ..dependencies import forbidExtraQueryParams, StrictBool +from ..common import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 1c38918..c0d3e80 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -2,7 +2,7 @@ from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5db7a18..508dbcf 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..dependencies import NamedObject +from ..common import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/status/models.py b/app/routers/status/models.py index ff5d4e4..d338e2c 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..dependencies import NamedObject +from ..common import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 2daf30e..d599866 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, From 85c8e766550c6a0783d781e798c610dbd4217b01 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 09:26:32 -0600 Subject: [PATCH 30/43] Do not swallow exceptions --- app/routers/compute/compute.py | 9 ++++----- app/routers/iri_router.py | 6 ++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 20081ee..cca45be 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,3 +1,4 @@ +"""Compute resource API router""" from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router @@ -6,13 +7,13 @@ from ..status.status import router as status_router from ..common import forbidExtraQueryParams, StrictBool + router = iri_router.IriRouter( facility_adapter.FacilityAdapter, prefix="/compute", tags=["compute"], ) - @router.post( "/job/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -204,8 +205,6 @@ async def cancel_job( # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) - try: - await router.adapter.cancel_job(resource, user, job_id) - except Exception as exc: - raise HTTPException(status_code=400, detail=f"Unable to cancel job: {str(exc)}") from exc + await router.adapter.cancel_job(resource, user, job_id) + return None diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index d3fc9e1..f0b5b49 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod +import traceback import os import logging import importlib -from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from .account.models import User @@ -12,9 +12,6 @@ def get_client_ip(request : Request) -> str|None: - # logging.debug("Request headers=%s" % request.headers) - # logging.debug("client=%s" % request.client.host) - forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: return forwarded_for.split(",")[0].strip() @@ -91,6 +88,7 @@ async def current_user( user_id = await self.adapter.get_current_user(api_key, get_client_ip(request)) except Exception as exc: logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") + traceback.print_exc() raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") From 29b6ff0abef5d057bb3442226ffc4e3d9fcd41c8 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 18:58:36 -0600 Subject: [PATCH 31/43] Ensure that computed fields are included in output --- app/routers/common.py | 14 ++++++++++++-- app/routers/status/models.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index 7bfaf0b..f46c888 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -129,13 +129,23 @@ class IRIBaseModel(BaseModel): model_config = ConfigDict(extra="allow") @model_serializer(mode="wrap") - def _hide_extra(self, handler): + def _hide_extra(self, handler, info): data = handler(self) + + model_fields = set(self.model_fields or {}) + computed_fields = set(self.model_computed_fields or {}) + print(model_fields) + print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: - data.pop(k, None) + if k not in model_fields and k not in computed_fields: + data.pop(k, None) + return data + + + def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) diff --git a/app/routers/status/models.py b/app/routers/status/models.py index d338e2c..448a7e2 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -36,7 +36,7 @@ def _self_path(self) -> str: current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - @computed_field(description="The list of past events in this incident") + @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] From 94094de9cdd86861e61fa82d262adb56e0a5b3af Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 09:57:21 -0600 Subject: [PATCH 32/43] Code base compliant with the official Spec --- app/demo_adapter.py | 203 +++++++++++++++++++++-- app/routers/account/account.py | 17 +- app/routers/account/facility_adapter.py | 3 +- app/routers/account/models.py | 22 +-- app/routers/common.py | 48 ++++-- app/routers/facility/facility.py | 57 +++++++ app/routers/facility/facility_adapter.py | 46 +++++ app/routers/facility/models.py | 35 +++- app/routers/status/facility_adapter.py | 6 +- app/routers/status/models.py | 2 + app/routers/status/status.py | 32 ++-- 11 files changed, 398 insertions(+), 73 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e63c720..e68635e 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -9,9 +9,10 @@ import subprocess import pathlib import base64 -from pydantic import BaseModel from typing import Any, Tuple +from pydantic import BaseModel from fastapi import HTTPException +from .routers.common import AllocationUnit, Capability from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter @@ -35,7 +36,7 @@ def get_base_temp_dir(cls): os.makedirs(cls._base_temp_dir, exist_ok=True) # create a test file - with open(f"{cls._base_temp_dir}/test.txt", "w") as f: + with open(f"{cls._base_temp_dir}/test.txt", encoding="utf-8", mode="w") as f: f.write("hello world") return cls._base_temp_dir @@ -67,12 +68,57 @@ def __init__(self): self.project_allocations = [] self.user_allocations = [] self.facility = {} + self.locations = [] + self.sites = [] self._init_state() def _init_state(self): now = utc_now() + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -81,22 +127,29 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - facility_uri="https://www.demo.example", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - street_address="1 main st", - latitude=38.410558, - longitude=-286.36999 + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], ) + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { - "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "hpss": account_models.Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), - "gpfs": account_models.Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), + "cpu": Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[AllocationUnit.node_hours]), + "gpu": Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[AllocationUnit.node_hours]), + "hpss": Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), + "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } pm = status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", capability_ids=[ @@ -226,6 +279,125 @@ async def get_facility( ) -> facility_models.Facility: return self.facility + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + + + # ---------------------------- # Status API # ---------------------------- @@ -239,6 +411,8 @@ async def get_resources( group : str | None = None, modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, + current_status : status_models.Status | None = None, + capability: Capability | None = None ) -> list[status_models.Resource]: return status_models.Resource.find(self.resources, name, description, group, modified_since, resource_type)[offset:offset + limit] @@ -288,6 +462,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: return status_models.Incident.find(self.incidents, name, description, status, type, from_, to, time_, modified_since, resource_id)[offset:offset + limit] @@ -301,7 +476,7 @@ async def get_incident( async def get_capabilities( self : "DemoAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: return self.capabilities.values() diff --git a/app/routers/account/account.py b/app/routers/account/account.py index fe16b8b..f856312 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -1,8 +1,8 @@ -from fastapi import HTTPException, Request, Depends +from fastapi import HTTPException, Request, Depends, Query from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import forbidExtraQueryParams +from ..common import forbidExtraQueryParams, StrictDateTime, Capability router = iri_router.IriRouter( @@ -22,8 +22,12 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.Capability]: + name : str = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), + ) -> list[Capability]: return await router.adapter.get_capabilities() @@ -37,8 +41,9 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.Capability: + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + ) -> Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) if not cc: diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 78b622f..235a2f7 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,5 +1,6 @@ from abc import abstractmethod from . import models as account_models +from ..common import Capability from ..iri_router import AuthenticatedAdapter @@ -13,7 +14,7 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_capabilities( self : "FacilityAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: pass diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 2158575..1a9333d 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,26 +1,6 @@ from pydantic import computed_field, Field -import enum from ... import config -from ..common import IRIBaseModel - - -class AllocationUnit(enum.Enum): - node_hours = "node_hours" - bytes = "bytes" - inodes = "inodes" - - -class Capability(IRIBaseModel): - """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) - """ - id: str - name: str - units: list[AllocationUnit] +from ..common import IRIBaseModel, AllocationUnit class User(IRIBaseModel): diff --git a/app/routers/common.py b/app/routers/common.py index f46c888..d580565 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -1,5 +1,6 @@ """Default models used by multiple routers.""" import datetime +import enum from typing import Optional from urllib.parse import parse_qs @@ -93,7 +94,11 @@ def __get_pydantic_json_schema__(cls, schema, handler): } -def forbidExtraQueryParams(*allowedParams: str): +def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): + multiParams = multiParams or set() + + print(allowedParams, multiParams) + async def checker(req: Request): if "*" in allowedParams: return @@ -110,20 +115,26 @@ async def checker(req: Request): detail=[{ "type": "extra_forbidden", "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) + "msg": f"Unexpected query parameter: {key}", + }], + ) - if len(values) > 1: + if len(values) > 1 and key not in multiParams: raise HTTPException( status_code=422, detail=[{ "type": "duplicate_forbidden", "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) + "msg": f"Duplicate query parameter: {key}", + }], + ) + return checker + + + class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") @@ -134,18 +145,12 @@ def _hide_extra(self, handler, info): model_fields = set(self.model_fields or {}) computed_fields = set(self.model_computed_fields or {}) - print(model_fields) - print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: if k not in model_fields and k not in computed_fields: data.pop(k, None) - return data - - - def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) @@ -188,3 +193,22 @@ def normalize(dt: datetime) -> datetime: modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] return a + + +class AllocationUnit(enum.Enum): + node_hours = "node_hours" + bytes = "bytes" + inodes = "inodes" + + +class Capability(IRIBaseModel): + """ + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + """ + id: str + name: str + units: list[AllocationUnit] \ No newline at end of file diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index c0d3e80..7c685e1 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -19,3 +19,60 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") +async def list_sites( + request: Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") +async def get_site( + request: Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") +async def get_site_location( + request : Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") +async def list_locations( + request : Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") +async def get_location( + request : Request, + location_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index cbee951..d316674 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -17,3 +17,49 @@ async def get_facility( ) -> facility_models.Facility | None: pass + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 508dbcf..fd164fa 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,13 +1,22 @@ """Facility-related models.""" -from typing import Optional +from typing import Optional, List from pydantic import Field, HttpUrl from ..common import NamedObject -class Facility(NamedObject): - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization's name.") - facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -16,7 +25,21 @@ class Facility(NamedObject): altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") +class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 6753a47..d7358c5 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -2,6 +2,7 @@ import datetime from fastapi import Query from . import models as status_models +from ..common import Capability class FacilityAdapter(ABC): @@ -21,7 +22,9 @@ async def get_resources( description : str | None = None, group : str | None = None, modified_since : datetime.datetime | None = None, - resource_type: status_models.ResourceType = Query(default=None) + resource_type: status_models.ResourceType = Query(default=None), + current_status: status_models.Status = Query(default=None), + capability: Capability | None = None, ) -> list[status_models.Resource]: pass @@ -75,6 +78,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 448a7e2..3650451 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -101,6 +101,7 @@ def find( class IncidentType(enum.Enum): planned = "planned" unplanned = "unplanned" + reservation = "reservation" class Resolution(enum.Enum): @@ -111,6 +112,7 @@ class Resolution(enum.Enum): pending = "pending" + class Incident(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/status/status.py b/app/routers/status/status.py index d599866..5290a8a 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,8 +1,9 @@ +from typing import Optional, List, Annotated from fastapi import HTTPException, Request, Query, Depends from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams, AllocationUnit router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -22,13 +23,16 @@ async def get_resources( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), group : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + current_status: models.Status = Query(default=None), + #event_uris: Optional[List[str]] = Query(default=None, min_length=1), + capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) @router.get( @@ -48,7 +52,7 @@ async def get_resource( return item -@router.get( +@router.get( "/incidents", summary="Get all incidents without their events", description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", @@ -66,11 +70,15 @@ async def get_incidents( to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), - _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + resolution : models.Resolution = Query(default=None), + resource_uris: Optional[List[str]] = Query(default=None, min_length=1), + event_uris: Optional[List[str]] = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", + "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) + return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) @router.get( @@ -109,8 +117,8 @@ async def get_events( time_ : StrictDateTime = Query(alias="time", default=None), to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) From 9c2eb9eb0e90ad5dc70d2bfb63a9f44616dbba06 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:08:21 -0600 Subject: [PATCH 33/43] Fully compliant with official spec --- app/routers/common.py | 28 +++++++++------------------- app/routers/status/status.py | 5 ++--- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index d580565..fd2882f 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -97,8 +97,6 @@ def __get_pydantic_json_schema__(cls, schema, handler): def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): multiParams = multiParams or set() - print(allowedParams, multiParams) - async def checker(req: Request): if "*" in allowedParams: return @@ -110,31 +108,23 @@ async def checker(req: Request): for key, values in parsed.items(): if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}"}]) + if len(values) > 1 and key not in multiParams: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}"}]) return checker - class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 5290a8a..6a0e948 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -28,11 +28,10 @@ async def get_resources( modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), current_status: models.Status = Query(default=None), - #event_uris: Optional[List[str]] = Query(default=None, min_length=1), - capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status, capability) @router.get( From 2bd894c0651b9cf02eae815aab91fe3775e9c8f5 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:45:23 -0600 Subject: [PATCH 34/43] Enforce Py 3.14 as used release for everything --- .github/workflows/api-validation.yml | 11 ++++++----- Dockerfile | 2 +- pyproject.toml | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index 624e84e..e4d688b 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -15,19 +15,18 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged - name: Checkout schema validator repository uses: actions/checkout@v4 with: - repository: juztas/iri-facility-api-docs - ref: schemavalidator + repository: doe-iri/iri-facility-api-docs + ref: main path: schema-validator token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" - name: Install uv run: pip install uv @@ -81,6 +80,8 @@ jobs: --report-name schemathesis-local echo "exitcode=$?" >> $GITHUB_OUTPUT + # TODO: Change back to https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json + # Once https://github.com/doe-iri/iri-facility-api-docs/pull/12 merged. - name: Run Schemathesis validation (official spec) id: schemathesis_official env: @@ -90,7 +91,7 @@ jobs: source .venv/bin/activate python schema-validator/verification/api-validator.py \ --baseurl http://localhost:8000 \ - --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --schema-url https://raw.githubusercontent.com/juztas/iri-facility-api-docs/refs/heads/newspec/specification/openapi/openapi_iri_facility_api_v1.json \ --report-name schemathesis-official echo "exitcode=$?" >> $GITHUB_OUTPUT diff --git a/Dockerfile b/Dockerfile index f3ad071..93c80d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.14 RUN mkdir /app COPY . /app diff --git a/pyproject.toml b/pyproject.toml index 1f5c0d6..6dbf14b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12" +requires-python = ">=3.12,<3.13" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", @@ -10,4 +10,4 @@ dependencies = [ "opentelemetry-sdk", "opentelemetry-instrumentation-fastapi", "opentelemetry-exporter-otlp" -] \ No newline at end of file +] From 21c03caa0171bbd0ea72e5f60c8339fd7f8bb73e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 17:10:38 -0600 Subject: [PATCH 35/43] Enable deepsource scanning --- .deepsource.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 0000000..01a1066 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,9 @@ +version = 1 + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" + max_line_length = 200 From 6cb85bf29142148079968df011425f96561f8f0f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sun, 25 Jan 2026 08:08:41 -0600 Subject: [PATCH 36/43] Enforce consistent package versions (pin major/minor version) --- pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6dbf14b..63f6c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12,<3.13" +requires-python = ">=3.14,<3.15" dependencies = [ - "fastapi[standard]>=0.100.0", - "uvicorn[standard]>=0.22.0", - "humps>=0.2.2", - "opentelemetry-api", - "opentelemetry-sdk", - "opentelemetry-instrumentation-fastapi", - "opentelemetry-exporter-otlp" -] + "fastapi[standard]>=0.128.0,<0.129.0", + "uvicorn[standard]>=0.40.0,<0.41.0", + "humps>=0.2.2,<0.3.0", + "opentelemetry-api>=1.39.1,<1.40.0", + "opentelemetry-sdk>=1.39.1,<1.40.0", + "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", + "opentelemetry-exporter-otlp>=1.39.1,<1.40.0" +] \ No newline at end of file From 6ccddc78883a11615dda7de6eaffb3c41b309924 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 3 Feb 2026 10:59:23 -0600 Subject: [PATCH 37/43] Make adapter forward compatible. add filtering, pagination, datetime normalization This pull request introduces: - Use and add **kwargs to all adapter interfaces (with a shared warning function for unused parameters). This enables facilities to implement on their own pace, while continue to be compatible. - Use explicit keyword args - Add common pagination helper and expand filtering based on input values; - Unify datetime handling, normalize to UTC and use it in find; --- app/demo_adapter.py | 157 +++++++++++++++++---- app/routers/account/account.py | 38 ++--- app/routers/account/facility_adapter.py | 12 +- app/routers/common.py | 114 ++++++++++++--- app/routers/compute/compute.py | 36 ++--- app/routers/compute/facility_adapter.py | 5 + app/routers/compute/models.py | 49 ++++++- app/routers/facility/facility.py | 8 +- app/routers/facility/facility_adapter.py | 6 + app/routers/facility/models.py | 21 ++- app/routers/filesystem/facility_adapter.py | 22 ++- app/routers/filesystem/filesystem.py | 112 +++++++-------- app/routers/iri_router.py | 8 ++ app/routers/status/facility_adapter.py | 14 +- app/routers/status/models.py | 127 ++++++++++------- app/routers/status/status.py | 9 +- app/routers/task/facility_adapter.py | 87 ++++++------ app/routers/task/task.py | 8 +- 18 files changed, 572 insertions(+), 261 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e68635e..1a520e8 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -12,7 +12,7 @@ from typing import Any, Tuple from pydantic import BaseModel from fastapi import HTTPException -from .routers.common import AllocationUnit, Capability +from .routers.common import AllocationUnit, Capability, paginate_list from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter @@ -276,7 +276,9 @@ def _init_state(self): async def get_facility( self: "DemoAdapter", modified_since: str | None = None, + **kwargs ) -> facility_models.Facility: + self._warn_on_unused_kwargs("get_facility", kwargs) return self.facility @@ -287,8 +289,9 @@ async def list_sites( offset: int | None = None, limit: int | None = None, short_name: str | None = None, + **kwargs ) -> list[facility_models.Site]: - + self._warn_on_unused_kwargs("list_sites", kwargs) sites = self.sites if name: @@ -310,8 +313,9 @@ async def get_site( self: "DemoAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Site: - + self._warn_on_unused_kwargs("get_site", kwargs) site = next((s for s in self.sites if s.id == site_id), None) if not site: raise HTTPException(status_code=404, detail="Site not found") @@ -328,8 +332,9 @@ async def get_site_location( self: "DemoAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location: - + self._warn_on_unused_kwargs("get_site_location", kwargs) site = await self.get_site(site_id) if not site.location_uri: @@ -356,8 +361,9 @@ async def list_locations( limit: int | None = None, short_name: str | None = None, country_name: str | None = None, + **kwargs ) -> list[facility_models.Location]: - + self._warn_on_unused_kwargs("list_locations", kwargs) locs = self.locations if name: @@ -382,8 +388,9 @@ async def get_location( self: "DemoAdapter", location_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location: - + self._warn_on_unused_kwargs("get_location", kwargs) location = next((l for l in self.locations if l.id == location_id), None) if not location: @@ -395,9 +402,6 @@ async def get_location( raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) return location - - - # ---------------------------- # Status API # ---------------------------- @@ -412,17 +416,22 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, current_status : status_models.Status | None = None, - capability: Capability | None = None + capability: Capability | None = None, + **kwargs ) -> list[status_models.Resource]: - return status_models.Resource.find(self.resources, name, description, group, modified_since, resource_type)[offset:offset + limit] + self._warn_on_unused_kwargs("get_resources", kwargs) + resources = status_models.Resource.find(self.resources, name=name, description=description, group=group, modified_since=modified_since, + resource_type=resource_type, current_status=current_status, capability=capability) + return paginate_list(resources, offset, limit) async def get_resource( self : "DemoAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Resource: - return status_models.Resource.find_by_id(self.resources, id) - + self._warn_on_unused_kwargs("get_resource", kwargs) + return status_models.Resource.find_by_id(self.resources, id_) async def get_events( self : "DemoAdapter", @@ -437,16 +446,22 @@ async def get_events( to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, + **kwargs ) -> list[status_models.Event]: - return status_models.Event.find([e for e in self.events if e.incident_id == incident_id], resource_id, name, description, status, from_, to, time_, modified_since)[offset:offset + limit] + self._warn_on_unused_kwargs("get_events", kwargs) + events = status_models.Event.find([e for e in self.events if e.incident_id == incident_id], resource_id=resource_id, name=name, description=description, + status=status, from_=from_, to=to, time_=time_, modified_since=modified_since) + return paginate_list(events, offset, limit) async def get_event( self : "DemoAdapter", incident_id : str, - id : str + id_ : str, + **kwargs ) -> status_models.Event: - return status_models.Event.find_by_id(self.events, id) + self._warn_on_unused_kwargs("get_event", kwargs) + return status_models.Event.find_by_id(self.events, id_) async def get_incidents( @@ -456,39 +471,57 @@ async def get_incidents( name : str | None = None, description : str | None = None, status : status_models.Status | None = None, - type : status_models.IncidentType | None = None, + type_ : status_models.IncidentType | None = None, from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, resolution: status_models.Resolution | None = None, + **kwargs ) -> list[status_models.Incident]: - return status_models.Incident.find(self.incidents, name, description, status, type, from_, to, time_, modified_since, resource_id)[offset:offset + limit] + self._warn_on_unused_kwargs("get_incidents", kwargs) + incidents = status_models.Incident.find(self.incidents, name=name, description=description, status=status, type_=type_,from_=from_, to=to, + time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) + return paginate_list(incidents, offset, limit) async def get_incident( self : "DemoAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Incident: - return status_models.Incident.find_by_id(self.incidents, id) + self._warn_on_unused_kwargs("get_incident", kwargs) + return status_models.Incident.find_by_id(self.incidents, id_) async def get_capabilities( self : "DemoAdapter", + name : str | None = None, + modified_since : str | None = None, + offset : int = 0, + limit : int = 1000, + **kwargs ) -> list[Capability]: - return self.capabilities.values() + self._warn_on_unused_kwargs("get_capabilities", kwargs) + caps = list(self.capabilities.values()) + if name: + caps = [c for c in caps if name.lower() in c.name.lower()] + + return paginate_list(caps, offset, limit) async def get_current_user( self : "DemoAdapter", api_key: str, client_ip: str, + **kwargs ) -> str: """ In a real deployment, this would decode the api_key jwt and return the current user's id. This method is not async. """ + self._warn_on_unused_kwargs("get_current_user", kwargs) return "gtorok" @@ -497,7 +530,9 @@ async def get_user( user_id: str, api_key: str, client_ip: str|None, + **kwargs ) -> account_models.User: + self._warn_on_unused_kwargs("get_user", kwargs) if user_id != self.user.id: raise HTTPException(status_code=401, detail="User not found") if api_key != self.user.api_key: @@ -507,16 +542,20 @@ async def get_user( async def get_projects( self : "DemoAdapter", - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.Project]: + self._warn_on_unused_kwargs("get_projects", kwargs) return self.projects async def get_project_allocations( self : "DemoAdapter", project: account_models.Project, - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.ProjectAllocation]: + self._warn_on_unused_kwargs("get_project_allocations", kwargs) return [pa for pa in self.project_allocations if pa.project_id == project.id] @@ -524,7 +563,9 @@ async def get_user_allocations( self : "DemoAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation, + **kwargs ) -> list[account_models.UserAllocation]: + self._warn_on_unused_kwargs("get_user_allocations", kwargs) return [ua for ua in self.user_allocations if ua.project_allocation_id == project_allocation.id] @@ -533,7 +574,9 @@ async def submit_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, + **kwargs, ) -> compute_models.Job: + self._warn_on_unused_kwargs("submit_job", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -552,7 +595,9 @@ async def submit_job_script( user: account_models.User, job_script_path: str, args: list[str] = [], + **kwargs ) -> compute_models.Job: + self._warn_on_unused_kwargs("submit_job_script", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -571,7 +616,9 @@ async def update_job( user: account_models.User, job_spec: compute_models.JobSpec, job_id: str, + **kwargs, ) -> compute_models.Job: + self._warn_on_unused_kwargs("update_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -591,7 +638,9 @@ async def get_job( job_id: str, historical: bool = False, include_spec: bool = False, + **kwargs, ) -> compute_models.Job: + self._warn_on_unused_kwargs("get_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -613,7 +662,9 @@ async def get_jobs( filters: dict[str, object] | None = None, historical: bool = False, include_spec: bool = False, + **kwargs, ) -> list[compute_models.Job]: + self._warn_on_unused_kwargs("get_jobs", kwargs) return [compute_models.Job( id=f"job_{i}", status=compute_models.JobStatus( @@ -631,7 +682,9 @@ async def cancel_job( resource: status_models.Resource, user: account_models.User, job_id: str, + **kwargs, ) -> bool: + self._warn_on_unused_kwargs("cancel_job", kwargs) # call slurm/etc. to cancel job return True @@ -701,8 +754,10 @@ async def chmod( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest + request_model: filesystem_models.PutFileChmodRequest, + **kwargs, ) -> filesystem_models.PutFileChmodResponse: + self._warn_on_unused_kwargs("chmod", kwargs) rp = self.validate_path(request_model.path) os.chmod(rp, int(request_model.mode, 8)) return filesystem_models.PutFileChmodResponse( @@ -714,8 +769,10 @@ async def chown( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChownRequest + request_model: filesystem_models.PutFileChownRequest, + **kwargs, ) -> filesystem_models.PutFileChownResponse: + self._warn_on_unused_kwargs("chown", kwargs) rp = self.validate_path(request_model.path) os.chown(rp, request_model.owner, request_model.group) return filesystem_models.PutFileChmodResponse( @@ -732,7 +789,9 @@ async def ls( numeric_uid: bool, recursive: bool, dereference: bool, + **kwargs, ) -> filesystem_models.GetDirectoryLsResponse: + self._warn_on_unused_kwargs("ls", kwargs) rp = self.validate_path(path) files = glob.glob(rp, recursive=recursive) return filesystem_models.GetDirectoryLsResponse( @@ -773,7 +832,9 @@ async def head( file_bytes: int | None, lines: int | None, skip_trailing: bool, + **kwargs, ) -> Tuple[Any, int]: + self._warn_on_unused_kwargs("head", kwargs) return self._headtail("head", path, file_bytes, lines) @@ -785,7 +846,9 @@ async def tail( file_bytes: int | None, lines: int | None, skip_trailing: bool, + **kwargs ) -> Tuple[Any, int]: + self._warn_on_unused_kwargs("tail", kwargs) return self._headtail("tail", path, file_bytes, lines) @@ -796,7 +859,9 @@ async def view( path: str, size: int, offset: int, + **kwargs ) -> filesystem_models.GetViewFileResponse: + self._warn_on_unused_kwargs("view", kwargs) rp = self.validate_path(path) result = subprocess.run( f"tail -c +{offset+1} {rp} | head -c {size}", @@ -815,7 +880,9 @@ async def checksum( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileChecksumResponse: + self._warn_on_unused_kwargs("checksum", kwargs) rp = self.validate_path(path) result = subprocess.run( ["sha256sum", rp], @@ -835,7 +902,9 @@ async def file( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileTypeResponse: + self._warn_on_unused_kwargs("file", kwargs) rp = self.validate_path(path) result = subprocess.run( ["file", "-b", rp], @@ -853,7 +922,9 @@ async def stat( user: account_models.User, path: str, dereference: bool, + **kwargs ) -> filesystem_models.GetFileStatResponse: + self._warn_on_unused_kwargs("stat", kwargs) rp = self.validate_path(path) if dereference: stat_info = os.stat(rp) @@ -880,7 +951,9 @@ async def rm( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ): + self._warn_on_unused_kwargs("rm", kwargs) rp = self.validate_path(path) if rp == PathSandbox.get_base_temp_dir(): raise HTTPException(status_code=400, detail="Cannot delete sandbox") @@ -893,7 +966,9 @@ async def mkdir( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest, + **kwargs ) -> filesystem_models.PostMkdirResponse: + self._warn_on_unused_kwargs("mkdir", kwargs) rp = self.validate_path(request_model.path) args = ["mkdir"] if request_model.parent: @@ -910,7 +985,9 @@ async def symlink( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest, + **kwargs ) -> filesystem_models.PostFileSymlinkResponse: + self._warn_on_unused_kwargs("symlink", kwargs) rp_src = self.validate_path(request_model.path) rp_dst = self.validate_path(request_model.link_path) subprocess.run(["ln", "-s", rp_src, rp_dst], check=True) @@ -924,7 +1001,9 @@ async def download( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> Any: + self._warn_on_unused_kwargs("download", kwargs) rp = self.validate_path(path) raw_content = pathlib.Path(rp).read_bytes() @@ -940,7 +1019,9 @@ async def upload( user: account_models.User, path: str, content: str, + **kwargs ) -> None: + self._warn_on_unused_kwargs("upload", kwargs) rp = self.validate_path(path) if isinstance(content, bytes): pathlib.Path(rp).write_bytes(content) @@ -955,7 +1036,9 @@ async def compress( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest, + **kwargs ) -> filesystem_models.PostCompressResponse: + self._warn_on_unused_kwargs("compress", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -988,7 +1071,9 @@ async def extract( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest, + **kwargs ) -> filesystem_models.PostExtractResponse: + self._warn_on_unused_kwargs("extract", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -1016,7 +1101,9 @@ async def mv( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest, + **kwargs ) -> filesystem_models.PostMoveResponse: + self._warn_on_unused_kwargs("mv", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) subprocess.run(["mv", src_rp, dst_rp], check=True) @@ -1030,7 +1117,9 @@ async def cp( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest, + **kwargs ) -> filesystem_models.PostCopyResponse: + self._warn_on_unused_kwargs("cp", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) args = ["cp"] @@ -1048,7 +1137,9 @@ async def get_task( self : "DemoAdapter", user: account_models.User, task_id: str, + **kwargs ) -> task_models.Task|None: + self._warn_on_unused_kwargs("get_task", kwargs) await DemoTaskQueue._process_tasks(self) return next((t for t in DemoTaskQueue.tasks if t.user.name == user.name and t.id == task_id), None) @@ -1056,7 +1147,9 @@ async def get_task( async def get_tasks( self : "DemoAdapter", user: account_models.User, + **kwargs ) -> list[task_models.Task]: + self._warn_on_unused_kwargs("get_tasks", kwargs) await DemoTaskQueue._process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] @@ -1065,15 +1158,17 @@ async def put_task( self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, - body: str + task: str, + **kwargs, ) -> str: + self._warn_on_unused_kwargs("put_task", kwargs) await DemoTaskQueue._process_tasks(self) - return DemoTaskQueue._create_task(user, resource, body) + return DemoTaskQueue._create_task(user, resource, task) class DemoTask(BaseModel): id: str - body: str + task: str resource: status_models.Resource user: account_models.User start: float @@ -1096,7 +1191,7 @@ async def _process_tasks(da: DemoAdapter): t.status = task_models.TaskStatus.active t.start = now elif t.status == task_models.TaskStatus.active and now - t.start > DEMO_QUEUE_UPDATE_SECS: - cmd = task_models.TaskCommand.model_validate_json(t.body) + cmd = task_models.TaskCommand.model_validate_json(t.task) (result, status) = await DemoAdapter.on_task(t.resource, t.user, cmd) t.result = result t.status = status @@ -1107,5 +1202,5 @@ async def _process_tasks(da: DemoAdapter): @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" - DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) + DemoTaskQueue.tasks.append(DemoTask(id=task_id, task=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) return task_id diff --git a/app/routers/account/account.py b/app/routers/account/account.py index f856312..e13cbde 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -28,7 +28,7 @@ async def get_capabilities( limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), ) -> list[Capability]: - return await router.adapter.get_capabilities() + return await router.adapter.get_capabilities(name=name, modified_since=modified_since, offset=offset, limit=limit) @router.get( @@ -44,7 +44,7 @@ async def get_capability( modified_since: StrictDateTime = Query(default=None), _forbid = Depends(forbidExtraQueryParams("modified_since")), ) -> Capability: - caps = await router.adapter.get_capabilities() + caps = await router.adapter.get_capabilities(name=None, modified_since=modified_since, offset=0, limit=100) cc = next((c for c in caps if c.id == capability_id), None) if not cc: raise HTTPException(status_code=404, detail="Capability not found") @@ -63,7 +63,7 @@ async def get_projects( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Project]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") return await router.adapter.get_projects(user) @@ -82,10 +82,10 @@ async def get_project( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> models.Project: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) pp = next((p for p in projects if p.id == project_id), None) if not pp: raise HTTPException(status_code=404, detail="Project not found") @@ -105,14 +105,14 @@ async def get_project_allocations( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: raise HTTPException(status_code=404, detail="Project not found") - return await router.adapter.get_project_allocations(project, user) + return await router.adapter.get_project_allocations(project=project, user=user) @router.get( @@ -129,12 +129,12 @@ async def get_project_allocation( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) - pas = await router.adapter.get_project_allocations(project, user) + pas = await router.adapter.get_project_allocations(project=project, user=user) pa = next((pa for pa in pas if pa.id == project_allocation_id), None) if not pa: raise HTTPException(status_code=404, detail="Project allocation not found") @@ -155,18 +155,18 @@ async def get_user_allocations( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.UserAllocation]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: raise HTTPException(status_code=404, detail="Project not found") - pas = await router.adapter.get_project_allocations(project, user) + pas = await router.adapter.get_project_allocations(project=project, user=user) pa = next((pa for pa in pas if pa.id == project_allocation_id), None) if not pa: raise HTTPException(status_code=404, detail="Project allocation not found") - return await router.adapter.get_user_allocations(user, pa) + return await router.adapter.get_user_allocations(user=user, project_allocation=pa) @router.get( @@ -184,18 +184,18 @@ async def get_user_allocation( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> models.UserAllocation: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: raise HTTPException(status_code=404, detail="Project not found") - pas = await router.adapter.get_project_allocations(project, user) + pas = await router.adapter.get_project_allocations(project=project, user=user) pa = next((pa for pa in pas if pa.id == project_allocation_id), None) if not pa: raise HTTPException(status_code=404, detail="Project allocation not found") - uas = await router.adapter.get_user_allocations(user, pa) + uas = await router.adapter.get_user_allocations(user=user, project_allocation=pa) ua = next((ua for ua in uas if ua.id == user_allocation_id), None) if not ua: raise HTTPException(status_code=404, detail="User allocation not found") diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 235a2f7..3abca87 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -14,6 +14,11 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_capabilities( self : "FacilityAdapter", + name : str | None = None, + modified_since : str | None = None, + offset : int = 0, + limit : int = 1000, + **kwargs ) -> list[Capability]: pass @@ -21,7 +26,8 @@ async def get_capabilities( @abstractmethod async def get_projects( self : "FacilityAdapter", - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.Project]: pass @@ -30,7 +36,8 @@ async def get_projects( async def get_project_allocations( self : "FacilityAdapter", project: account_models.Project, - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.ProjectAllocation]: pass @@ -40,5 +47,6 @@ async def get_user_allocations( self : "FacilityAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation, + **kwargs ) -> list[account_models.UserAllocation]: pass diff --git a/app/routers/common.py b/app/routers/common.py index fd2882f..2826dab 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -1,16 +1,25 @@ """Default models used by multiple routers.""" import datetime +from email.utils import parsedate_to_datetime import enum from typing import Optional from urllib.parse import parse_qs from pydantic_core import core_schema -from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer -from fastapi import Request, HTTPException +from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer, field_validator +from fastapi import Request, HTTPException, status from .. import config +def paginate_list(items, offset: int | None, limit: int | None): + """Return a sliced items using offset and limit.""" + if offset is not None and offset > 0: + items = items[offset:] + if limit is not None and limit >= 0: + items = items[:limit] + return items + # These are Pydantic custom types for strict validation # that are not implmented in Pydantic by default. # ----------------------------------------------------------------------- @@ -79,6 +88,51 @@ def validate(value): return StrictDateTime._normalize(dt) + + @staticmethod + def modifiedSinceDatetime( + modified_since: str | None, + header_modified_since: str | None + ) -> datetime.datetime | None: + """ + Combine modified_since (ISO8601) and If-Modified-Since (RFC1123). + If both are provided, the most recent timestamp is used. + """ + + parsed_times: list[datetime.datetime] = [] + + # Query param (ISO 8601) + if modified_since is not None: + try: + dt = StrictDateTime.validate(modified_since) + parsed_times.append(dt) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid modified_since query param: {exc}", + ) from exc + + # Header (RFC 1123) + if header_modified_since is not None: + try: + dt = parsedate_to_datetime(header_modified_since) + if dt is None: + raise ValueError("Invalid RFC1123 date") + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + parsed_times.append(dt.astimezone(datetime.timezone.utc)) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid If-Modified-Since header format (must be RFC1123)", + ) from exc + + if not parsed_times: + return None + + # Stricter constraint wins + return max(parsed_times) + @staticmethod def _normalize(dt: datetime.datetime) -> datetime.datetime: if dt.tzinfo is None: @@ -123,8 +177,6 @@ async def checker(req: Request): return checker - - class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") @@ -150,6 +202,23 @@ class NamedObject(IRIBaseModel): def _self_path(self) -> str: raise NotImplementedError + @classmethod + def normalize_dt(cls, dt: datetime | None) -> datetime | None: + """Normalize datetime to UTC-aware.""" + # Convert naive datetimes into UTC-aware versions + if dt is None: + return None + if isinstance(dt, str): + dt = StrictDateTime.validate(dt) + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @field_validator("last_modified", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + @computed_field(description="The canonical URL of this object") @property def self_uri(self) -> str: @@ -160,29 +229,32 @@ def self_uri(self) -> str: description: Optional[str] = Field(None, description="Human-readable description of the object.") last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): + @classmethod + def find_by_id(cls, items, id_, allow_name: bool = False): + """ Find an object by its id or name == id. """ # Find a resource by its id. # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + matches = [r for r in items if r.id == id_ or (allow_name and r.name == id_)] + if not matches: + return None + if len(matches) > 1: + raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id}'") + return matches[0] - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt + @classmethod + def find(cls, items, name=None, description=None, modified_since=None): + """ Find objects matching the given criteria. """ + if not any((name, description, modified_since)): + return items if name: - a = [aa for aa in a if aa.name == name] + items = [item for item in items if item.name == name] if description: - a = [aa for aa in a if description in aa.description] + items = [item for item in items if item.description and description in item.description] if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a + modified_since = cls.normalize_dt(modified_since) + items = [item for item in items if item.last_modified >= modified_since] + return items class AllocationUnit(enum.Enum): @@ -201,4 +273,4 @@ class Capability(IRIBaseModel): """ id: str name: str - units: list[AllocationUnit] \ No newline at end of file + units: list[AllocationUnit] diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index cca45be..804f9a5 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -36,16 +36,16 @@ async def submit_job( This command will attempt to submit a job and return its id. """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job(resource, user, job_spec) + return await router.adapter.submit_job(resource=resource, user=user, job_spec=job_spec) # TODO: this conflicts with PUT commented out while we finalize the API design @@ -73,16 +73,16 @@ async def submit_job( # # This command will attempt to submit a job and return its id. # """ -# user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) +# user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) # if not user: # raise HTTPException(status_code=404, detail="User not found") # # # look up the resource (todo: maybe ensure it's available) -# resource = await status_router.adapter.get_resource(resource_id) +# resource = await status_router.adapter.get_resource(resource_id=resource_id) # # # the handler can use whatever means it wants to submit the job and then fill in its id # # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs -# return await router.adapter.submit_job_script(resource, user, job_script_path, args) +# return await router.adapter.submit_job_script(resource=resource, user=user, job_script_path=job_script_path, args=args) @router.put( @@ -108,16 +108,16 @@ async def update_job( - **job_request**: a PSIJ job spec as defined here """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.update_job(resource, user, job_spec, job_id) + return await router.adapter.update_job(resource=resource, user=user, job_spec=job_spec, job_id=job_id) @router.get( @@ -137,15 +137,15 @@ async def get_job_status( _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) - job = await router.adapter.get_job(resource, user, job_id, historical, include_spec) + job = await router.adapter.get_job(resource=resource, user=user, job_id=job_id, historical=historical, include_spec=include_spec) return job @@ -169,15 +169,15 @@ async def get_job_statuses( _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) - jobs = await router.adapter.get_jobs(resource, user, offset, limit, filters, historical, include_spec) + jobs = await router.adapter.get_jobs(resource=resource, user=user, offset=offset, limit=limit, filters=filters, historical=historical, include_spec=include_spec) return jobs @@ -198,13 +198,13 @@ async def cancel_job( _forbid = Depends(forbidExtraQueryParams()), ): """Cancel a job""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) - await router.adapter.cancel_job(resource, user, job_id) + await router.adapter.cancel_job(resource=resource, user=user, job_id=job_id) return None diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index 6cf0bb2..d70ec51 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -19,6 +19,7 @@ async def submit_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, + **kwargs ) -> compute_models.Job: pass @@ -30,6 +31,7 @@ async def submit_job_script( user: account_models.User, job_script_path: str, args: list[str] = [], + **kwargs ) -> compute_models.Job: pass @@ -41,6 +43,7 @@ async def update_job( user: account_models.User, job_spec: compute_models.JobSpec, job_id: str, + **kwargs ) -> compute_models.Job: pass @@ -67,6 +70,7 @@ async def get_jobs( filters: dict[str, object] | None = None, historical: bool = False, include_spec: bool = False, + **kwargs ) -> list[compute_models.Job]: pass @@ -77,5 +81,6 @@ async def cancel_job( resource: status_models.Resource, user: account_models.User, job_id: str, + **kwargs ) -> bool: pass diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index a56d4fe..3aa4121 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,10 +1,11 @@ from typing import Annotated from enum import IntEnum -from pydantic import field_serializer, ConfigDict, StrictBool, Field +from pydantic import field_serializer, StrictBool, Field from ..common import IRIBaseModel class ResourceSpec(IRIBaseModel): +<<<<<<< HEAD """ Specification of computational resources required for a job. """ @@ -27,14 +28,37 @@ class JobAttributes(IRIBaseModel): reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} +======= + node_count: int | None = Field(default=None, description="Number of compute nodes to allocate") + process_count: int | None = Field(default=None, description="Total number of processes to launch") + processes_per_node: int | None = Field(default=None, description="Number of processes to launch per node") + cpu_cores_per_process: int | None = Field(default=None, description="Number of CPU cores to allocate per process") + gpu_cores_per_process: int | None = Field(default=None, description="Number of GPU cores to allocate per process") + exclusive_node_use: StrictBool = Field(default=True, description="Whether to request exclusive use of allocated nodes") + memory: int | None = Field(default=None, description="Amount of memory to allocate in bytes") + + +class JobAttributes(IRIBaseModel): + duration: int = Field(default=None, ge=1, description="Duration in seconds", examples=[30, 60, 120]) + queue_name: str | None = Field(default=None, min_length=1, description="Name of the queue or partition to submit the job to") + account: str | None = Field(default=None, min_length=1, description="Account or project to charge for resource usage") + reservation_id: str | None = Field(default=None, min_length=1, description="ID of a reservation to use for the job") + custom_attributes: dict[str, str] = Field(default={}, description="Custom scheduler-specific attributes as key-value pairs") +>>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ +<<<<<<< HEAD source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True +======= + source: str = Field(description="The source path on the host system to mount") + target: str = Field(description="The target path inside the container where the volume will be mounted") + read_only: StrictBool = Field(default=True, description="Whether the mount should be read-only") +>>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class Container(IRIBaseModel): """ @@ -45,6 +69,7 @@ class Container(IRIBaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ +<<<<<<< HEAD image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] @@ -69,6 +94,28 @@ class JobSpec(IRIBaseModel): pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None +======= + image: str = Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')") + volume_mounts: list[VolumeMount] = Field(default=[], description="List of volume mounts for the container") + + +class JobSpec(IRIBaseModel): + executable: str | None = Field(default=None, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.") + container: Container | None = Field(default=None, description="Container specification for containerized execution") + arguments: list[str] = Field(default=[], description="Command-line arguments to pass to the executable or container") + directory: str | None = Field(default=None, description="Working directory for the job") + name: str | None = Field(default=None, description="Name of the job") + inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment variables from the submission environment") + environment: dict[str, str] = Field(default={}, description="Environment variables to set for the job. If container is specified, these will be set inside the container.") + stdin_path: str | None = Field(default=None, description="Path to file to use as standard input") + stdout_path: str | None = Field(default=None, description="Path to file to write standard output") + stderr_path: str | None = Field(default=None, description="Path to file to write standard error") + resources: ResourceSpec | None = Field(default=None, description="Resource requirements for the job") + attributes: JobAttributes | None = Field(default=None, description="Additional job attributes such as duration, queue, and account") + pre_launch: str | None = Field(default=None, description="Script or commands to run before launching the job") + post_launch: str | None = Field(default=None, description="Script or commands to run after the job completes") + launcher: str | None = Field(default=None, description="Job launcher to use (e.g., 'mpirun', 'srun')") +>>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class CommandResult(IRIBaseModel): diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 7c685e1..7d264af 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -5,11 +5,9 @@ from ..common import StrictDateTime, forbidExtraQueryParams -router = iri_router.IriRouter( - facility_adapter.FacilityAdapter, - prefix="/facility", - tags=["facility"], -) +router = iri_router.IriRouter(facility_adapter.FacilityAdapter, + prefix="/facility", + tags=["facility"]) @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674..fc6777f 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -14,6 +14,7 @@ class FacilityAdapter(AuthenticatedAdapter): async def get_facility( self: "FacilityAdapter", modified_since: str | None = None, + **kwargs ) -> facility_models.Facility | None: pass @@ -25,6 +26,7 @@ async def list_sites( offset: int | None = None, limit: int | None = None, short_name: str | None = None, + **kwargs ) -> list[facility_models.Site]: pass @@ -33,6 +35,7 @@ async def get_site( self: "FacilityAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Site | None: pass @@ -41,6 +44,7 @@ async def get_site_location( self: "FacilityAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location | None: pass @@ -53,6 +57,7 @@ async def list_locations( limit: int | None = None, short_name: str | None = None, country_name: str | None = None, + **kwargs ) -> list[facility_models.Location]: pass @@ -61,5 +66,6 @@ async def get_location( self: "FacilityAdapter", location_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location | None: pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index fd164fa..bee399f 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -13,6 +13,14 @@ def _self_path(self) -> str: location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, short_name=None): + """ Find Sites matching the given criteria. """ + items = super().find(items, name=name, description=description, modified_since=modified_since) + if short_name: + items = [item for item in items if item.short_name == short_name] + return items + class Location(NamedObject): def _self_path(self) -> str: return f"/facility/locations/{self.id}" @@ -27,6 +35,18 @@ def _self_path(self) -> str: longitude: Optional[float] = Field(None, description="Longitude of the Location.") site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, short_name=None, country_name=None): + """ Find Locations matching the given criteria. """ + items = super().find(items, name=name, description=description, modified_since=modified_since) + if short_name: + items = [item for item in items if item.short_name == short_name] + if country_name: + items = [item for item in items if item.country_name == country_name] + return items + + + class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" @@ -42,4 +62,3 @@ def _self_path(self) -> str: project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") - diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index a70efb0..8506f1c 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -29,7 +29,8 @@ async def chmod( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest + request_model: filesystem_models.PutFileChmodRequest, + **kwargs ) -> filesystem_models.PutFileChmodResponse: pass @@ -39,7 +40,8 @@ async def chown( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChownRequest + request_model: filesystem_models.PutFileChownRequest, + **kwargs ) -> filesystem_models.PutFileChownResponse: pass @@ -54,6 +56,7 @@ async def ls( numeric_uid: bool, recursive: bool, dereference: bool, + **kwargs ) -> filesystem_models.GetDirectoryLsResponse: pass @@ -67,6 +70,7 @@ async def head( file_bytes: int, lines: int, skip_trailing: bool, + **kwargs ) -> Tuple[Any, int]: pass @@ -80,6 +84,7 @@ async def tail( file_bytes: int | None, lines: int | None, skip_trailing: bool, + **kwargs ) -> Tuple[Any, int]: pass @@ -92,6 +97,7 @@ async def view( path: str, size: int, offset: int, + **kwargs ) -> filesystem_models.GetViewFileResponse: pass @@ -102,6 +108,7 @@ async def checksum( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileChecksumResponse: pass @@ -112,6 +119,7 @@ async def file( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileTypeResponse: pass @@ -123,6 +131,7 @@ async def stat( user: account_models.User, path: str, dereference: bool, + **kwargs ) -> filesystem_models.GetFileStatResponse: pass @@ -133,6 +142,7 @@ async def rm( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ): pass @@ -143,6 +153,7 @@ async def mkdir( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest, + **kwargs ) -> filesystem_models.PostMkdirResponse: pass @@ -153,6 +164,7 @@ async def symlink( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest, + **kwargs ) -> filesystem_models.PostFileSymlinkResponse: pass @@ -163,6 +175,7 @@ async def download( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> Any: pass @@ -174,6 +187,7 @@ async def upload( user: account_models.User, path: str, content: str, + **kwargs ) -> None: pass @@ -184,6 +198,7 @@ async def compress( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest, + **kwargs ) -> filesystem_models.PostCompressResponse: pass @@ -194,6 +209,7 @@ async def extract( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest, + **kwargs ) -> filesystem_models.PostExtractResponse: pass @@ -204,6 +220,7 @@ async def mv( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest, + **kwargs ) -> filesystem_models.PostMoveResponse: pass @@ -214,5 +231,6 @@ async def cp( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest, + **kwargs ) -> filesystem_models.PostCopyResponse: pass diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index d583c64..fb27427 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -34,12 +34,12 @@ async def _user_resource( resource_id: str, request: Request, ) -> tuple[account_models.User, status_models.Resource]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) if not resource: raise HTTPException(status_code=404, detail="Resource not found") return (user, resource) @@ -62,9 +62,9 @@ async def put_chmod( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="chmod", args={ @@ -91,9 +91,9 @@ async def put_chown( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="chown", args={ @@ -121,9 +121,9 @@ async def get_file( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="file", args={ @@ -151,9 +151,9 @@ async def get_stat( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="stat", args={ @@ -181,9 +181,9 @@ async def post_mkdir( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="mkdir", args={ @@ -211,9 +211,9 @@ async def post_symlink( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="symlink", args={ @@ -257,9 +257,9 @@ async def get_ls_async( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="ls", args={ @@ -325,9 +325,9 @@ async def get_head( detail="Exactly one of `bytes` or `lines` must be specified." ) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="head", args={ @@ -363,9 +363,9 @@ async def get_view( user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="view", args={ @@ -422,9 +422,9 @@ async def get_tail( detail="Exactly one of `bytes` or `lines` must be specified." ) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="tail", args={ @@ -455,9 +455,9 @@ async def get_checksum( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="checksum", args={ @@ -481,9 +481,9 @@ async def delete_rm( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="rm", args={ @@ -510,9 +510,9 @@ async def post_compress( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="compress", args={ @@ -539,9 +539,9 @@ async def post_extract( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="extract", args={ @@ -568,9 +568,9 @@ async def move_mv( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="mv", args={ @@ -597,9 +597,9 @@ async def post_cp( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="cp", args={ @@ -625,9 +625,9 @@ async def get_download( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="download", args={ @@ -665,9 +665,9 @@ async def post_upload( ) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="upload", args={ diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index f0b5b49..d4cb6f2 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -98,11 +98,18 @@ async def current_user( class AuthenticatedAdapter(ABC): + def _warn_on_unused_kwargs(self, func_name: str, kwargs: dict) -> None: + if not kwargs: + return + logging.getLogger().warning("Adapter method '%s' received unused kwargs: %s", func_name, + ", ".join(sorted(kwargs.keys()))) + @abstractmethod async def get_current_user( self : "AuthenticatedAdapter", api_key: str, client_ip: str|None, + **kwargs ) -> str: """ Decode the api_key and return the authenticated user's id. @@ -118,6 +125,7 @@ async def get_user( user_id: str, api_key: str, client_ip: str|None, + **kwargs ) -> User: """ Retrieve additional user information (name, email, etc.) for the given user_id. diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index d7358c5..1d774cb 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -25,6 +25,7 @@ async def get_resources( resource_type: status_models.ResourceType = Query(default=None), current_status: status_models.Status = Query(default=None), capability: Capability | None = None, + **kwargs ) -> list[status_models.Resource]: pass @@ -32,7 +33,8 @@ async def get_resources( @abstractmethod async def get_resource( self : "FacilityAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Resource: pass @@ -49,8 +51,9 @@ async def get_events( status : status_models.Status | None = None, from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, - time : datetime.datetime | None = None, + time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, + **kwargs ) -> list[status_models.Event]: pass @@ -59,7 +62,8 @@ async def get_events( async def get_event( self : "FacilityAdapter", incident_id : str, - id : str + id_ : str, + **kwargs ) -> status_models.Event: pass @@ -79,6 +83,7 @@ async def get_incidents( modified_since : datetime.datetime | None = None, resource_id : str | None = None, resolution: status_models.Resolution | None = None, + **kwargs ) -> list[status_models.Incident]: pass @@ -86,6 +91,7 @@ async def get_incidents( @abstractmethod async def get_incident( self : "FacilityAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Incident: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 3650451..013323f 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -1,6 +1,6 @@ import datetime import enum -from pydantic import BaseModel, computed_field, Field +from pydantic import BaseModel, computed_field, Field, field_validator from ... import config from ..common import NamedObject @@ -29,33 +29,47 @@ class ResourceType(enum.Enum): class Resource(NamedObject): def _self_path(self) -> str: + """ Return the API path for this resource. """ return f"/status/resources/{self.id}" - capability_ids: list[str] = Field(exclude=True) + capability_ids: list[str] = Field(default_factory=list, exclude=True) group: str | None - current_status: Status | None = Field("The current status comes from the status of the last event for this resource") + current_status: Status | None = Field(default=None, description="The current status comes from the status of the last event for this resource") resource_type: ResourceType @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: + """ Return the list of capability URIs for this resource. """ return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] - @staticmethod - def find(resources, name, description, group, modified_since, resource_type): - a = NamedObject.find(resources, name, description, modified_since) + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, group=None, resource_type=None, current_status=None, capability=None) -> list: + items = super().find(items, name=name, description=description, modified_since=modified_since) if group: - a = [aa for aa in a if aa.group == group] + items = [item for item in items if item.group == group] if resource_type: - a = [aa for aa in a if aa.resource_type == resource_type] - return a - + if isinstance(resource_type, str): + resource_type = ResourceType(resource_type) + items = [item for item in items if item.resource_type == resource_type] + if current_status: + items = [item for item in items if item.current_status == current_status] + if capability: + items = [item for item in items + if any(cap_id in item.capability_ids for cap_id in capability)] + return items class Event(NamedObject): def _self_path(self) -> str: + """ Return the API path for this event. """ return f"/status/incidents/{self.incident_id}/events/{self.id}" + @field_validator("occurred_at", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + occurred_at : datetime.datetime status : Status resource_id : str = Field(exclude=True) @@ -64,38 +78,39 @@ def _self_path(self) -> str: @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: + """ Return the resource URI for this event. """ return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" @computed_field(description="The event's incident") @property def incident_uri(self) -> str|None: + """ Return the incident URI for this event. """ return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}" if self.incident_id else None - @staticmethod - def find( - events : list, - resource_id : str | None = None, - name : str | None = None, - description : str | None = None, - status : Status | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - ) -> list: - events = NamedObject.find(events, name, description, modified_since) + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, + resource_id=None, status=None, from_=None, to=None, time_=None) -> list: + items = super().find(items, name=name, description=description, modified_since=modified_since) + if resource_id: - events = [e for e in events if e.resource_id == resource_id] + items = [e for e in items if e.resource_id == resource_id] if status: - events = [e for e in events if e.status == status] + if isinstance(status, str): + status = Status(status) + items = [e for e in items if e.status == status] + + from_ = cls.normalize_dt(from_) if from_ else None + to = cls.normalize_dt(to) if to else None + time_ = cls.normalize_dt(time_) if time_ else None + if from_: - events = [e for e in events if e.occurred_at >= from_] + items = [e for e in items if e.occurred_at >= from_] if to: - events = [e for e in events if e.occurred_at < to] + items = [e for e in items if e.occurred_at < to] if time_: - events = [e for e in events if e.occurred_at == time_] - return events + items = [e for e in items if e.occurred_at == time_] + return items class IncidentType(enum.Enum): @@ -116,11 +131,17 @@ class Resolution(enum.Enum): class Incident(NamedObject): def _self_path(self) -> str: + """ Return the API path for this incident. """ return f"/status/incidents/{self.id}" + @field_validator("start", "end", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + status : Status - resource_ids : list[str] = Field(exclude=True) - event_ids : list[str] = Field(exclude=True) + resource_ids : list[str] = Field(default_factory=list, exclude=True) + event_ids : list[str] = Field(default_factory=list, exclude=True) start : datetime.datetime end : datetime.datetime | None type : IncidentType @@ -129,37 +150,39 @@ def _self_path(self) -> str: @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: + """ Return the list of event URIs for this incident. """ return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: + """ Return the list of resource URIs for this incident. """ return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] - @staticmethod - def find( - incidents : list, - name : str | None = None, - description : str | None = None, - status : Status | None = None, - type_ : IncidentType | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - resource_id : str | None = None, - ) -> list: - incidents = NamedObject.find(incidents, name, description, modified_since) + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, status=None, + type_=None, from_= None, to = None, time_ = None, resource_id = None, resolution=None) -> list: + items = super().find(items, name=name, description=description, modified_since=modified_since) + if resource_id: - incidents = [e for e in incidents if resource_id in e.resource_ids] + items = [e for e in items if resource_id in e.resource_ids] if status: - incidents = [e for e in incidents if e.status == status] + items = [e for e in items if e.status == status] if type_: - incidents = [e for e in incidents if e.type == type_] + items = [e for e in items if e.type == type_] + if resolution: + items = [e for e in items if e.resolution == resolution] + + from_ = cls.normalize_dt(from_) if from_ else None + to = cls.normalize_dt(to) if to else None + time_ = cls.normalize_dt(time_) if time_ else None + if from_: - incidents = [e for e in incidents if e.start >= from_] + items = [e for e in items if e.start >= from_] if to: - incidents = [e for e in incidents if e.end < to] + items = [e for e in items if e.end and e.end < to] + if time_: - incidents = [e for e in incidents if e.start <= time_ and e.end > time_] - return incidents + items = [e for e in items + if e.start <= time_ and (e.end is None or e.end > time_)] + return items \ No newline at end of file diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 6a0e948..eb78109 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -31,7 +31,8 @@ async def get_resources( capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status, capability) + return await router.adapter.get_resources(offset=offset, limit=limit, name=name, description=description, group=group, modified_since=modified_since, + resource_type=resource_type, current_status=current_status, capability=capability) @router.get( @@ -77,7 +78,8 @@ async def get_incidents( _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) + return await router.adapter.get_incidents(offset=offset, limit=limit, name=name, description=description, status=status, type_=type_, from_=from_, to=to, + time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) @router.get( @@ -120,7 +122,8 @@ async def get_events( limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: - return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) + return await router.adapter.get_events(incident_id, offset=offset, limit=limit, resource_id=resource_id, name=name, description=description, status=status, + from_=from_, to=to, time_=time_, modified_since=modified_since) @router.get( diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index 6659d15..27654c2 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -1,5 +1,4 @@ from abc import abstractmethod -from typing import Any from . import models as task_models from ..account import models as account_models from ..status import models as status_models @@ -20,6 +19,7 @@ async def get_task( self : "FacilityAdapter", user: account_models.User, task_id: str, + **kwargs ) -> task_models.Task|None: pass @@ -28,6 +28,7 @@ async def get_task( async def get_tasks( self : "FacilityAdapter", user: account_models.User, + **kwargs ) -> list[task_models.Task]: pass @@ -37,7 +38,8 @@ async def put_task( self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource|None, - command: task_models.TaskCommand + task: task_models.TaskCommand, + **kwargs ) -> str: pass @@ -46,78 +48,79 @@ async def put_task( async def on_task( resource: status_models.Resource, user: account_models.User, - cmd: task_models.TaskCommand, + task: task_models.TaskCommand, + **kwargs ) -> tuple[str, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) try: r = None - if cmd.router == "filesystem": - fs_adapter = IriRouter.create_adapter(cmd.router, filesystem_adapter.FacilityAdapter) - if cmd.command == "chmod": - request_model = filesystem_models.PutFileChmodRequest.model_validate(cmd.args["request_model"]) + if task.router == "filesystem": + fs_adapter = IriRouter.create_adapter(task.router, filesystem_adapter.FacilityAdapter) + if task.command == "chmod": + request_model = filesystem_models.PutFileChmodRequest.model_validate(task.args["request_model"]) o = await fs_adapter.chmod(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "chown": - request_model = filesystem_models.PutFileChownRequest.model_validate(cmd.args["request_model"]) + elif task.command == "chown": + request_model = filesystem_models.PutFileChownRequest.model_validate(task.args["request_model"]) o = await fs_adapter.chown(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "file": - o = await fs_adapter.file(resource, user, **cmd.args) + elif task.command == "file": + o = await fs_adapter.file(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "stat": - o = await fs_adapter.stat(resource, user, **cmd.args) + elif task.command == "stat": + o = await fs_adapter.stat(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "mkdir": - request_model = filesystem_models.PostMakeDirRequest.model_validate(cmd.args["request_model"]) + elif task.command == "mkdir": + request_model = filesystem_models.PostMakeDirRequest.model_validate(task.args["request_model"]) o = await fs_adapter.mkdir(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "symlink": - request_model = filesystem_models.PostFileSymlinkRequest.model_validate(cmd.args["request_model"]) + elif task.command == "symlink": + request_model = filesystem_models.PostFileSymlinkRequest.model_validate(task.args["request_model"]) o = await fs_adapter.symlink(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "ls": - o = await fs_adapter.ls(resource, user, **cmd.args) + elif task.command == "ls": + o = await fs_adapter.ls(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "head": - o = await fs_adapter.head(resource, user, **cmd.args) + elif task.command == "head": + o = await fs_adapter.head(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "view": - o = await fs_adapter.view(resource, user, **cmd.args) + elif task.command == "view": + o = await fs_adapter.view(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "tail": - o = await fs_adapter.tail(resource, user, **cmd.args) + elif task.command == "tail": + o = await fs_adapter.tail(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "checksum": - o = await fs_adapter.checksum(resource, user, **cmd.args) + elif task.command == "checksum": + o = await fs_adapter.checksum(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "rm": - o = await fs_adapter.rm(resource, user, **cmd.args) + elif task.command == "rm": + o = await fs_adapter.rm(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "compress": - request_model = filesystem_models.PostCompressRequest.model_validate(cmd.args["request_model"]) + elif task.command == "compress": + request_model = filesystem_models.PostCompressRequest.model_validate(task.args["request_model"]) o = await fs_adapter.compress(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "extract": - request_model = filesystem_models.PostExtractRequest.model_validate(cmd.args["request_model"]) + elif task.command == "extract": + request_model = filesystem_models.PostExtractRequest.model_validate(task.args["request_model"]) o = await fs_adapter.extract(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "mv": - request_model = filesystem_models.PostMoveRequest.model_validate(cmd.args["request_model"]) + elif task.command == "mv": + request_model = filesystem_models.PostMoveRequest.model_validate(task.args["request_model"]) o = await fs_adapter.mv(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "cp": - request_model = filesystem_models.PostCopyRequest.model_validate(cmd.args["request_model"]) + elif task.command == "cp": + request_model = filesystem_models.PostCopyRequest.model_validate(task.args["request_model"]) o = await fs_adapter.cp(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "download": - r = await fs_adapter.download(resource, user, **cmd.args) - elif cmd.command == "upload": - o = await fs_adapter.upload(resource, user, **cmd.args) + elif task.command == "download": + r = await fs_adapter.download(resource, user, **task.args) + elif task.command == "upload": + o = await fs_adapter.upload(resource, user, **task.args) r = "File uploaded successfully" if r: return (r, task_models.TaskStatus.completed) else: - return (f"Task was cancelled due to unknown router/command: {cmd.router}:{cmd.command}", task_models.TaskStatus.failed) + return (f"Task was cancelled due to unknown router/command: {task.router}:{task.command}", task_models.TaskStatus.failed) except Exception as exc: return (f"Error: {exc}", task_models.TaskStatus.failed) diff --git a/app/routers/task/task.py b/app/routers/task/task.py index 094cdd0..1a65459 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -22,10 +22,10 @@ async def get_task( task_id : str, ) -> models.Task: """Get a task""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - task = await router.adapter.get_task(user, task_id) + task = await router.adapter.get_task(user=user, task_id=task_id) if not task: raise HTTPException(status_code=404, detail=f"Task {task_id} not found") return task @@ -42,7 +42,7 @@ async def get_tasks( request : Request, ) -> list[models.Task]: """Get all tasks""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - return await router.adapter.get_tasks(user) + return await router.adapter.get_tasks(user=user) \ No newline at end of file From ccfd2fe1218ceda731484c11050c13b24b0b5b8d Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 3 Feb 2026 20:14:19 -0600 Subject: [PATCH 38/43] Remove kwargs (versioning agreed) --- app/demo_adapter.py | 175 +++++---------------- app/routers/account/facility_adapter.py | 12 +- app/routers/common.py | 11 +- app/routers/compute/facility_adapter.py | 17 +- app/routers/compute/models.py | 52 +----- app/routers/facility/facility_adapter.py | 28 ++-- app/routers/filesystem/facility_adapter.py | 83 ++++------ app/routers/iri_router.py | 12 +- app/routers/status/facility_adapter.py | 18 +-- app/routers/task/facility_adapter.py | 16 +- 10 files changed, 120 insertions(+), 304 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 1a520e8..1a99679 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -275,10 +275,8 @@ def _init_state(self): async def get_facility( self: "DemoAdapter", - modified_since: str | None = None, - **kwargs - ) -> facility_models.Facility: - self._warn_on_unused_kwargs("get_facility", kwargs) + modified_since: str | None = None + ) -> facility_models.Facility: return self.facility @@ -288,10 +286,8 @@ async def list_sites( name: str | None = None, offset: int | None = None, limit: int | None = None, - short_name: str | None = None, - **kwargs - ) -> list[facility_models.Site]: - self._warn_on_unused_kwargs("list_sites", kwargs) + short_name: str | None = None + ) -> list[facility_models.Site]: sites = self.sites if name: @@ -312,10 +308,8 @@ async def list_sites( async def get_site( self: "DemoAdapter", site_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Site: - self._warn_on_unused_kwargs("get_site", kwargs) + modified_since: str | None = None + ) -> facility_models.Site: site = next((s for s in self.sites if s.id == site_id), None) if not site: raise HTTPException(status_code=404, detail="Site not found") @@ -331,10 +325,8 @@ async def get_site( async def get_site_location( self: "DemoAdapter", site_id: str, - modified_since: str | None = None, - **kwargs + modified_since: str | None = None ) -> facility_models.Location: - self._warn_on_unused_kwargs("get_site_location", kwargs) site = await self.get_site(site_id) if not site.location_uri: @@ -360,10 +352,8 @@ async def list_locations( offset: int | None = None, limit: int | None = None, short_name: str | None = None, - country_name: str | None = None, - **kwargs - ) -> list[facility_models.Location]: - self._warn_on_unused_kwargs("list_locations", kwargs) + country_name: str | None = None + ) -> list[facility_models.Location]: locs = self.locations if name: @@ -387,10 +377,8 @@ async def list_locations( async def get_location( self: "DemoAdapter", location_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Location: - self._warn_on_unused_kwargs("get_location", kwargs) + modified_since: str | None = None + ) -> facility_models.Location: location = next((l for l in self.locations if l.id == location_id), None) if not location: @@ -416,10 +404,8 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, current_status : status_models.Status | None = None, - capability: Capability | None = None, - **kwargs + capability: Capability | None = None ) -> list[status_models.Resource]: - self._warn_on_unused_kwargs("get_resources", kwargs) resources = status_models.Resource.find(self.resources, name=name, description=description, group=group, modified_since=modified_since, resource_type=resource_type, current_status=current_status, capability=capability) return paginate_list(resources, offset, limit) @@ -427,10 +413,8 @@ async def get_resources( async def get_resource( self : "DemoAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Resource: - self._warn_on_unused_kwargs("get_resource", kwargs) return status_models.Resource.find_by_id(self.resources, id_) async def get_events( @@ -445,10 +429,8 @@ async def get_events( from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - **kwargs + modified_since : datetime.datetime | None = None ) -> list[status_models.Event]: - self._warn_on_unused_kwargs("get_events", kwargs) events = status_models.Event.find([e for e in self.events if e.incident_id == incident_id], resource_id=resource_id, name=name, description=description, status=status, from_=from_, to=to, time_=time_, modified_since=modified_since) return paginate_list(events, offset, limit) @@ -457,10 +439,8 @@ async def get_events( async def get_event( self : "DemoAdapter", incident_id : str, - id_ : str, - **kwargs + id_ : str ) -> status_models.Event: - self._warn_on_unused_kwargs("get_event", kwargs) return status_models.Event.find_by_id(self.events, id_) @@ -477,10 +457,8 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, - resolution: status_models.Resolution | None = None, - **kwargs + resolution: status_models.Resolution | None = None ) -> list[status_models.Incident]: - self._warn_on_unused_kwargs("get_incidents", kwargs) incidents = status_models.Incident.find(self.incidents, name=name, description=description, status=status, type_=type_,from_=from_, to=to, time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) return paginate_list(incidents, offset, limit) @@ -488,10 +466,8 @@ async def get_incidents( async def get_incident( self : "DemoAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Incident: - self._warn_on_unused_kwargs("get_incident", kwargs) return status_models.Incident.find_by_id(self.incidents, id_) @@ -501,9 +477,7 @@ async def get_capabilities( modified_since : str | None = None, offset : int = 0, limit : int = 1000, - **kwargs ) -> list[Capability]: - self._warn_on_unused_kwargs("get_capabilities", kwargs) caps = list(self.capabilities.values()) if name: caps = [c for c in caps if name.lower() in c.name.lower()] @@ -515,13 +489,11 @@ async def get_current_user( self : "DemoAdapter", api_key: str, client_ip: str, - **kwargs ) -> str: """ In a real deployment, this would decode the api_key jwt and return the current user's id. This method is not async. """ - self._warn_on_unused_kwargs("get_current_user", kwargs) return "gtorok" @@ -530,9 +502,7 @@ async def get_user( user_id: str, api_key: str, client_ip: str|None, - **kwargs ) -> account_models.User: - self._warn_on_unused_kwargs("get_user", kwargs) if user_id != self.user.id: raise HTTPException(status_code=401, detail="User not found") if api_key != self.user.api_key: @@ -542,10 +512,8 @@ async def get_user( async def get_projects( self : "DemoAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[account_models.Project]: - self._warn_on_unused_kwargs("get_projects", kwargs) return self.projects @@ -553,9 +521,7 @@ async def get_project_allocations( self : "DemoAdapter", project: account_models.Project, user: account_models.User, - **kwargs ) -> list[account_models.ProjectAllocation]: - self._warn_on_unused_kwargs("get_project_allocations", kwargs) return [pa for pa in self.project_allocations if pa.project_id == project.id] @@ -563,9 +529,7 @@ async def get_user_allocations( self : "DemoAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation, - **kwargs ) -> list[account_models.UserAllocation]: - self._warn_on_unused_kwargs("get_user_allocations", kwargs) return [ua for ua in self.user_allocations if ua.project_allocation_id == project_allocation.id] @@ -574,9 +538,7 @@ async def submit_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, - **kwargs, ) -> compute_models.Job: - self._warn_on_unused_kwargs("submit_job", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -595,9 +557,7 @@ async def submit_job_script( user: account_models.User, job_script_path: str, args: list[str] = [], - **kwargs ) -> compute_models.Job: - self._warn_on_unused_kwargs("submit_job_script", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -616,9 +576,7 @@ async def update_job( user: account_models.User, job_spec: compute_models.JobSpec, job_id: str, - **kwargs, ) -> compute_models.Job: - self._warn_on_unused_kwargs("update_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -638,9 +596,7 @@ async def get_job( job_id: str, historical: bool = False, include_spec: bool = False, - **kwargs, ) -> compute_models.Job: - self._warn_on_unused_kwargs("get_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -662,9 +618,7 @@ async def get_jobs( filters: dict[str, object] | None = None, historical: bool = False, include_spec: bool = False, - **kwargs, ) -> list[compute_models.Job]: - self._warn_on_unused_kwargs("get_jobs", kwargs) return [compute_models.Job( id=f"job_{i}", status=compute_models.JobStatus( @@ -682,9 +636,7 @@ async def cancel_job( resource: status_models.Resource, user: account_models.User, job_id: str, - **kwargs, ) -> bool: - self._warn_on_unused_kwargs("cancel_job", kwargs) # call slurm/etc. to cancel job return True @@ -754,10 +706,8 @@ async def chmod( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest, - **kwargs, + request_model: filesystem_models.PutFileChmodRequest ) -> filesystem_models.PutFileChmodResponse: - self._warn_on_unused_kwargs("chmod", kwargs) rp = self.validate_path(request_model.path) os.chmod(rp, int(request_model.mode, 8)) return filesystem_models.PutFileChmodResponse( @@ -770,9 +720,7 @@ async def chown( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChownRequest, - **kwargs, ) -> filesystem_models.PutFileChownResponse: - self._warn_on_unused_kwargs("chown", kwargs) rp = self.validate_path(request_model.path) os.chown(rp, request_model.owner, request_model.group) return filesystem_models.PutFileChmodResponse( @@ -789,9 +737,7 @@ async def ls( numeric_uid: bool, recursive: bool, dereference: bool, - **kwargs, ) -> filesystem_models.GetDirectoryLsResponse: - self._warn_on_unused_kwargs("ls", kwargs) rp = self.validate_path(path) files = glob.glob(rp, recursive=recursive) return filesystem_models.GetDirectoryLsResponse( @@ -832,9 +778,7 @@ async def head( file_bytes: int | None, lines: int | None, skip_trailing: bool, - **kwargs, ) -> Tuple[Any, int]: - self._warn_on_unused_kwargs("head", kwargs) return self._headtail("head", path, file_bytes, lines) @@ -845,10 +789,8 @@ async def tail( path: str, file_bytes: int | None, lines: int | None, - skip_trailing: bool, - **kwargs + skip_trailing: bool ) -> Tuple[Any, int]: - self._warn_on_unused_kwargs("tail", kwargs) return self._headtail("tail", path, file_bytes, lines) @@ -858,10 +800,8 @@ async def view( user: account_models.User, path: str, size: int, - offset: int, - **kwargs - ) -> filesystem_models.GetViewFileResponse: - self._warn_on_unused_kwargs("view", kwargs) + offset: int + ) -> filesystem_models.GetViewFileResponse: rp = self.validate_path(path) result = subprocess.run( f"tail -c +{offset+1} {rp} | head -c {size}", @@ -879,10 +819,8 @@ async def checksum( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> filesystem_models.GetFileChecksumResponse: - self._warn_on_unused_kwargs("checksum", kwargs) + path: str + ) -> filesystem_models.GetFileChecksumResponse: rp = self.validate_path(path) result = subprocess.run( ["sha256sum", rp], @@ -901,10 +839,8 @@ async def file( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs + path: str ) -> filesystem_models.GetFileTypeResponse: - self._warn_on_unused_kwargs("file", kwargs) rp = self.validate_path(path) result = subprocess.run( ["file", "-b", rp], @@ -921,10 +857,8 @@ async def stat( resource: status_models.Resource, user: account_models.User, path: str, - dereference: bool, - **kwargs + dereference: bool ) -> filesystem_models.GetFileStatResponse: - self._warn_on_unused_kwargs("stat", kwargs) rp = self.validate_path(path) if dereference: stat_info = os.stat(rp) @@ -951,9 +885,7 @@ async def rm( resource: status_models.Resource, user: account_models.User, path: str, - **kwargs ): - self._warn_on_unused_kwargs("rm", kwargs) rp = self.validate_path(path) if rp == PathSandbox.get_base_temp_dir(): raise HTTPException(status_code=400, detail="Cannot delete sandbox") @@ -965,10 +897,8 @@ async def mkdir( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMakeDirRequest, - **kwargs - ) -> filesystem_models.PostMkdirResponse: - self._warn_on_unused_kwargs("mkdir", kwargs) + request_model: filesystem_models.PostMakeDirRequest + ) -> filesystem_models.PostMkdirResponse: rp = self.validate_path(request_model.path) args = ["mkdir"] if request_model.parent: @@ -984,10 +914,8 @@ async def symlink( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostFileSymlinkRequest, - **kwargs + request_model: filesystem_models.PostFileSymlinkRequest ) -> filesystem_models.PostFileSymlinkResponse: - self._warn_on_unused_kwargs("symlink", kwargs) rp_src = self.validate_path(request_model.path) rp_dst = self.validate_path(request_model.link_path) subprocess.run(["ln", "-s", rp_src, rp_dst], check=True) @@ -1000,10 +928,8 @@ async def download( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs + path: str ) -> Any: - self._warn_on_unused_kwargs("download", kwargs) rp = self.validate_path(path) raw_content = pathlib.Path(rp).read_bytes() @@ -1018,10 +944,8 @@ async def upload( resource: status_models.Resource, user: account_models.User, path: str, - content: str, - **kwargs + content: str ) -> None: - self._warn_on_unused_kwargs("upload", kwargs) rp = self.validate_path(path) if isinstance(content, bytes): pathlib.Path(rp).write_bytes(content) @@ -1035,10 +959,8 @@ async def compress( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCompressRequest, - **kwargs - ) -> filesystem_models.PostCompressResponse: - self._warn_on_unused_kwargs("compress", kwargs) + request_model: filesystem_models.PostCompressRequest + ) -> filesystem_models.PostCompressResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -1070,10 +992,8 @@ async def extract( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostExtractRequest, - **kwargs - ) -> filesystem_models.PostExtractResponse: - self._warn_on_unused_kwargs("extract", kwargs) + request_model: filesystem_models.PostExtractRequest + ) -> filesystem_models.PostExtractResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -1100,10 +1020,8 @@ async def mv( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMoveRequest, - **kwargs - ) -> filesystem_models.PostMoveResponse: - self._warn_on_unused_kwargs("mv", kwargs) + request_model: filesystem_models.PostMoveRequest + ) -> filesystem_models.PostMoveResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) subprocess.run(["mv", src_rp, dst_rp], check=True) @@ -1116,10 +1034,8 @@ async def cp( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCopyRequest, - **kwargs - ) -> filesystem_models.PostCopyResponse: - self._warn_on_unused_kwargs("cp", kwargs) + request_model: filesystem_models.PostCopyRequest + ) -> filesystem_models.PostCopyResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) args = ["cp"] @@ -1136,20 +1052,16 @@ async def cp( async def get_task( self : "DemoAdapter", user: account_models.User, - task_id: str, - **kwargs + task_id: str ) -> task_models.Task|None: - self._warn_on_unused_kwargs("get_task", kwargs) await DemoTaskQueue._process_tasks(self) return next((t for t in DemoTaskQueue.tasks if t.user.name == user.name and t.id == task_id), None) async def get_tasks( self : "DemoAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[task_models.Task]: - self._warn_on_unused_kwargs("get_tasks", kwargs) await DemoTaskQueue._process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] @@ -1158,10 +1070,7 @@ async def put_task( self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, - task: str, - **kwargs, - ) -> str: - self._warn_on_unused_kwargs("put_task", kwargs) + task: str) -> str: await DemoTaskQueue._process_tasks(self) return DemoTaskQueue._create_task(user, resource, task) diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 3abca87..334e682 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -17,8 +17,7 @@ async def get_capabilities( name : str | None = None, modified_since : str | None = None, offset : int = 0, - limit : int = 1000, - **kwargs + limit : int = 1000 ) -> list[Capability]: pass @@ -26,8 +25,7 @@ async def get_capabilities( @abstractmethod async def get_projects( self : "FacilityAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[account_models.Project]: pass @@ -36,8 +34,7 @@ async def get_projects( async def get_project_allocations( self : "FacilityAdapter", project: account_models.Project, - user: account_models.User, - **kwargs + user: account_models.User ) -> list[account_models.ProjectAllocation]: pass @@ -46,7 +43,6 @@ async def get_project_allocations( async def get_user_allocations( self : "FacilityAdapter", user: account_models.User, - project_allocation: account_models.ProjectAllocation, - **kwargs + project_allocation: account_models.ProjectAllocation ) -> list[account_models.UserAllocation]: pass diff --git a/app/routers/common.py b/app/routers/common.py index 2826dab..62e2fe7 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -4,7 +4,7 @@ import enum from typing import Optional from urllib.parse import parse_qs - +from collections.abc import Iterable from pydantic_core import core_schema from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer, field_validator from fastapi import Request, HTTPException, status @@ -245,8 +245,14 @@ def find_by_id(cls, items, id_, allow_name: bool = False): @classmethod def find(cls, items, name=None, description=None, modified_since=None): """ Find objects matching the given criteria. """ + single = False if not any((name, description, modified_since)): return items + + if not isinstance(items, Iterable) or isinstance(items, BaseModel): + items = [items] + single = True + if name: items = [item for item in items if item.name == name] if description: @@ -254,6 +260,9 @@ def find(cls, items, name=None, description=None, modified_since=None): if modified_since: modified_since = cls.normalize_dt(modified_since) items = [item for item in items if item.last_modified >= modified_since] + + if single: + return items[0] if items else None return items diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index d70ec51..910cacd 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -18,9 +18,8 @@ async def submit_job( self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - job_spec: compute_models.JobSpec, - **kwargs - ) -> compute_models.Job: + job_spec: compute_models.JobSpec + ) -> compute_models.Job: pass @@ -30,8 +29,7 @@ async def submit_job_script( resource: status_models.Resource, user: account_models.User, job_script_path: str, - args: list[str] = [], - **kwargs + args: list[str] = [] ) -> compute_models.Job: pass @@ -42,8 +40,7 @@ async def update_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, - job_id: str, - **kwargs + job_id: str ) -> compute_models.Job: pass @@ -69,8 +66,7 @@ async def get_jobs( limit : int, filters: dict[str, object] | None = None, historical: bool = False, - include_spec: bool = False, - **kwargs + include_spec: bool = False ) -> list[compute_models.Job]: pass @@ -80,7 +76,6 @@ async def cancel_job( self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - job_id: str, - **kwargs + job_id: str ) -> bool: pass diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 3aa4121..3ec1053 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,11 +1,10 @@ from typing import Annotated from enum import IntEnum -from pydantic import field_serializer, StrictBool, Field +from pydantic import field_serializer, ConfigDict, StrictBool, Field from ..common import IRIBaseModel class ResourceSpec(IRIBaseModel): -<<<<<<< HEAD """ Specification of computational resources required for a job. """ @@ -28,37 +27,13 @@ class JobAttributes(IRIBaseModel): reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} -======= - node_count: int | None = Field(default=None, description="Number of compute nodes to allocate") - process_count: int | None = Field(default=None, description="Total number of processes to launch") - processes_per_node: int | None = Field(default=None, description="Number of processes to launch per node") - cpu_cores_per_process: int | None = Field(default=None, description="Number of CPU cores to allocate per process") - gpu_cores_per_process: int | None = Field(default=None, description="Number of GPU cores to allocate per process") - exclusive_node_use: StrictBool = Field(default=True, description="Whether to request exclusive use of allocated nodes") - memory: int | None = Field(default=None, description="Amount of memory to allocate in bytes") - - -class JobAttributes(IRIBaseModel): - duration: int = Field(default=None, ge=1, description="Duration in seconds", examples=[30, 60, 120]) - queue_name: str | None = Field(default=None, min_length=1, description="Name of the queue or partition to submit the job to") - account: str | None = Field(default=None, min_length=1, description="Account or project to charge for resource usage") - reservation_id: str | None = Field(default=None, min_length=1, description="ID of a reservation to use for the job") - custom_attributes: dict[str, str] = Field(default={}, description="Custom scheduler-specific attributes as key-value pairs") ->>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) - class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ -<<<<<<< HEAD source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True -======= - source: str = Field(description="The source path on the host system to mount") - target: str = Field(description="The target path inside the container where the volume will be mounted") - read_only: StrictBool = Field(default=True, description="Whether the mount should be read-only") ->>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class Container(IRIBaseModel): """ @@ -69,11 +44,9 @@ class Container(IRIBaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ -<<<<<<< HEAD image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] - class JobSpec(IRIBaseModel): """ Specification for job. @@ -94,28 +67,6 @@ class JobSpec(IRIBaseModel): pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None -======= - image: str = Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')") - volume_mounts: list[VolumeMount] = Field(default=[], description="List of volume mounts for the container") - - -class JobSpec(IRIBaseModel): - executable: str | None = Field(default=None, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.") - container: Container | None = Field(default=None, description="Container specification for containerized execution") - arguments: list[str] = Field(default=[], description="Command-line arguments to pass to the executable or container") - directory: str | None = Field(default=None, description="Working directory for the job") - name: str | None = Field(default=None, description="Name of the job") - inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment variables from the submission environment") - environment: dict[str, str] = Field(default={}, description="Environment variables to set for the job. If container is specified, these will be set inside the container.") - stdin_path: str | None = Field(default=None, description="Path to file to use as standard input") - stdout_path: str | None = Field(default=None, description="Path to file to write standard output") - stderr_path: str | None = Field(default=None, description="Path to file to write standard error") - resources: ResourceSpec | None = Field(default=None, description="Resource requirements for the job") - attributes: JobAttributes | None = Field(default=None, description="Additional job attributes such as duration, queue, and account") - pre_launch: str | None = Field(default=None, description="Script or commands to run before launching the job") - post_launch: str | None = Field(default=None, description="Script or commands to run after the job completes") - launcher: str | None = Field(default=None, description="Job launcher to use (e.g., 'mpirun', 'srun')") ->>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class CommandResult(IRIBaseModel): @@ -157,7 +108,6 @@ class JobState(IntEnum): CANCELED = 5 """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" - class JobStatus(IRIBaseModel): state : JobState time : float | None = None diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index fc6777f..0bb9ca3 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -13,9 +13,8 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_facility( self: "FacilityAdapter", - modified_since: str | None = None, - **kwargs - ) -> facility_models.Facility | None: + modified_since: str | None = None + ) -> facility_models.Facility | None: pass @abstractmethod @@ -25,27 +24,24 @@ async def list_sites( name: str | None = None, offset: int | None = None, limit: int | None = None, - short_name: str | None = None, - **kwargs - ) -> list[facility_models.Site]: + short_name: str | None = None + ) -> list[facility_models.Site]: pass @abstractmethod async def get_site( self: "FacilityAdapter", site_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Site | None: + modified_since: str | None = None + ) -> facility_models.Site | None: pass @abstractmethod async def get_site_location( self: "FacilityAdapter", site_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Location | None: + modified_since: str | None = None + ) -> facility_models.Location | None: pass @abstractmethod @@ -56,8 +52,7 @@ async def list_locations( offset: int | None = None, limit: int | None = None, short_name: str | None = None, - country_name: str | None = None, - **kwargs + country_name: str | None = None ) -> list[facility_models.Location]: pass @@ -65,7 +60,6 @@ async def list_locations( async def get_location( self: "FacilityAdapter", location_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Location | None: + modified_since: str | None = None + ) -> facility_models.Location | None: pass diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 8506f1c..636b0a9 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -29,9 +29,8 @@ async def chmod( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest, - **kwargs - ) -> filesystem_models.PutFileChmodResponse: + request_model: filesystem_models.PutFileChmodRequest + ) -> filesystem_models.PutFileChmodResponse: pass @@ -40,9 +39,8 @@ async def chown( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChownRequest, - **kwargs - ) -> filesystem_models.PutFileChownResponse: + request_model: filesystem_models.PutFileChownRequest + ) -> filesystem_models.PutFileChownResponse: pass @@ -55,8 +53,7 @@ async def ls( show_hidden: bool, numeric_uid: bool, recursive: bool, - dereference: bool, - **kwargs + dereference: bool ) -> filesystem_models.GetDirectoryLsResponse: pass @@ -69,8 +66,7 @@ async def head( path: str, file_bytes: int, lines: int, - skip_trailing: bool, - **kwargs + skip_trailing: bool ) -> Tuple[Any, int]: pass @@ -83,9 +79,8 @@ async def tail( path: str, file_bytes: int | None, lines: int | None, - skip_trailing: bool, - **kwargs - ) -> Tuple[Any, int]: + skip_trailing: bool + ) -> Tuple[Any, int]: pass @@ -96,9 +91,8 @@ async def view( user: account_models.User, path: str, size: int, - offset: int, - **kwargs - ) -> filesystem_models.GetViewFileResponse: + offset: int + ) -> filesystem_models.GetViewFileResponse: pass @@ -107,9 +101,8 @@ async def checksum( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> filesystem_models.GetFileChecksumResponse: + path: str + ) -> filesystem_models.GetFileChecksumResponse: pass @@ -118,9 +111,8 @@ async def file( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> filesystem_models.GetFileTypeResponse: + path: str + ) -> filesystem_models.GetFileTypeResponse: pass @@ -130,9 +122,8 @@ async def stat( resource: status_models.Resource, user: account_models.User, path: str, - dereference: bool, - **kwargs - ) -> filesystem_models.GetFileStatResponse: + dereference: bool + ) -> filesystem_models.GetFileStatResponse: pass @@ -141,9 +132,7 @@ async def rm( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ): + path: str): pass @@ -152,9 +141,8 @@ async def mkdir( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMakeDirRequest, - **kwargs - ) -> filesystem_models.PostMkdirResponse: + request_model: filesystem_models.PostMakeDirRequest + ) -> filesystem_models.PostMkdirResponse: pass @@ -164,8 +152,7 @@ async def symlink( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest, - **kwargs - ) -> filesystem_models.PostFileSymlinkResponse: + ) -> filesystem_models.PostFileSymlinkResponse: pass @@ -174,9 +161,8 @@ async def download( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> Any: + path: str + ) -> Any: pass @@ -186,9 +172,8 @@ async def upload( resource: status_models.Resource, user: account_models.User, path: str, - content: str, - **kwargs - ) -> None: + content: str + ) -> None: pass @@ -197,9 +182,8 @@ async def compress( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCompressRequest, - **kwargs - ) -> filesystem_models.PostCompressResponse: + request_model: filesystem_models.PostCompressRequest + ) -> filesystem_models.PostCompressResponse: pass @@ -208,9 +192,8 @@ async def extract( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostExtractRequest, - **kwargs - ) -> filesystem_models.PostExtractResponse: + request_model: filesystem_models.PostExtractRequest + ) -> filesystem_models.PostExtractResponse: pass @@ -219,9 +202,8 @@ async def mv( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMoveRequest, - **kwargs - ) -> filesystem_models.PostMoveResponse: + request_model: filesystem_models.PostMoveRequest + ) -> filesystem_models.PostMoveResponse: pass @@ -230,7 +212,6 @@ async def cp( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCopyRequest, - **kwargs - ) -> filesystem_models.PostCopyResponse: + request_model: filesystem_models.PostCopyRequest + ) -> filesystem_models.PostCopyResponse: pass diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index d4cb6f2..be31762 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -98,18 +98,11 @@ async def current_user( class AuthenticatedAdapter(ABC): - def _warn_on_unused_kwargs(self, func_name: str, kwargs: dict) -> None: - if not kwargs: - return - logging.getLogger().warning("Adapter method '%s' received unused kwargs: %s", func_name, - ", ".join(sorted(kwargs.keys()))) - @abstractmethod async def get_current_user( self : "AuthenticatedAdapter", api_key: str, - client_ip: str|None, - **kwargs + client_ip: str|None ) -> str: """ Decode the api_key and return the authenticated user's id. @@ -124,8 +117,7 @@ async def get_user( self : "AuthenticatedAdapter", user_id: str, api_key: str, - client_ip: str|None, - **kwargs + client_ip: str|None ) -> User: """ Retrieve additional user information (name, email, etc.) for the given user_id. diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 1d774cb..274e9f5 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -24,8 +24,7 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type: status_models.ResourceType = Query(default=None), current_status: status_models.Status = Query(default=None), - capability: Capability | None = None, - **kwargs + capability: Capability | None = None ) -> list[status_models.Resource]: pass @@ -33,8 +32,7 @@ async def get_resources( @abstractmethod async def get_resource( self : "FacilityAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Resource: pass @@ -52,8 +50,7 @@ async def get_events( from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - **kwargs + modified_since : datetime.datetime | None = None ) -> list[status_models.Event]: pass @@ -62,8 +59,7 @@ async def get_events( async def get_event( self : "FacilityAdapter", incident_id : str, - id_ : str, - **kwargs + id_ : str ) -> status_models.Event: pass @@ -82,8 +78,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, - resolution: status_models.Resolution | None = None, - **kwargs + resolution: status_models.Resolution | None = None ) -> list[status_models.Incident]: pass @@ -91,7 +86,6 @@ async def get_incidents( @abstractmethod async def get_incident( self : "FacilityAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Incident: pass diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index 27654c2..47f686e 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -18,8 +18,7 @@ class FacilityAdapter(AuthenticatedAdapter): async def get_task( self : "FacilityAdapter", user: account_models.User, - task_id: str, - **kwargs + task_id: str ) -> task_models.Task|None: pass @@ -27,8 +26,7 @@ async def get_task( @abstractmethod async def get_tasks( self : "FacilityAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[task_models.Task]: pass @@ -38,9 +36,8 @@ async def put_task( self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource|None, - task: task_models.TaskCommand, - **kwargs - ) -> str: + task: task_models.TaskCommand + ) -> str: pass @@ -48,9 +45,8 @@ async def put_task( async def on_task( resource: status_models.Resource, user: account_models.User, - task: task_models.TaskCommand, - **kwargs - ) -> tuple[str, task_models.TaskStatus]: + task: task_models.TaskCommand + ) -> tuple[str, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) try: From 1206be92768c653a05065702a4803628a4086130 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 4 Feb 2026 05:39:49 -0600 Subject: [PATCH 39/43] get_resource single id --- app/routers/compute/compute.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 804f9a5..5f44d52 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -41,7 +41,7 @@ async def submit_job( raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs @@ -78,7 +78,7 @@ async def submit_job( # raise HTTPException(status_code=404, detail="User not found") # # # look up the resource (todo: maybe ensure it's available) -# resource = await status_router.adapter.get_resource(resource_id=resource_id) +# resource = await status_router.adapter.get_resource(resource_id) # # # the handler can use whatever means it wants to submit the job and then fill in its id # # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs @@ -113,7 +113,7 @@ async def update_job( raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs @@ -143,7 +143,7 @@ async def get_job_status( # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) job = await router.adapter.get_job(resource=resource, user=user, job_id=job_id, historical=historical, include_spec=include_spec) @@ -175,7 +175,7 @@ async def get_job_statuses( # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) jobs = await router.adapter.get_jobs(resource=resource, user=user, offset=offset, limit=limit, filters=filters, historical=historical, include_spec=include_spec) @@ -203,7 +203,7 @@ async def cancel_job( raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) await router.adapter.cancel_job(resource=resource, user=user, job_id=job_id) From b141dbdd807aaf50e9c571e791e5f686c90bfcc8 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 4 Feb 2026 19:28:14 -0600 Subject: [PATCH 40/43] Add paginate in demo_adapter. Fix number of params for override method --- app/demo_adapter.py | 12 ++++++++++++ app/routers/common.py | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index a36c27e..81b8460 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -22,6 +22,14 @@ DEMO_QUEUE_UPDATE_SECS = 5 +def paginate_list(items, offset: int | None, limit: int | None): + """Return a sliced items using offset and limit.""" + if offset is not None and offset > 0: + items = items[offset:] + if limit is not None and limit >= 0: + items = items[:limit] + return items + class PathSandbox: _base_temp_dir = None @@ -473,6 +481,10 @@ async def get_incident( async def get_capabilities( self : "DemoAdapter", + name : str | None = None, + modified_since : str | None = None, + offset : int = 0, + limit : int = 1000 ) -> list[Capability]: return self.capabilities.values() diff --git a/app/routers/common.py b/app/routers/common.py index ce91f4a..75879b9 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -11,7 +11,6 @@ from .. import config - # These are Pydantic custom types for strict validation # that are not implmented in Pydantic by default. # ----------------------------------------------------------------------- From ef5c6e24c1ad2ebe926799c6bc38eaecbda0b6de Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 5 Feb 2026 06:51:48 -0600 Subject: [PATCH 41/43] Harden validation and response allingment with spec This includes: - application/problem+json for error responses - Generate URL-safe instance values - Normalize non-string validation error into strings (fastapi validation can give list/dict) - Exclude None/empty fields from locations/capabilities reponses - Capability (according to spec) should use NamedObject, but overrides last_modified as optional. --- app/routers/account/account.py | 3 +-- app/routers/common.py | 15 +++++++----- app/routers/error_handlers.py | 41 +++++++++++++++++++++++++++----- app/routers/facility/facility.py | 2 +- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index e13cbde..6aa6e23 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -18,8 +18,7 @@ description="Get a list of capabilities at this facility.", responses=DEFAULT_RESPONSES, operation_id="getCapabilities", - -) + response_model_exclude_none=True) async def get_capabilities( request : Request, name : str = Query(default=None, min_length=1), diff --git a/app/routers/common.py b/app/routers/common.py index 75879b9..a6a91ec 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -229,7 +229,7 @@ def find_by_id(cls, items, id_, allow_name: bool = False): if not matches: return None if len(matches) > 1: - raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id}'") + raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id_}'") return matches[0] @@ -250,8 +250,8 @@ def find(cls, items, name=None, description=None, modified_since=None): items = [item for item in items if item.description and description in item.description] if modified_since: modified_since = cls.normalize_dt(modified_since) - items = [item for item in items if item.last_modified >= modified_since] - + items = [item for item in items + if item.last_modified and item.last_modified >= modified_since] if single: return items[0] if items else None return items @@ -263,7 +263,7 @@ class AllocationUnit(enum.Enum): inodes = "inodes" -class Capability(IRIBaseModel): +class Capability(NamedObject): """ An aspect of a resource that can have an allocation. For example, Perlmutter nodes with GPUs @@ -271,6 +271,9 @@ class Capability(IRIBaseModel): It is a way to further subdivide a resource into allocatable sub-resources. The word "capability" is also known to users as something they need for a job to run. (eg. gpu) """ - id: str - name: str + def _self_path(self) -> str: + return f"/account/capabilities/{self.id}" + + last_modified: StrictDateTime | None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.") + units: list[AllocationUnit] diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 337b5fc..bf781be 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -3,7 +3,7 @@ Default problem schema and example responses for various HTTP status codes. """ import logging -from urllib.parse import unquote +from urllib.parse import urlsplit, urlunsplit, quote from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse from fastapi.exceptions import RequestValidationError @@ -16,12 +16,36 @@ def get_url_base(request: Request) -> str: proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}/problems" +def safe_instance_url(request: Request) -> str: + """Return a URL-safe version of the request URL for the 'instance' field.""" + parts = urlsplit(str(request.url)) + + # Encode unsafe characters in each component + safe_path = quote(parts.path, safe="/:@&+$,;=-._~") + safe_query = quote(parts.query, safe="=&?/:@+$,;=-._~") + safe_fragment = quote(parts.fragment, safe="=&?/:@+$,;=-._~") + + return urlunsplit((parts.scheme, parts.netloc, safe_path, safe_query, safe_fragment)) + def problem_response(*, request: Request, status: int, - title: str, detail: str, problem_type: str, + title, detail, problem_type: str, invalid_params=None, extra_headers=None): """Return a JSON problem response with the given status, title, and detail.""" - instance = unquote(str(request.url)) + instance = safe_instance_url(request) url_base = get_url_base(request) + + # Normalize title and detail to strings (Official spec says they must be strings) + # but fastapi validation errors may provide lists/dicts + if not isinstance(title, str): + title = "Error" + + if not isinstance(detail, str): + if isinstance(detail, list): + detail = ", ".join(err.get("msg", str(err)) if isinstance(err, dict) else str(err) + for err in detail) + else: + detail = str(detail) + body = { "type": f"{url_base}/{problem_type}", "title": title, @@ -34,7 +58,13 @@ def problem_response(*, request: Request, status: int, body["invalid_params"] = invalid_params headers = extra_headers or {} - return JSONResponse(status_code=status, content=body, headers=headers) + return JSONResponse( + status_code=status, + content=body, + headers=headers, + media_type="application/problem+json" + ) + def install_error_handlers(app: FastAPI): @@ -45,8 +75,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError invalid_params = [] for err in exc.errors(): - loc = err.get("loc", []) - name = loc[-1] if loc else "unknown" + name = str((err.get("loc") or ["unknown"])[-1]) reason = err.get("msg", "Invalid parameter") invalid_params.append({"name": name, "reason": reason}) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 7d264af..d804b9c 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -51,7 +51,7 @@ async def get_site_location( """Get site location by site ID""" return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations", response_model_exclude_none=True) async def list_locations( request : Request, modified_since: StrictDateTime = Query(default=None), From 3273b8af3e97d41ff72441f5c044ec802a104b5a Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 6 Feb 2026 22:41:12 -0600 Subject: [PATCH 42/43] Add site_id under resource. Fix schemathesis run with excluding none values --- app/demo_adapter.py | 44 ++++++++++++++++---------- app/routers/status/facility_adapter.py | 3 +- app/routers/status/models.py | 8 ++++- app/routers/status/status.py | 1 + 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index d9c72fd..9779e9e 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -134,21 +134,30 @@ def _init_state(self): "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } - pm = status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", capability_ids=[ - self.capabilities["cpu"].id, - self.capabilities["gpu"].id, - ], current_status=status_models.Status.degraded, last_modified=day_ago, resource_type=status_models.ResourceType.compute) - hpss = status_models.Resource(id=str(uuid.uuid4()), group="hpss", name="hpss", description="hpss tape storage", capability_ids=[self.capabilities["hpss"].id], current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.storage) - cfs = status_models.Resource(id=str(uuid.uuid4()), group="cfs", name="cfs", description="cfs storage", capability_ids=[self.capabilities["gpfs"].id], current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.storage) - - self.resources = [ - pm, - hpss, - cfs, - status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="login nodes", description="the perlmutter computer login nodes", capability_ids=[], current_status=status_models.Status.degraded, last_modified=day_ago, resource_type=status_models.ResourceType.system), - status_models.Resource(id=str(uuid.uuid4()), group="services", name="Iris", description="Iris webapp", capability_ids=[], current_status=status_models.Status.down, last_modified=day_ago, resource_type=status_models.ResourceType.website), - status_models.Resource(id=str(uuid.uuid4()), group="services", name="sfapi", description="the Superfacility API", capability_ids=[], current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.service), - ] + pm = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", + capability_ids=[self.capabilities["cpu"].id, self.capabilities["gpu"].id,], current_status=status_models.Status.degraded, + last_modified=day_ago, resource_type=status_models.ResourceType.compute, located_at_uri=site1.self_uri) + + hpss = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="hpss", name="hpss", description="hpss tape storage", + capability_ids=[self.capabilities["hpss"].id], current_status=status_models.Status.up, + last_modified=day_ago, resource_type=status_models.ResourceType.storage, located_at_uri=site1.self_uri) + + cfs = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="cfs", name="cfs", description="cfs storage", + capability_ids=[self.capabilities["gpfs"].id], current_status=status_models.Status.up, + last_modified=day_ago, resource_type=status_models.ResourceType.storage, located_at_uri=site1.self_uri) + + login = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="perlmutter", name="login nodes", description="the perlmutter computer login nodes", + capability_ids=[], current_status=status_models.Status.degraded, + last_modified=day_ago, resource_type=status_models.ResourceType.system, located_at_uri=site2.self_uri) + + iris = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="services", name="Iris", description="Iris webapp", + capability_ids=[], current_status=status_models.Status.down, + last_modified=day_ago, resource_type=status_models.ResourceType.website, located_at_uri=site2.self_uri) + sfapi = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="services", name="sfapi", description="the Superfacility API", + capability_ids=[], current_status=status_models.Status.up, + last_modified=day_ago, resource_type=status_models.ResourceType.service, located_at_uri=site2.self_uri) + + self.resources = [pm, hpss, cfs, login, iris, sfapi] self.projects = [ account_models.Project( @@ -318,10 +327,11 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, current_status : status_models.Status | None = None, - capability: Capability | None = None + capability: Capability | None = None, + site_id: str | None = None ) -> list[status_models.Resource]: resources = status_models.Resource.find(self.resources, name=name, description=description, group=group, modified_since=modified_since, - resource_type=resource_type, current_status=current_status, capability=capability) + resource_type=resource_type, current_status=current_status, capability=capability, site_id=site_id) return paginate_list(resources, offset, limit) diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 274e9f5..a17f610 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -24,7 +24,8 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type: status_models.ResourceType = Query(default=None), current_status: status_models.Status = Query(default=None), - capability: Capability | None = None + capability: Capability | None = None, + site_id: str | None = None ) -> list[status_models.Resource]: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 88fb3b4..d721306 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -34,6 +34,9 @@ def _self_path(self) -> str: """ Return the API path for this resource. """ return f"/status/resources/{self.id}" + # NOTE (TBR): If site_id is required, then located_at_uri should be also required. This can be easily identified by Site.self_uri + # Is there a specific Resource, that has no Site? + site_id: str = Field(..., description="The site identifier this resource is located at") capability_ids: list[str] = Field(default_factory=list, exclude=True) group: str | None current_status: Status | None = Field(default=None, description="The current status comes from the status of the last event for this resource") @@ -49,7 +52,8 @@ def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] @classmethod - def find(cls, items, name=None, description=None, modified_since=None, group=None, resource_type=None, current_status=None, capability=None) -> list: + def find(cls, items, name=None, description=None, modified_since=None, group=None, + resource_type=None, current_status=None, capability=None, site_id=None) -> list: items = super().find(items, name=name, description=description, modified_since=modified_since) if group: items = [item for item in items if item.group == group] @@ -62,6 +66,8 @@ def find(cls, items, name=None, description=None, modified_since=None, group=Non if capability: items = [item for item in items if any(cap_id in item.capability_ids for cap_id in capability)] + if site_id: + items = [item for item in items if item.site_id == site_id] return items class Event(NamedObject): diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 5b50a4a..424564a 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -17,6 +17,7 @@ description="Get a list of all resources at this facility. You can optionally filter the returned list by specifying attribtes.", responses=DEFAULT_RESPONSES, operation_id="getResources", + response_model_exclude_none=True ) async def get_resources( request : Request, From 68a72bc1fc14465280820c0e92e42bf1bdbcbf4c Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sat, 7 Feb 2026 08:15:38 -0600 Subject: [PATCH 43/43] Split common.py into it's own types --- app/demo_adapter.py | 39 ++-- app/routers/account/account.py | 10 +- app/routers/account/facility_adapter.py | 5 +- app/routers/account/models.py | 6 +- app/routers/common.py | 279 ------------------------ app/routers/compute/compute.py | 18 +- app/routers/compute/models.py | 8 +- app/routers/facility/facility.py | 9 +- app/routers/facility/models.py | 7 +- app/routers/status/facility_adapter.py | 6 +- app/routers/status/models.py | 6 +- app/routers/status/status.py | 11 +- app/types/__init__.py | 0 app/types/base.py | 101 +++++++++ app/types/http.py | 90 ++++++++ app/types/models.py | 21 ++ app/types/scalars.py | 97 ++++++++ 17 files changed, 383 insertions(+), 330 deletions(-) delete mode 100644 app/routers/common.py create mode 100644 app/types/__init__.py create mode 100644 app/types/base.py create mode 100644 app/types/http.py create mode 100644 app/types/models.py create mode 100644 app/types/scalars.py diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9779e9e..b028554 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,24 +1,33 @@ +import base64 import datetime -import random -import uuid +import glob +import grp import os -import stat +import pathlib import pwd -import grp -import glob +import random +import stat import subprocess -import pathlib -import base64 +import uuid from typing import Any, Tuple -from pydantic import BaseModel + from fastapi import HTTPException -from .routers.common import AllocationUnit, Capability -from .routers.facility import models as facility_models, facility_adapter as facility_adapter -from .routers.status import models as status_models, facility_adapter as status_adapter -from .routers.account import models as account_models, facility_adapter as account_adapter -from .routers.compute import models as compute_models, facility_adapter as compute_adapter -from .routers.filesystem import models as filesystem_models, facility_adapter as filesystem_adapter -from .routers.task import models as task_models, facility_adapter as task_adapter +from pydantic import BaseModel + +from .routers.account import facility_adapter as account_adapter +from .routers.account import models as account_models +from .routers.compute import facility_adapter as compute_adapter +from .routers.compute import models as compute_models +from .routers.facility import facility_adapter +from .routers.facility import models as facility_models +from .routers.filesystem import facility_adapter as filesystem_adapter +from .routers.filesystem import models as filesystem_models +from .routers.status import facility_adapter as status_adapter +from .routers.status import models as status_models +from .routers.task import facility_adapter as task_adapter +from .routers.task import models as task_models +from .types.models import Capability +from .types.scalars import AllocationUnit DEMO_QUEUE_UPDATE_SECS = 5 diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 6aa6e23..abd55ad 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -1,9 +1,11 @@ -from fastapi import HTTPException, Request, Depends, Query -from . import models, facility_adapter +from fastapi import Depends, HTTPException, Query, Request + +from ...types.http import forbidExtraQueryParams +from ...types.models import Capability +from ...types.scalars import StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import forbidExtraQueryParams, StrictDateTime, Capability - +from . import facility_adapter, models router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 334e682..fb2a66f 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,7 +1,8 @@ from abc import abstractmethod -from . import models as account_models -from ..common import Capability + +from ...types.models import Capability from ..iri_router import AuthenticatedAdapter +from . import models as account_models class FacilityAdapter(AuthenticatedAdapter): diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 1a9333d..6bde3be 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,8 @@ -from pydantic import computed_field, Field +from pydantic import Field, computed_field + from ... import config -from ..common import IRIBaseModel, AllocationUnit +from ...types.base import IRIBaseModel +from ...types.scalars import AllocationUnit class User(IRIBaseModel): diff --git a/app/routers/common.py b/app/routers/common.py deleted file mode 100644 index a6a91ec..0000000 --- a/app/routers/common.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Default models used by multiple routers.""" -import datetime -from email.utils import parsedate_to_datetime -import enum -from typing import Optional -from urllib.parse import parse_qs -from collections.abc import Iterable -from pydantic_core import core_schema -from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer, field_validator -from fastapi import Request, HTTPException, status - -from .. import config - -# These are Pydantic custom types for strict validation -# that are not implmented in Pydantic by default. -# ----------------------------------------------------------------------- -# StrictBool: a strict boolean type -class StrictBool: - """Strict boolean: - - Accepts: real booleans, 'true', 'false' - - Rejects everything else. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - """Validate the input value as a strict boolean.""" - if isinstance(value, bool): - return value - if isinstance(value, str): - v = value.strip().lower() - if v == "true": - return True - if v == "false": - return False - raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") - raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "boolean", - "description": "Strict boolean. Only true/false allowed (bool or string)." - } - -# ----------------------------------------------------------------------- -# StrictDateTime: a strict ISO8601 datetime type - -class StrictDateTime: - """ - Strict ISO8601 datetime: - - Accepts datetime objects - - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 - - Converts 'Z' → UTC - - Converts naive datetimes → UTC - - Rejects integers ("0"), null, garbage strings, etc. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - if isinstance(value, datetime.datetime): - return StrictDateTime._normalize(value) - if not isinstance(value, str): - raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") - v = value.strip() - if v.endswith("Z"): - v = v[:-1] + "+00:00" - try: - dt = datetime.datetime.fromisoformat(v) - except Exception as ex: - raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex - - return StrictDateTime._normalize(dt) - - - @staticmethod - def modifiedSinceDatetime( - modified_since: str | None, - header_modified_since: str | None - ) -> datetime.datetime | None: - """ - Combine modified_since (ISO8601) and If-Modified-Since (RFC1123). - If both are provided, the most recent timestamp is used. - """ - - parsed_times: list[datetime.datetime] = [] - - # Query param (ISO 8601) - if modified_since is not None: - try: - dt = StrictDateTime.validate(modified_since) - parsed_times.append(dt) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid modified_since query param: {exc}", - ) from exc - - # Header (RFC 1123) - if header_modified_since is not None: - try: - dt = parsedate_to_datetime(header_modified_since) - if dt is None: - raise ValueError("Invalid RFC1123 date") - if dt.tzinfo is None: - dt = dt.replace(tzinfo=datetime.timezone.utc) - parsed_times.append(dt.astimezone(datetime.timezone.utc)) - except Exception as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invalid If-Modified-Since header format (must be RFC1123)", - ) from exc - - if not parsed_times: - return None - - # Stricter constraint wins - return max(parsed_times) - - @staticmethod - def _normalize(dt: datetime.datetime) -> datetime.datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } - - -def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): - multiParams = multiParams or set() - - async def checker(req: Request): - if "*" in allowedParams: - return - - raw_qs = req.scope.get("query_string", b"") - parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) - - allowed = set(allowedParams) - - for key, values in parsed.items(): - if key not in allowed: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}"}]) - - - if len(values) > 1 and key not in multiParams: - raise HTTPException(status_code=422, - detail=[{"type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}"}]) - - return checker - - -class IRIBaseModel(BaseModel): - """Base model for IRI models.""" - model_config = ConfigDict(extra="allow") - - @model_serializer(mode="wrap") - def _hide_extra(self, handler, info): - data = handler(self) - - model_fields = set(self.model_fields or {}) - computed_fields = set(self.model_computed_fields or {}) - extra = getattr(self, "__pydantic_extra__", {}) or {} - for k in extra: - if k not in model_fields and k not in computed_fields: - data.pop(k, None) - return data - - def get_extra(self, key, default=None): - return getattr(self, "__pydantic_extra__", {}).get(key, default) - - -class NamedObject(IRIBaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - - @classmethod - def normalize_dt(cls, dt: datetime | None) -> datetime | None: - """Normalize datetime to UTC-aware.""" - # Convert naive datetimes into UTC-aware versions - if dt is None: - return None - if isinstance(dt, str): - dt = StrictDateTime.validate(dt) - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @field_validator("last_modified", mode="before") - @classmethod - def _norm_dt_field(cls, v): - return cls.normalize_dt(v) - - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - - @classmethod - def find_by_id(cls, items, id_, allow_name: bool = False): - """ Find an object by its id or name == id. """ - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - matches = [r for r in items if r.id == id_ or (allow_name and r.name == id_)] - if not matches: - return None - if len(matches) > 1: - raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id_}'") - - return matches[0] - - @classmethod - def find(cls, items, name=None, description=None, modified_since=None): - """ Find objects matching the given criteria. """ - single = False - if not any((name, description, modified_since)): - return items - - if not isinstance(items, Iterable) or isinstance(items, BaseModel): - items = [items] - single = True - - if name: - items = [item for item in items if item.name == name] - if description: - items = [item for item in items if item.description and description in item.description] - if modified_since: - modified_since = cls.normalize_dt(modified_since) - items = [item for item in items - if item.last_modified and item.last_modified >= modified_since] - if single: - return items[0] if items else None - return items - - -class AllocationUnit(enum.Enum): - node_hours = "node_hours" - bytes = "bytes" - inodes = "inodes" - - -class Capability(NamedObject): - """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) - """ - def _self_path(self) -> str: - return f"/account/capabilities/{self.id}" - - last_modified: StrictDateTime | None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.") - - units: list[AllocationUnit] diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 5a11c8a..d1eadf4 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,12 +1,12 @@ """Compute resource API router""" -from fastapi import HTTPException, Request, Depends, status, Query -from . import models, facility_adapter -from .. import iri_router +from fastapi import Depends, HTTPException, Query, Request, status +from ...types.http import forbidExtraQueryParams +from ...types.scalars import StrictHTTPBool +from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router -from ..common import forbidExtraQueryParams, StrictBool - +from . import facility_adapter, models router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -132,8 +132,8 @@ async def get_job_status( resource_id : str, job_id : str, request : Request, - historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), - include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + historical : StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" @@ -164,8 +164,8 @@ async def get_job_statuses( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, - historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), - include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + historical : StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index a56d4fe..87142a4 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,7 +1,9 @@ -from typing import Annotated from enum import IntEnum -from pydantic import field_serializer, ConfigDict, StrictBool, Field -from ..common import IRIBaseModel +from typing import Annotated + +from pydantic import ConfigDict, Field, StrictBool, field_serializer + +from ...types.base import IRIBaseModel class ResourceSpec(IRIBaseModel): diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index d842224..7a1a1ea 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,9 +1,10 @@ -from fastapi import Request, Depends, Query +from fastapi import Depends, Query, Request + +from ...types.http import forbidExtraQueryParams +from ...types.scalars import StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from .import models, facility_adapter -from ..common import StrictDateTime, forbidExtraQueryParams - +from . import facility_adapter, models router = iri_router.IriRouter(facility_adapter.FacilityAdapter, prefix="/facility", diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 32bc866..021d9a2 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,8 +1,9 @@ """Facility-related models.""" -from typing import Optional, List +from typing import List, Optional + from pydantic import Field, HttpUrl -from ..common import NamedObject +from ...types.base import NamedObject class Site(NamedObject): @@ -20,7 +21,6 @@ def _self_path(self) -> str: longitude: Optional[float] = Field(None, description="Longitude of the Location.") resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - @classmethod def find(cls, items, name=None, description=None, modified_since=None, short_name=None, country_name=None): """ Find Locations matching the given criteria. """ @@ -32,7 +32,6 @@ def find(cls, items, name=None, description=None, modified_since=None, short_nam return items - class Facility(NamedObject): def _self_path(self) -> str: return "/facility" diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index a17f610..6be6c88 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -1,8 +1,10 @@ -from abc import ABC, abstractmethod import datetime +from abc import ABC, abstractmethod + from fastapi import Query + +from ...types.models import Capability from . import models as status_models -from ..common import Capability class FacilityAdapter(ABC): diff --git a/app/routers/status/models.py b/app/routers/status/models.py index d721306..7f0d51d 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -1,9 +1,11 @@ import datetime import enum from typing import Optional -from pydantic import BaseModel, computed_field, Field, field_validator, HttpUrl + +from pydantic import BaseModel, Field, HttpUrl, computed_field, field_validator + from ... import config -from ..common import NamedObject +from ...types.base import NamedObject class Link(BaseModel): diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 424564a..6ffa0ba 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,9 +1,12 @@ -from typing import Optional, List, Annotated -from fastapi import HTTPException, Request, Query, Depends -from . import models, facility_adapter +from typing import Annotated, List, Optional + +from fastapi import Depends, HTTPException, Query, Request + +from ...types.http import forbidExtraQueryParams +from ...types.scalars import AllocationUnit, StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import StrictDateTime, forbidExtraQueryParams, AllocationUnit +from . import facility_adapter, models router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/types/__init__.py b/app/types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/types/base.py b/app/types/base.py new file mode 100644 index 0000000..4a3b245 --- /dev/null +++ b/app/types/base.py @@ -0,0 +1,101 @@ +"""Default models used by multiple routers.""" +import datetime +from collections.abc import Iterable +from typing import Optional + +from pydantic import (BaseModel, ConfigDict, Field, computed_field, + field_validator, model_serializer) + +from .. import config +from .scalars import StrictDateTime + + +class IRIBaseModel(BaseModel): + """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _hide_extra(self, handler, info): + data = handler(self) + + model_fields = set(self.model_fields or {}) + computed_fields = set(self.model_computed_fields or {}) + extra = getattr(self, "__pydantic_extra__", {}) or {} + for k in extra: + if k not in model_fields and k not in computed_fields: + data.pop(k, None) + return data + + def get_extra(self, key, default=None): + """Get an extra field value that is not defined in the model. Returns default if not found.""" + return getattr(self, "__pydantic_extra__", {}).get(key, default) + + +class NamedObject(IRIBaseModel): + """Base model for named objects.""" + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @classmethod + def normalize_dt(cls, dt: datetime | None) -> datetime | None: + """Normalize datetime to UTC-aware.""" + # Convert naive datetimes into UTC-aware versions + if dt is None: + return None + if isinstance(dt, str): + dt = StrictDateTime.validate(dt) + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @field_validator("last_modified", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @classmethod + def find_by_id(cls, items, id_, allow_name: bool = False): + """ Find an object by its id or name == id. """ + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + matches = [r for r in items if r.id == id_ or (allow_name and r.name == id_)] + if not matches: + return None + if len(matches) > 1: + raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id_}'") + + return matches[0] + + @classmethod + def find(cls, items, name=None, description=None, modified_since=None): + """ Find objects matching the given criteria. """ + single = False + if not any((name, description, modified_since)): + return items + + if not isinstance(items, Iterable) or isinstance(items, BaseModel): + items = [items] + single = True + + if name: + items = [item for item in items if item.name == name] + if description: + items = [item for item in items if item.description and description in item.description] + if modified_since: + modified_since = cls.normalize_dt(modified_since) + items = [item for item in items + if item.last_modified and item.last_modified >= modified_since] + if single: + return items[0] if items else None + return items diff --git a/app/types/http.py b/app/types/http.py new file mode 100644 index 0000000..c59c8ef --- /dev/null +++ b/app/types/http.py @@ -0,0 +1,90 @@ +"""HTTP-related types and utilities for the IRI Facility API""" +import datetime +from email.utils import parsedate_to_datetime +from urllib.parse import parse_qs + +from fastapi import HTTPException, Request, status + +from .scalars import StrictDateTime + +# ----------------------------------------------------------------------- +# modifiedSinceDatetime: combine modified_since (ISO8601) and If-Modified-Since (RFC1123) +# If both are provided, the most recent timestamp is used. Strict validation is applied to both formats. +# modified_since must be a valid ISO8601 datetime string. +# If-Modified-Since must be a valid RFC1123 datetime string. +# TODO: If-Modified-Since is not yet supported/used by the API. + +def modifiedSinceDatetime( + modified_since: str | None, + header_modified_since: str | None +) -> datetime.datetime | None: + """ + Combine modified_since (ISO8601) and If-Modified-Since (RFC1123). + If both are provided, the most recent timestamp is used. + """ + + parsed_times: list[datetime.datetime] = [] + + # Query param (ISO 8601) + if modified_since is not None: + try: + dt = StrictDateTime.validate(modified_since) + parsed_times.append(dt) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid modified_since query param: {exc}", + ) from exc + + # Header (RFC 1123) + if header_modified_since is not None: + try: + dt = parsedate_to_datetime(header_modified_since) + if dt is None: + raise ValueError("Invalid RFC1123 date") + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + parsed_times.append(dt.astimezone(datetime.timezone.utc)) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid If-Modified-Since header format (must be RFC1123)", + ) from exc + + if not parsed_times: + return None + + # Stricter constraint wins + return max(parsed_times) + +# ----------------------------------------------------------------------- +# forbidExtraQueryParams: a dependency to forbid extra query parameters + +def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): + """Dependency to forbid extra query parameters. If allowedParams contains "*", all params are allowed.""" + multiParams = multiParams or set() + + async def checker(req: Request): + if "*" in allowedParams: + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + + allowed = set(allowedParams) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException(status_code=422, + detail=[{"type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}"}]) + + + if len(values) > 1 and key not in multiParams: + raise HTTPException(status_code=422, + detail=[{"type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}"}]) + + return checker diff --git a/app/types/models.py b/app/types/models.py new file mode 100644 index 0000000..9378f62 --- /dev/null +++ b/app/types/models.py @@ -0,0 +1,21 @@ +"""Models for the IRI Facility API.""" +from pydantic import Field + +from .base import NamedObject +from .scalars import AllocationUnit, StrictDateTime + + +class Capability(NamedObject): + """ + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + """ + def _self_path(self) -> str: + return f"/account/capabilities/{self.id}" + + last_modified: StrictDateTime | None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.") + + units: list[AllocationUnit] diff --git a/app/types/scalars.py b/app/types/scalars.py new file mode 100644 index 0000000..582efce --- /dev/null +++ b/app/types/scalars.py @@ -0,0 +1,97 @@ +"""Scalar types for the IRI Facility API""" +# pylint: disable=unused-argument +import datetime +import enum + +from pydantic_core import core_schema + +# ----------------------------------------------------------------------- +# StrictHTTPBool: a strict boolean type + +class StrictHTTPBool: + """Strict boolean: + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + v = value.strip().lower() + if v == "true": + return True + if v == "false": + return False + raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") + raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "boolean", + "description": "Strict boolean. Only true/false allowed (bool or string)." + } + +# ----------------------------------------------------------------------- +# StrictDateTime: a strict ISO8601 datetime type + +class StrictDateTime: + """ + Strict ISO8601 datetime: + - Accepts datetime objects + - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 + - Converts 'Z' → UTC + - Converts naive datetimes → UTC + - Rejects integers ("0"), null, garbage strings, etc. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict ISO8601 datetime.""" + if isinstance(value, datetime.datetime): + return StrictDateTime._normalize(value) + if not isinstance(value, str): + raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") + v = value.strip() + if v.endswith("Z"): + v = v[:-1] + "+00:00" + try: + dt = datetime.datetime.fromisoformat(v) + except Exception as ex: + raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex + + return StrictDateTime._normalize(dt) + + @staticmethod + def _normalize(dt: datetime.datetime) -> datetime.datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "string", + "format": "date-time", + "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." + } + +# ----------------------------------------------------------------------- +# AllocationUnit: an enum for allocation units + +class AllocationUnit(enum.Enum): + """Units for allocation""" + node_hours = "node_hours" + bytes = "bytes" + inodes = "inodes"