added submission scores and pending model payouts. soft deprecated round model performances v2. added better tests

numerapi/base_api.py

@@ -3,6 +3,7 @@
 import datetime
 import logging
 import os
+import warnings
 from io import BytesIO
 from typing import Dict, List, Tuple, Union
 
@@ -841,9 +842,145 @@ def diagnostics(self, model_id: str, diagnostics_id: str | None = None) -> Dict:
         utils.replace(results, "updatedAt", utils.parse_datetime_string)
         return results
 
+    def submission_scores(
+        self,
+        model_id: str,
+        display_name: str | None = None,
+        version: str | None = None,
+        day: int | None = None,
+        resolved: bool | None = None,
+        tournament: int | None = None,
+        last_n_rounds: int | None = None,
+        distinct_on_round: bool | None = None,
+    ) -> List[Dict]:
+        """Fetch submission score history for a model.
+
+        Args:
+            model_id (str): target model UUID
+            display_name (str, optional): score metric name filter
+            version (str, optional): score version filter
+            day (int, optional): day filter
+            resolved (bool, optional): resolved-state filter
+            tournament (int, optional): tournament filter, defaults to the
+                API instance tournament
+            last_n_rounds (int, optional): limit by most recent rounds
+            distinct_on_round (bool, optional): keep only the latest score per
+                round after applying other filters
+
+        Returns:
+            list of dicts: list of submission score entries
+        """
+
+        query = """
+          query($modelId: ID!
+                $displayName: String
+                $version: String
+                $day: Int
+                $resolved: Boolean
+                $tournament: Int
+                $lastNRounds: Int
+                $distinctOnRound: Boolean) {
+            submissionScores(modelId: $modelId
+                             displayName: $displayName
+                             version: $version
+                             day: $day
+                             resolved: $resolved
+                             tournament: $tournament
+                             lastNRounds: $lastNRounds
+                             distinctOnRound: $distinctOnRound) {
+                roundId
+                submissionId
+                roundNumber
+                roundResolveTime
+                roundScoreTime
+                roundCloseStakingTime
+                value
+                percentile
+                displayName
+                version
+                date
+                day
+                resolveDate
+                resolved
+            }
+          }
+        """
+        arguments = {
+            "modelId": model_id,
+            "displayName": display_name,
+            "version": version,
+            "day": day,
+            "resolved": resolved,
+            "tournament": self.tournament_id if tournament is None else tournament,
+            "lastNRounds": last_n_rounds,
+            "distinctOnRound": distinct_on_round,
+        }
+        scores = self.raw_query(query, arguments)["data"]["submissionScores"]
+        for score in scores:
+            utils.replace(score, "roundResolveTime", utils.parse_datetime_string)
+            utils.replace(score, "roundScoreTime", utils.parse_datetime_string)
+            utils.replace(score, "roundCloseStakingTime", utils.parse_datetime_string)
+            utils.replace(score, "date", utils.parse_datetime_string)
+            utils.replace(score, "resolveDate", utils.parse_datetime_string)
+        return scores
+
+    def pending_model_payouts(self, tournament: int | None = None) -> Dict:
+        """Fetch actual and pending payouts for the authenticated user's models.
+
+        Args:
+            tournament (int, optional): tournament filter, defaults to the API
+                instance tournament
+
+        Returns:
+            dict: payout groups with `actual` and `pending` lists
+        """
+
+        query = """
+          query($tournament: Int!) {
+            pendingModelPayouts(tournament: $tournament) {
+                actual {
+                    roundId
+                    roundNumber
+                    roundResolveTime
+                    modelId
+                    modelName
+                    modelDisplayName
+                    payoutNmr
+                    payoutValue
+                    currencySymbol
+                }
+                pending {
+                    roundId
+                    roundNumber
+                    roundResolveTime
+                    modelId
+                    modelName
+                    modelDisplayName
+                    payoutNmr
+                    payoutValue
+                    currencySymbol
+                }
+            }
+          }
+        """
+        arguments = {
+            "tournament": self.tournament_id if tournament is None else tournament
+        }
+        payouts = self.raw_query(query, arguments, authorization=True)["data"][
+            "pendingModelPayouts"
+        ]
+        for payout_type in ["actual", "pending"]:
+            for payout in payouts[payout_type]:
+                utils.replace(payout, "roundResolveTime", utils.parse_datetime_string)
+                utils.replace(payout, "payoutNmr", utils.parse_float_string)
+                utils.replace(payout, "payoutValue", utils.parse_float_string)
+        return payouts
+
     def round_model_performances_v2(self, model_id: str):
         """Fetch round model performance of a user.
 
+        DEPRECATED - please use `submission_scores` instead when possible.
+
         Args:
             model_id (str)
 
@@ -871,6 +1008,13 @@ def round_model_performances_v2(self, model_id: str):
                     * percentile (`float`)
                     * value (`float`): value of the metric
         """
+        warnings.warn(
+            "`round_model_performances_v2` is deprecated because it relies on "
+            "`v2RoundModelPerformances`. Use `submission_scores` instead when "
+            "possible.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
 
         query = """
           query($modelId: String!

tests/conftest.py

@@ -5,8 +5,71 @@
 import sys
 from pathlib import Path
 
+import pytest
+
 ROOT = Path(__file__).resolve().parents[1]
 root_str = str(ROOT)
 if root_str not in sys.path:
     sys.path.insert(0, root_str)
 
+from numerapi import base_api
+
+DEFAULT_FAKE_GRAPHQL_API_URL = "https://fake-api-tournament.numer.ai"
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("graphql")
+    group.addoption(
+        "--mode",
+        action="store",
+        choices=("mock", "integration"),
+        default="mock",
+        help=(
+            "Select whether tests use a fake GraphQL API base URL or a real one. "
+            "Live API tests only run in integration mode."
+        ),
+    )
+    group.addoption(
+        "--api-url",
+        action="store",
+        default=None,
+        help=(
+            "Base GraphQL API URL to use in tests when --mode=integration. "
+            "Defaults to numerapi.base_api.API_TOURNAMENT_URL."
+        ),
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "live_api: test requires a real GraphQL API backend",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--mode") == "integration":
+        return
+
+    skip_live_api = pytest.mark.skip(
+        reason="requires --mode=integration to run against a real GraphQL API backend"
+    )
+    for item in items:
+        if "live_api" in item.keywords:
+            item.add_marker(skip_live_api)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def configure_graphql_api_url(pytestconfig):
+    original_url = base_api.API_TOURNAMENT_URL
+    mode = pytestconfig.getoption("--mode")
+    configured_url = pytestconfig.getoption("--api-url")
+
+    if mode == "integration":
+        base_api.API_TOURNAMENT_URL = configured_url or original_url
+    else:
+        base_api.API_TOURNAMENT_URL = DEFAULT_FAKE_GRAPHQL_API_URL
+
+    yield base_api.API_TOURNAMENT_URL
+
+    base_api.API_TOURNAMENT_URL = original_url