Skip to content
This repository was archived by the owner on Apr 13, 2026. It is now read-only.

Commit b76ea2e

Browse files
haoruizhouhaoruizhou
committed
Add multi-season support to data-downloader
Introduce multi-season configuration and handling across the data-downloader service and frontend. Key changes: - Config: add SeasonConfig and _parse_seasons to read SEASONS env var; expose seasons in Settings with sensible defaults. - Backend services: manage per-season Runs/Sensors repositories (file suffixing), return seasons list via /api/seasons, accept optional season parameter for runs/sensors/note/query endpoints, scan all configured seasons and store results per-season, and default to newest season when none provided. - Influx/query: fetch_signal_series now accepts a database override and service query routes pass season-specific DBs. - Scanner/Storage: adjust season start/end boundaries (season starts Aug previous year, ends Jan 1 next year); storage filenames include season suffix. - Periodic worker: support scheduling either by interval or daily time via SCAN_DAILY_TIME env var. - Frontend: fetch seasons, add season selector UI, propagate selected season to all API calls and adapt visuals (accent color), and add season type to types. - Docker compose: add SEASONS and SCAN_DAILY_TIME environment wiring and minor formatting fixes. - .gitignore: ignore installer/data-downloader/data. Fallbacks and defaults are provided when SEASONS is unset, and scanning continues per-season even if individual season scans fail.
1 parent 7a41e17 commit b76ea2e

13 files changed

Lines changed: 383 additions & 146 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,4 +206,5 @@ installer/slackbot/logs/*
206206

207207
installer/sandbox/prompt-guide.txt
208208

209-
wfr-telemetry
209+
wfr-telemetry
210+
/installer/data-downloader/data

‎installer/data-downloader/backend/app.py‎

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,19 @@ def healthcheck() -> dict:
4141
return {"status": "ok"}
4242

4343

44+
@app.get("/api/seasons")
45+
def list_seasons() -> List[dict]:
46+
return service.get_seasons()
47+
48+
4449
@app.get("/api/runs")
45-
def list_runs() -> dict:
46-
return service.get_runs()
50+
def list_runs(season: str | None = None) -> dict:
51+
return service.get_runs(season=season)
4752

4853

4954
@app.get("/api/sensors")
50-
def list_sensors() -> dict:
51-
return service.get_sensors()
55+
def list_sensors(season: str | None = None) -> dict:
56+
return service.get_sensors(season=season)
5257

5358

5459
@app.get("/api/scanner-status")
@@ -57,10 +62,10 @@ def scanner_status() -> dict:
5762

5863

5964
@app.post("/api/runs/{key}/note")
60-
def save_note(key: str, payload: NotePayload) -> dict:
61-
run = service.update_note(key, payload.note.strip())
65+
def save_note(key: str, payload: NotePayload, season: str | None = None) -> dict:
66+
run = service.update_note(key, payload.note.strip(), season=season)
6267
if not run:
63-
raise HTTPException(status_code=404, detail=f"Run {key} not found")
68+
raise HTTPException(status_code=404, detail=f"Run {key} not found (season={season})")
6469
return run
6570

6671

@@ -70,8 +75,16 @@ def trigger_scan(background_tasks: BackgroundTasks) -> dict:
7075
return {"status": "scheduled"}
7176

7277

78+
@app.post("/api/query")
79+
def query_signal(payload: DataQueryPayload, season: str | None = None) -> dict:
7380
limit = None if payload.no_limit else (payload.limit or 2000)
74-
return service.query_signal_series(payload.signal, payload.start, payload.end, limit)
81+
return service.query_signal_series(
82+
payload.signal,
83+
payload.start,
84+
payload.end,
85+
limit,
86+
season=season
87+
)
7588

7689

7790
@app.get("/", response_class=HTMLResponse)
@@ -87,9 +100,12 @@ def index():
87100
influx_status = f"Error: {e}"
88101
influx_color = "red"
89102

103+
# Default to first season for overview
90104
runs = service.get_runs()
91105
sensors = service.get_sensors()
92106
scanner_status = service.get_scanner_status()
107+
seasons_list = service.get_seasons()
108+
seasons_html = ", ".join([f"{s['name']} ({s['year']})" for s in seasons_list])
93109

94110
html = f"""
95111
<!DOCTYPE html>
@@ -112,26 +128,23 @@ def index():
112128
<h2>System Status</h2>
113129
<p><strong>InfluxDB Connection:</strong> <span style="color: {influx_color}">{influx_status}</span></p>
114130
<p><strong>Scanner Status:</strong> {scanner_status.get('status', 'Unknown')} (Last run: {scanner_status.get('last_run', 'Never')})</p>
115-
<p><strong>API Version:</strong> 1.0.0</p>
131+
<p><strong>API Version:</strong> 1.1.0 (Multi-Season Support)</p>
116132
</div>
117133
118134
<div class="card">
119-
<h2>Data Stats</h2>
120-
<ul>
121-
<li><strong>Runs Found:</strong> {len(runs.get('runs', []))}</li>
122-
<li><strong>Sensors Found:</strong> {len(sensors)}</li>
123-
</ul>
135+
<h2>Active Config</h2>
136+
<p><strong>Seasons Configured:</strong> {seasons_html}</p>
124137
</div>
125138
126139
<div class="card">
127-
<h2>Configuration</h2>
140+
<h2>Default Season Stats ({seasons_list[0]['name'] if seasons_list else 'None'})</h2>
128141
<ul>
129-
<li><strong>Influx Host:</strong> <code>{settings.influx_host}</code></li>
130-
<li><strong>Database:</strong> <code>{settings.influx_database}</code></li>
142+
<li><strong>Runs Found:</strong> {len(runs.get('runs', []))}</li>
143+
<li><strong>Sensors Found:</strong> {len(sensors.get('sensors', []))}</li>
131144
</ul>
132145
</div>
133146
134-
<p><a href="/docs">View API Documentation</a> | <a href="http://localhost:3000">Open Frontend</a></p>
147+
<p><a href="/docs">API Docs</a> | <a href="/api/seasons">JSON Seasons List</a> | <a href="http://localhost:3000">Frontend</a></p>
135148
</body>
136149
</html>
137150
"""

‎installer/data-downloader/backend/config.py‎

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,26 +12,76 @@ def _parse_origins(raw: str | None) -> List[str]:
1212
return [origin.strip() for origin in raw.split(",") if origin.strip()]
1313

1414

15+
class SeasonConfig(BaseModel):
16+
name: str # e.g. "WFR25"
17+
year: int # e.g. 2025
18+
database: str # e.g. "WFR25"
19+
color: str | None = None # e.g. "222 76 153"
20+
21+
22+
def _parse_seasons(raw: str | None) -> List[SeasonConfig]:
23+
"""Parse SEASONS env var: "WFR25:2025:222 76 153,WFR26:2026:..."."""
24+
if not raw:
25+
# Default fallback if not set
26+
return [SeasonConfig(name="WFR25", year=2025, database="WFR25", color="#DE4C99")]
27+
28+
seasons = []
29+
for part in raw.split(","):
30+
part = part.strip()
31+
if not part:
32+
continue
33+
try:
34+
# Split into at most 3 parts: Name, Year, Color
35+
parts = part.split(":", 2)
36+
name = parts[0]
37+
38+
if len(parts) >= 2:
39+
year = int(parts[1])
40+
else:
41+
# Malformed or simple format not supported purely by regex?
42+
# Actually if just "WFR25", split gives ['WFR25']
43+
# require at least year
44+
continue
45+
46+
color = parts[2] if len(parts) > 2 else None
47+
48+
# Assume DB name matches Season Name
49+
seasons.append(SeasonConfig(name=name, year=year, database=name, color=color))
50+
except ValueError:
51+
continue
52+
53+
if not seasons:
54+
return [SeasonConfig(name="WFR25", year=2025, database="WFR25")]
55+
56+
# Sort by year descending (newest first)
57+
seasons.sort(key=lambda s: s.year, reverse=True)
58+
return seasons
59+
60+
1561
class Settings(BaseModel):
1662
"""Centralised configuration pulled from environment variables."""
1763

1864
data_dir: str = Field(default_factory=lambda: os.getenv("DATA_DIR", "./data"))
1965

2066
influx_host: str = Field(default_factory=lambda: os.getenv("INFLUX_HOST", "http://localhost:9000"))
2167
influx_token: str = Field(default_factory=lambda: os.getenv("INFLUX_TOKEN", ""))
22-
influx_database: str = Field(default_factory=lambda: os.getenv("INFLUX_DATABASE", "WFR25"))
68+
69+
# Global/Default Influx settings (used for connectivity check or default fallback)
2370
influx_schema: str = Field(default_factory=lambda: os.getenv("INFLUX_SCHEMA", "iox"))
2471
influx_table: str = Field(default_factory=lambda: os.getenv("INFLUX_TABLE", "WFR25"))
2572

26-
scanner_year: int = Field(default_factory=lambda: int(os.getenv("SCANNER_YEAR", "2025")))
27-
scanner_bin: str = Field(default_factory=lambda: os.getenv("SCANNER_BIN", "hour")) # hour or day
73+
seasons: List[SeasonConfig] = Field(default_factory=lambda: _parse_seasons(os.getenv("SEASONS")))
74+
75+
# Scanner settings common to all seasons (unless we want per-season granularity later)
76+
scanner_bin: str = Field(default_factory=lambda: os.getenv("SCANNER_BIN", "hour"))
2877
scanner_include_counts: bool = Field(default_factory=lambda: os.getenv("SCANNER_INCLUDE_COUNTS", "true").lower() == "true")
2978
scanner_initial_chunk_days: int = Field(default_factory=lambda: int(os.getenv("SCANNER_INITIAL_CHUNK_DAYS", "31")))
3079

3180
sensor_window_days: int = Field(default_factory=lambda: int(os.getenv("SENSOR_WINDOW_DAYS", "7")))
3281
sensor_lookback_days: int = Field(default_factory=lambda: int(os.getenv("SENSOR_LOOKBACK_DAYS", "30")))
3382

3483
periodic_interval_seconds: int = Field(default_factory=lambda: int(os.getenv("SCAN_INTERVAL_SECONDS", "3600")))
84+
scan_daily_time: str | None = Field(default_factory=lambda: os.getenv("SCAN_DAILY_TIME"))
3585

3686
allowed_origins: List[str] = Field(default_factory=lambda: _parse_origins(os.getenv("ALLOWED_ORIGINS", "*")))
3787

‎installer/data-downloader/backend/influx_queries.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,14 @@ def _normalize(dt: datetime) -> datetime:
1414
return dt.astimezone(timezone.utc)
1515

1616

17-
def fetch_signal_series(settings: Settings, signal: str, start: datetime, end: datetime, limit: int | None) -> dict:
17+
def fetch_signal_series(
18+
settings: Settings,
19+
signal: str,
20+
start: datetime,
21+
end: datetime,
22+
limit: int | None,
23+
database: str | None = None
24+
) -> dict:
1825
start_dt = _normalize(start)
1926
end_dt = _normalize(end)
2027
if start_dt >= end_dt:
@@ -35,8 +42,11 @@ def fetch_signal_series(settings: Settings, signal: str, start: datetime, end: d
3542
AND time <= TIMESTAMP '{end_dt.isoformat()}'
3643
ORDER BY time{limit_clause}
3744
"""
45+
46+
# Use provided database or fallback to default setting
47+
target_db = database if database else settings.influx_database
3848

39-
with InfluxDBClient3(host=settings.influx_host, token=settings.influx_token, database=settings.influx_database) as client:
49+
with InfluxDBClient3(host=settings.influx_host, token=settings.influx_token, database=target_db) as client:
4050
tbl = client.query(sql)
4151
points = []
4252
for idx in range(tbl.num_rows):
@@ -56,6 +66,7 @@ def fetch_signal_series(settings: Settings, signal: str, start: datetime, end: d
5666
"start": start_dt.isoformat(),
5767
"end": end_dt.isoformat(),
5868
"limit": limit,
69+
"database": target_db,
5970
"row_count": len(points),
6071
"points": points,
6172
"sql": " ".join(line.strip() for line in sql.strip().splitlines()),

‎installer/data-downloader/backend/periodic_worker.py‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import asyncio
44
import logging
5+
from datetime import datetime, timedelta
56

67
from backend.config import get_settings
78
from backend.services import DataDownloaderService
@@ -12,14 +13,37 @@
1213
async def run_worker():
1314
settings = get_settings()
1415
service = DataDownloaderService(settings)
16+
1517
interval = max(30, settings.periodic_interval_seconds)
16-
logging.info("Starting periodic scanner loop (interval=%ss)", interval)
18+
daily_time = settings.scan_daily_time
19+
20+
if daily_time:
21+
logging.info(f"Starting periodic scanner loop (daily at {daily_time})")
22+
else:
23+
logging.info(f"Starting periodic scanner loop (interval={interval}s)")
24+
1725
while True:
1826
try:
1927
logging.info("Running scheduled scan...")
2028
service.run_full_scan(source="periodic")
2129
logging.info("Finished scheduled scan.")
22-
await asyncio.sleep(interval)
30+
31+
if daily_time:
32+
# Calculate seconds until next occurrence of daily_time
33+
now = datetime.now()
34+
target_hour, target_minute = map(int, daily_time.split(":"))
35+
target = now.replace(hour=target_hour, minute=target_minute, second=0, microsecond=0)
36+
37+
if target <= now:
38+
# If target time has passed today, schedule for tomorrow
39+
target += timedelta(days=1)
40+
41+
sleep_seconds = (target - now).total_seconds()
42+
logging.info(f"Next scan scheduled for {target} (in {sleep_seconds:.0f}s)")
43+
await asyncio.sleep(sleep_seconds)
44+
else:
45+
await asyncio.sleep(interval)
46+
2347
except Exception:
2448
logging.exception("Scheduled scan failed. Retrying in 60s...")
2549
await asyncio.sleep(60)

‎installer/data-downloader/backend/server_scanner.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,12 @@ def tz(self) -> ZoneInfo:
3737

3838
@property
3939
def start(self) -> datetime:
40-
return datetime(self.year, 1, 1, tzinfo=UTC)
40+
# Season starts in August of the previous year
41+
return datetime(self.year - 1, 8, 1, tzinfo=UTC)
4142

4243
@property
4344
def end(self) -> datetime:
45+
# Season ends at the end of the configured year (Jan 1 of year + 1)
4446
return datetime(self.year + 1, 1, 1, tzinfo=UTC)
4547

4648

0 commit comments

Comments
 (0)