diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 6a1010fb..8e1ecf95 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -17,6 +17,7 @@ jobs: outputs: should_build: ${{ steps.check.outputs.should_build }} version: ${{ steps.extract.outputs.version }} + publish_version: ${{ steps.plan.outputs.publish_version }} steps: - name: Checkout repository @@ -32,19 +33,36 @@ jobs: echo "version=${VERSION}" >> $GITHUB_OUTPUT echo "Extracted version: ${VERSION}" - - name: Check if version exists on PyPI or TestPyPI - id: check + - name: Plan publish version + id: plan run: | - VERSION="${{ steps.extract.outputs.version }}" - - # Determine which PyPI to check based on event type + BASE_VERSION="${{ steps.extract.outputs.version }}" + if [ "${{ github.event_name }}" = "pull_request" ]; then - PYPI_URL="https://test.pypi.org/pypi/dsf-mobility/${VERSION}/json" - PYPI_NAME="TestPyPI" + # TestPyPI only: use a numeric dev suffix so the version is PEP 440 compliant. + # Concatenating run id and attempt keeps it unique across PRs and re-runs. + PUBLISH_VERSION="${BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}" else - PYPI_URL="https://pypi.org/pypi/dsf-mobility/${VERSION}/json" - PYPI_NAME="PyPI" + # PyPI: publish the exact release version with no run-id suffix. + PUBLISH_VERSION="${BASE_VERSION}" + fi + + echo "publish_version=${PUBLISH_VERSION}" >> "$GITHUB_OUTPUT" + echo "Planned publish version: ${PUBLISH_VERSION}" + + - name: Check publish policy for PyPI + id: check + run: | + VERSION="${{ steps.plan.outputs.publish_version }}" + + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "PR build uses unique TestPyPI version ${VERSION}; skipping existence check." + echo "should_build=true" >> "$GITHUB_OUTPUT" + exit 0 fi + + PYPI_URL="https://pypi.org/pypi/dsf-mobility/${VERSION}/json" + PYPI_NAME="PyPI" echo "Checking if dsf-mobility version ${VERSION} exists on ${PYPI_NAME}..." @@ -53,7 +71,8 @@ jobs: if [ "$HTTP_STATUS" = "200" ]; then echo "Version ${VERSION} already exists on ${PYPI_NAME}" - echo "should_build=false" >> $GITHUB_OUTPUT + echo "ERROR: Refusing to overwrite an existing version on PyPI." + exit 1 else echo "Version ${VERSION} does not exist on ${PYPI_NAME} (HTTP ${HTTP_STATUS})" echo "should_build=true" >> $GITHUB_OUTPUT @@ -90,6 +109,7 @@ jobs: - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" + DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} run: | rm -rf dist wheelhouse python -m build --wheel @@ -145,6 +165,7 @@ jobs: - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" + DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} run: python -m build --wheel - name: Repair wheel (bundle libraries) @@ -195,6 +216,7 @@ jobs: - name: Build wheel env: CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF" + DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} run: python -m build --wheel - name: Upload wheels as artifacts @@ -229,6 +251,8 @@ jobs: python -m pip install build - name: Build source distribution + env: + DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }} run: python -m build --sdist - name: Upload sdist as artifact diff --git a/setup.py b/setup.py index 99db3f35..99221703 100644 --- a/setup.py +++ b/setup.py @@ -485,8 +485,8 @@ def run_stubgen(self): with open("README.md", "r", encoding="utf-8") as f: LONG_DESCRIPTION = f.read() -# Get version from header file -PROJECT_VERSION = get_version_from_header() +# Get version from header file, unless explicitly overridden for CI pre-releases. +PROJECT_VERSION = os.environ.get("DSF_PACKAGE_VERSION", get_version_from_header()) setup( name="dsf-mobility", diff --git a/src/dsf/base/Dynamics.hpp b/src/dsf/base/Dynamics.hpp index 036234f7..162e7058 100644 --- a/src/dsf/base/Dynamics.hpp +++ b/src/dsf/base/Dynamics.hpp @@ -103,13 +103,17 @@ namespace dsf { inline auto concurrency() const { return static_cast(m_taskArena.max_concurrency()); } - - inline void connectDataBase(std::string const& dbPath) { + /// @brief Connect to a SQLite database, creating it if it doesn't exist, and executing optional initialization queries + /// @param dbPath The path to the SQLite database file + /// @param queries Optional SQL queries to execute upon connecting to the database (default is a set of pragmas for performance optimization : "PRAGMA busy_timeout = 5000;PRAGMA journal_mode = WAL;PRAGMA synchronous=NORMAL;PRAGMA temp_store=MEMORY;PRAGMA cache_size=-20000;") + inline void connectDataBase( + std::string const& dbPath, + std::string const& queries = + "PRAGMA busy_timeout = 5000;PRAGMA journal_mode = WAL;PRAGMA " + "synchronous=NORMAL;PRAGMA temp_store=MEMORY;PRAGMA cache_size=-20000;") { m_database = std::make_unique( dbPath, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); - // Enable WAL mode for better concurrency and set busy timeout - m_database->exec("PRAGMA journal_mode = WAL;"); - m_database->exec("PRAGMA busy_timeout = 5000;"); // 5 seconds + m_database->exec(queries); } /// @brief Get the graph diff --git a/src/dsf/bindings.cpp b/src/dsf/bindings.cpp index c55a4ed2..b2397b1d 100644 --- a/src/dsf/bindings.cpp +++ b/src/dsf/bindings.cpp @@ -514,6 +514,9 @@ PYBIND11_MODULE(dsf_cpp, m) { .def("connectDataBase", &dsf::mobility::FirstOrderDynamics::connectDataBase, pybind11::arg("dbPath"), + pybind11::arg("queries") = + "PRAGMA busy_timeout = 5000;PRAGMA journal_mode = WAL;PRAGMA " + "synchronous=NORMAL;PRAGMA temp_store=MEMORY;PRAGMA cache_size=-20000;", dsf::g_docstrings.at("dsf::Dynamics::connectDataBase").c_str()) .def("setForcePriorities", &dsf::mobility::FirstOrderDynamics::setForcePriorities, diff --git a/src/dsf/dsf.hpp b/src/dsf/dsf.hpp index fdee9aba..f339db36 100644 --- a/src/dsf/dsf.hpp +++ b/src/dsf/dsf.hpp @@ -9,7 +9,7 @@ static constexpr uint8_t DSF_VERSION_MAJOR = 5; static constexpr uint8_t DSF_VERSION_MINOR = 3; -static constexpr uint8_t DSF_VERSION_PATCH = 0; +static constexpr uint8_t DSF_VERSION_PATCH = 1; static auto const DSF_VERSION = std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH); diff --git a/src/dsf/mobility/FirstOrderDynamics.cpp b/src/dsf/mobility/FirstOrderDynamics.cpp index becf68f2..5f1e0c2c 100644 --- a/src/dsf/mobility/FirstOrderDynamics.cpp +++ b/src/dsf/mobility/FirstOrderDynamics.cpp @@ -1531,9 +1531,21 @@ namespace dsf::mobility { this->m_evolveAgents(); if (bComputeStats) { + auto const datetime = this->strDateTime(); + auto const step = static_cast(this->time_step()); + auto const simulationId = static_cast(this->id()); + + bool const hasWritePayload = (m_bSaveStreetData && !streetDataRecords.empty()) || + (m_bSaveTravelData && !m_travelDTs.empty()) || + m_bSaveAverageStats; + + std::optional transaction; + if (hasWritePayload) { + transaction.emplace(*this->database()); + } + // Batch insert street data collected during parallel section - if (m_bSaveStreetData) { - SQLite::Transaction transaction(*this->database()); + if (m_bSaveStreetData && !streetDataRecords.empty()) { SQLite::Statement insertStmt( *this->database(), "INSERT INTO road_data (datetime, time_step, simulation_id, street_id, " @@ -1542,9 +1554,9 @@ namespace dsf::mobility { "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); for (auto const& record : streetDataRecords) { - insertStmt.bind(1, this->strDateTime()); - insertStmt.bind(2, static_cast(this->time_step())); - insertStmt.bind(3, static_cast(this->id())); + insertStmt.bind(1, datetime); + insertStmt.bind(2, step); + insertStmt.bind(3, simulationId); insertStmt.bind(4, static_cast(record.streetId)); if (record.coilName.has_value()) { insertStmt.bind(5, record.coilName.value()); @@ -1569,46 +1581,34 @@ namespace dsf::mobility { insertStmt.exec(); insertStmt.reset(); } - transaction.commit(); } - if (m_bSaveTravelData) { // Begin transaction for better performance - SQLite::Transaction transaction(*this->database()); + if (m_bSaveTravelData && !m_travelDTs.empty()) { SQLite::Statement insertStmt(*this->database(), "INSERT INTO travel_data (datetime, time_step, " "simulation_id, distance_m, travel_time_s) " "VALUES (?, ?, ?, ?, ?)"); for (auto const& [distance, time] : m_travelDTs) { - insertStmt.bind(1, this->strDateTime()); - insertStmt.bind(2, static_cast(this->time_step())); - insertStmt.bind(3, static_cast(this->id())); + insertStmt.bind(1, datetime); + insertStmt.bind(2, step); + insertStmt.bind(3, simulationId); insertStmt.bind(4, distance); insertStmt.bind(5, time); insertStmt.exec(); insertStmt.reset(); } - transaction.commit(); m_travelDTs.clear(); } if (m_bSaveAverageStats) { // Average Stats Table - mean_speed.store(mean_speed.load() / nValidEdges.load()); - mean_density.store(mean_density.load() / numEdges); - mean_traveltime.store(mean_traveltime.load() / nValidEdges.load()); - mean_queue_length.store(mean_queue_length.load() / numEdges); - { - double std_speed_val = std_speed.load(); - double mean_speed_val = mean_speed.load(); - std_speed.store(std::sqrt(std_speed_val / nValidEdges.load() - - mean_speed_val * mean_speed_val)); - } - { - double std_density_val = std_density.load(); - double mean_density_val = mean_density.load(); - std_density.store(std::sqrt(std_density_val / numEdges - - mean_density_val * mean_density_val)); - } + auto const validEdges = nValidEdges.load(); + auto const edgeCount = static_cast(numEdges); + auto const meanDensity = mean_density.load() / edgeCount; + auto const meanQueueLength = mean_queue_length.load() / edgeCount; + auto const densityVariance = + std::max(0.0, std_density.load() / edgeCount - meanDensity * meanDensity); + SQLite::Statement insertStmt( *this->database(), "INSERT INTO avg_stats (" @@ -1616,24 +1616,36 @@ namespace dsf::mobility { "mean_speed_kph, std_speed_kph, mean_density_vpk, std_density_vpk, " "mean_travel_time_s, mean_queue_length) " "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - insertStmt.bind(1, static_cast(this->id())); - insertStmt.bind(2, this->strDateTime()); - insertStmt.bind(3, static_cast(this->time_step())); + insertStmt.bind(1, simulationId); + insertStmt.bind(2, datetime); + insertStmt.bind(3, step); insertStmt.bind(4, static_cast(m_agents.size())); insertStmt.bind(5, static_cast(this->nAgents())); - if (nValidEdges.load() > 0) { - insertStmt.bind(6, mean_speed); - insertStmt.bind(7, std_speed); + + if (validEdges > 0) { + auto const validEdgeCount = static_cast(validEdges); + auto const meanSpeed = mean_speed.load() / validEdgeCount; + auto const meanTravelTime = mean_traveltime.load() / validEdgeCount; + auto const speedVariance = + std::max(0.0, std_speed.load() / validEdgeCount - meanSpeed * meanSpeed); + insertStmt.bind(6, meanSpeed); + insertStmt.bind(7, std::sqrt(speedVariance)); + insertStmt.bind(10, meanTravelTime); } else { insertStmt.bind(6); insertStmt.bind(7); + insertStmt.bind(10); } - insertStmt.bind(8, mean_density); - insertStmt.bind(9, std_density); - insertStmt.bind(10, mean_traveltime); - insertStmt.bind(11, mean_queue_length); + insertStmt.bind(8, meanDensity); + insertStmt.bind(9, std::sqrt(densityVariance)); + insertStmt.bind(11, meanQueueLength); insertStmt.exec(); } + + if (transaction.has_value()) { + transaction->commit(); + } + // Special case: if m_savingInterval == 0, it was a triggered saveData() call, so we need to reset all flags if (m_savingInterval.value() == 0) { m_savingInterval.reset(); diff --git a/webapp/script.js b/webapp/script.js index d3a8a7d7..88f128fa 100644 --- a/webapp/script.js +++ b/webapp/script.js @@ -1677,15 +1677,16 @@ document.addEventListener('DOMContentLoaded', () => { loadDbBtn.disabled = true; try { + // Snapshot and read the selected file first so later async work cannot invalidate access. + const fileSnapshot = file.slice(0, file.size); + const arrayBuffer = await fileSnapshot.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + // Initialize sql.js const SQL = await initSqlJs({ locateFile: file => `https://cdnjs.cloudflare.com/ajax/libs/sql.js/1.10.3/${file}` }); - // Read the file - const arrayBuffer = await file.arrayBuffer(); - const uint8Array = new Uint8Array(arrayBuffer); - // Open the database db = new SQL.Database(uint8Array); @@ -1721,7 +1722,11 @@ document.addEventListener('DOMContentLoaded', () => { } catch (error) { console.error('Database loading error:', error); dbStatus.className = 'db-status error'; - dbStatus.textContent = `Error: ${error.message}`; + if (error instanceof DOMException && error.name === 'NotReadableError') { + dbStatus.textContent = 'Error: Could not read the selected file. Re-select it, or move it to a local folder you own and try again.'; + } else { + dbStatus.textContent = `Error: ${error.message}`; + } loadDbBtn.disabled = false; } });