Add Podman deployment scripts

LICENSES/CLA-signed-list.md

@@ -21,3 +21,4 @@ C/ My company has custom contribution contract with Lutra Consulting Ltd. or I a
 * luxusko, 25th August 2023
 * jozef-budac, 30th January 2024
 * fernandinand, 13th March 2025
+* xkello, 27th January 2025

deployment/community/run_podman.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+set -e
+
+NETWORK="merginmaps"
+
+# Create network if it doesn't exist
+if ! podman network exists $NETWORK; then
+    echo "Creating network $NETWORK..."
+    podman network create $NETWORK
+fi
+
+# Function to run a container if it doesn't exist (optional cleanup can be added)
+# For now, we assume this script starts the stack. Users should `podman rm -f` manually to restart.
+
+echo "Starting Database..."
+podman run -d \
+    --name merginmaps-db \
+    --network $NETWORK \
+    --network-alias db \
+    --restart always \
+    -e POSTGRES_DB=mergin \
+    -e POSTGRES_USER=postgres \
+    -e POSTGRES_PASSWORD=postgres \
+    -v ./mergin_db:/var/lib/postgresql/data \
+    postgres:14
+
+echo "Starting Redis..."
+podman run -d \
+    --name merginmaps-redis \
+    --network $NETWORK \
+    --network-alias redis \
+    --restart always \
+    redis:6.2.17
+
+echo "Starting Server..."
+# Note: Ensure .prod.env exists or provided variables match production requirements
+podman run -d \
+    --name merginmaps-server \
+    --network $NETWORK \
+    --network-alias server \
+    --restart always \
+    --user 901:999 \
+    -v ./projects:/data \
+    -v ./diagnostic_logs:/diagnostic_logs \
+    --env-file .prod.env \
+    lutraconsulting/merginmaps-backend:2025.7.3 \
+    gunicorn --config config.py application:application
+
+echo "Starting Celery Beat..."
+podman run -d \
+    --name celery-beat \
+    --network $NETWORK \
+    --network-alias celery-beat \
+    --restart always \
+    --env-file .prod.env \
+    -e GEVENT_WORKER=0 \
+    -e NO_MONKEY_PATCH=1 \
+    lutraconsulting/merginmaps-backend:2025.7.3 \
+    celery -A application.celery beat --loglevel=info
+
+echo "Starting Celery Worker..."
+podman run -d \
+    --name celery-worker \
+    --network $NETWORK \
+    --network-alias celery-worker \
+    --restart always \
+    --user 901:999 \
+    --env-file .prod.env \
+    -e GEVENT_WORKER=0 \
+    -e NO_MONKEY_PATCH=1 \
+    -v ./projects:/data \
+    lutraconsulting/merginmaps-backend:2025.7.3 \
+    celery -A application.celery worker --loglevel=info
+
+echo "Starting Web..."
+podman run -d \
+    --name merginmaps-web \
+    --network $NETWORK \
+    --network-alias web \
+    --restart always \
+    --user 101:999 \
+    lutraconsulting/merginmaps-frontend:2025.7.3
+
+echo "Starting Proxy..."
+podman run -d \
+    --name merginmaps-proxy \
+    --network $NETWORK \
+    --network-alias proxy \
+    --restart always \
+    --user 101:999 \
+    -p 8080:8080 \
+    -v ./projects:/data \
+    -v ../common/nginx.conf:/etc/nginx/conf.d/default.conf \
+    nginxinc/nginx-unprivileged:1.27
+
+echo "Stack started successfully with Podman."

deployment/community/stop_podman.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+CONTAINERS="merginmaps-proxy merginmaps-web celery-worker celery-beat merginmaps-server merginmaps-redis merginmaps-db"
+
+echo "Stopping containers..."
+# Ignore errors if containers don't exist
+podman stop $CONTAINERS 2>/dev/null || true
+
+echo "Removing containers..."
+podman rm $CONTAINERS 2>/dev/null || true
+
+# Optional: Remove network
+# podman network rm merginmaps 2>/dev/null || true
+
+echo "Stack stopped and removed."

server/mergin/sync/commands.py

@@ -127,3 +127,50 @@ def remove(project_name):
         project.removed_by = None
         db.session.commit()
         click.secho("Project removed", fg="green")
+
+    @project.command()
+    @click.argument("project-name", callback=normalize_input(lowercase=False))
+    @click.option("--since", type=int, required=False)
+    @click.option("--to", type=int, required=False)
+    def create_checkpoint(project_name, since=None, to=None):
+        """Create project delta checkpoint, corresponding lower checkpoints and merged diffs for project"""
+        ws, name = split_project_path(project_name)
+        workspace = current_app.ws_handler.get_by_name(ws)
+        if not workspace:
+            click.secho("ERROR: Workspace does not exist", fg="red", err=True)
+            sys.exit(1)
+        project = (
+            Project.query.filter_by(workspace_id=workspace.id, name=name)
+            .filter(Project.storage_params.isnot(None))
+            .first()
+        )
+        if not project:
+            click.secho("ERROR: Project does not exist", fg="red", err=True)
+            sys.exit(1)
+
+        since = since if since is not None else 0
+        to = to if to is not None else project.latest_version
+        if since < 0 or to < 1:
+            click.secho(
+                "ERROR: Invalid version number, minimum version for 'since' is 0 and minimum version for 'to' is 1",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        if to > project.latest_version:
+            click.secho(
+                "ERROR: 'to' version exceeds latest project version", fg="red", err=True
+            )
+            sys.exit(1)
+
+        if since >= to:
+            click.secho(
+                "ERROR: 'since' version must be less than 'to' version",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        project.get_delta_changes(since, to)
+        click.secho("Project checkpoint(s) created", fg="green")

server/mergin/sync/config.py

@@ -75,6 +75,8 @@ class Configuration(object):
     UPLOAD_CHUNKS_EXPIRATION = config(
         "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int
     )
+    # whether client can pull using v2 apis
+    V2_PULL_ENABLED = config("V2_PULL_ENABLED", default=True, cast=bool)
     EXCLUDED_CLONE_FILENAMES = config(
         "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv()
     )

server/mergin/sync/errors.py

@@ -95,3 +95,10 @@ def to_dict(self) -> Dict:
 class BigChunkError(ResponseError):
     code = "BigChunkError"
     detail = f"Chunk size exceeds maximum allowed size {MAX_CHUNK_SIZE} MB"
+
+
+class DiffDownloadError(ResponseError):
+    code = "DiffDownloadError"
+    detail = (
+        "Required diff file could not be downloaded as it could not be reconstructed"
+    )

server/mergin/sync/files.py

@@ -4,11 +4,18 @@
 import datetime
 from enum import Enum
 import os
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Optional, List
 import uuid
 from flask import current_app
-from marshmallow import ValidationError, fields, EXCLUDE, post_dump, validates_schema
+from marshmallow import (
+    ValidationError,
+    fields,
+    EXCLUDE,
+    post_dump,
+    validates_schema,
+    post_load,
+)
 from pathvalidate import sanitize_filename
 
 from .utils import (
@@ -231,11 +238,125 @@ def validate(self, data, **kwargs):
 
 class ProjectFileSchema(FileSchema):
     mtime = DateTimeWithZ()
-    diff = fields.Nested(FileSchema())
+    diff = fields.Nested(
+        FileSchema(),
+    )
 
     @post_dump
     def patch_field(self, data, **kwargs):
         # drop 'diff' key entirely if empty or None as clients would expect
         if not data.get("diff"):
             data.pop("diff", None)
         return data
+
+
+@dataclass
+class DeltaDiffFile:
+    """Diff file path in diffs list"""
+
+    id: str
+
+
+class DeltaChangeDiffFileSchema(ma.Schema):
+    """Schema for diff file path in diffs list"""
+
+    id = fields.String(required=True)
+
+
+@dataclass
+class DeltaChangeBase(File):
+    """Base class for changes stored in json list or returned from delta endpoint"""
+
+    change: PushChangeType
+    version: int
+
+
+@dataclass
+class DeltaChangeMerged(DeltaChangeBase):
+    """Delta item with merged diffs to list of multiple diff files"""
+
+    diffs: List[DeltaDiffFile] = field(default_factory=list)
+
+    def to_data_delta(self):
+        """Convert DeltaMerged to DeltaData with single diff"""
+        result = DeltaChange(
+            path=self.path,
+            size=self.size,
+            checksum=self.checksum,
+            change=self.change,
+            version=self.version,
+        )
+        if self.diffs:
+            result.diff = self.diffs[0].id
+        return result
+
+
+@dataclass
+class DeltaChange(DeltaChangeBase):
+    """Change items stored in database as list of this item with single diff file"""
+
+    diff: Optional[str] = None
+
+    def to_merged(self) -> DeltaChangeMerged:
+        """Convert to DeltaMerged with multiple diffs"""
+        result = DeltaChangeMerged(
+            path=self.path,
+            size=self.size,
+            checksum=self.checksum,
+            change=self.change,
+            version=self.version,
+        )
+        if self.diff:
+            result.diffs = [DeltaDiffFile(id=self.diff)]
+        return result
+
+
+class DeltaChangeBaseSchema(ma.Schema):
+    """Base schema for delta json and response from delta endpoint"""
+
+    path = fields.String(required=True)
+    size = fields.Integer(required=True)
+    checksum = fields.String(required=True)
+    version = fields.Integer(required=True)
+    change = fields.Enum(PushChangeType, by_value=True, required=True)
+
+
+class DeltaChangeSchema(DeltaChangeBaseSchema):
+    """Schema for change data in changes column"""
+
+    diff = fields.String(required=False)
+
+    @post_load
+    def make_object(self, data, **kwargs):
+        return DeltaChange(**data)
+
+    @post_dump
+    def patch_field(self, data, **kwargs):
+        # drop 'diff' key entirely if empty or None as database would expect
+        if not data.get("diff"):
+            data.pop("diff", None)
+        return data
+
+
+class DeltaChangeItemSchema(DeltaChangeBaseSchema):
+    """Schema for delta changes response"""
+
+    version = fields.Function(lambda obj: f"v{obj.version}")
+    diffs = fields.List(fields.Nested(DeltaChangeDiffFileSchema()))
+
+    @post_dump
+    def patch_field(self, data, **kwargs):
+        # drop 'diffs' key entirely if empty or None as clients would expect
+        if not data.get("diffs"):
+            data.pop("diffs", None)
+        return data
+
+
+class DeltaChangeRespSchema(ma.Schema):
+    """Schema for list of delta changes wrapped in items field"""
+
+    items = fields.List(fields.Nested(DeltaChangeItemSchema()))
+    to_version = fields.String(required=True)
+
+    class Meta:
+        unknown = EXCLUDE