From 98198aea8a069031f3b6bb4af35b903398335df8 Mon Sep 17 00:00:00 2001 From: damiancodes Date: Fri, 13 Feb 2026 12:30:37 +0300 Subject: [PATCH 1/5] Initial feature branch setup --- Dockerfile | 20 +++++++++-------- docker-compose-full.yml | 49 +++++++++++++++++++++++++++++++++++++++++ pycaret/Dockerfile | 4 ++-- start.sh | 11 +++++++++ 4 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 docker-compose-full.yml create mode 100755 start.sh diff --git a/Dockerfile b/Dockerfile index d245d40..58a145b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,18 +27,20 @@ ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 RUN R CMD javareconf # ---- Copy and install R packages ---- -COPY packages.R /tmp/packages.R +COPY no-code-devops/packages.R /tmp/packages.R RUN Rscript /tmp/packages.R -# ---- Install Rautoml from GitHub ---- -RUN R -e "remotes::install_github('aphrc-nocode/Rautoml')" +# ---- Install Rautoml from local source ---- +COPY Rautoml /tmp/Rautoml +RUN R -e "install.packages(c('caret', 'GGally', 'gtsummary', 'DatabaseConnector', 'naniar', 'haven', 'openxlsx', 'readr', 'readxl', 'recipes', 'rlang', 'rsample', 'shapviz'), repos='https://cloud.r-project.org', dependencies=TRUE)" && \ + R -e "if (!require('gemini.R', quietly=TRUE)) remotes::install_github('abresler/gemini.R', upgrade='never')" && \ + R -e "remotes::install_local('/tmp/Rautoml', dependencies=FALSE, upgrade='never', force=TRUE)" && \ + R -e "library(Rautoml)" || (echo "Rautoml installation failed" && exit 1) -# ---- Clone your Shiny app ---- +# ---- Copy your Shiny app ---- RUN rm -rf /usr/no-code-app/* -RUN git clone https://github.com/aphrc-nocode/no-code-app.git /usr/no-code-app - -COPY .env /usr/no-code-app/ +COPY no-code-app /usr/no-code-app RUN chmod -R 777 /usr/local/lib/R/site-library @@ -54,7 +56,7 @@ RUN mkdir -p /usr/no-code-app/datasets \ && chown -R shiny:shiny /usr/no-code-app # ---- Copy local users.sqlite template ---- -COPY users.sqlite /usr/no-code-app/users_db/users_template.sqlite +COPY no-code-devops/users.sqlite /usr/no-code-app/users_db/users_template.sqlite # ---- Define volumes ---- VOLUME ["/usr/no-code-app/datasets", \ @@ -73,7 +75,7 @@ WORKDIR /usr/no-code-app EXPOSE 3838 # ---- Copy entrypoint script ---- -COPY entrypoint.sh /usr/no-code-app/entrypoint.sh +COPY no-code-devops/entrypoint.sh /usr/no-code-app/entrypoint.sh RUN chmod +x /usr/no-code-app/entrypoint.sh # ---- Run as non-root user ---- diff --git a/docker-compose-full.yml b/docker-compose-full.yml new file mode 100644 index 0000000..3f30847 --- /dev/null +++ b/docker-compose-full.yml @@ -0,0 +1,49 @@ +services: + no-code-app: + build: + context: .. + dockerfile: no-code-devops/Dockerfile + container_name: no-code-app + restart: always + ports: + - "3838:3838" + volumes: + - datasets:/usr/no-code-app/datasets + - log_files:/usr/no-code-app/.log_files + - models:/usr/no-code-app/models + - recipes:/usr/no-code-app/recipes + - outputs:/usr/no-code-app/outputs + - output:/usr/no-code-app/output + - logs:/usr/no-code-app/logs + - ../no-code-app/.env:/usr/no-code-app/.env:ro + networks: + - nocode-network + environment: + - FASTAPI_BASE=http://no-code-pycaret:8000 + + no-code-pycaret: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile + container_name: no-code-pycaret + restart: always + ports: + - "8000:8000" + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + +volumes: + datasets: + log_files: + models: + recipes: + outputs: + output: + logs: + +networks: + nocode-network: + driver: bridge diff --git a/pycaret/Dockerfile b/pycaret/Dockerfile index 117d1a4..0f71427 100644 --- a/pycaret/Dockerfile +++ b/pycaret/Dockerfile @@ -13,8 +13,8 @@ RUN mkdir /usr/no-code-pycaret WORKDIR /usr/no-code-pycaret -RUN wget https://raw.githubusercontent.com/aphrc-nocode/no-code-app/refs/heads/main/py/main.py -RUN wget https://raw.githubusercontent.com/aphrc-nocode/no-code-app/refs/heads/main/py/requirements.txt +COPY no-code-app/py/main.py . +COPY no-code-app/py/requirements.txt . # ---- Create writable directories ---- RUN mkdir -p /usr/no-code-pycaret/logs \ diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..628a766 --- /dev/null +++ b/start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +echo "Starting No-Code Platform..." +echo "This will build and start both containers:" +echo " - Shiny app on http://localhost:3838" +echo " - Python API on http://localhost:8000" +echo "" + +docker compose -f docker-compose-full.yml up --build From f8fc8fb1d09f24b110b9f013f38f5aabb8253f31 Mon Sep 17 00:00:00 2001 From: damiancodes Date: Fri, 13 Feb 2026 16:08:02 +0300 Subject: [PATCH 2/5] Add Celery workers and Redis for concurrent training --- .gitignore | 2 + docker-compose-full.yml | 89 ++++++++++++++++++++++++++++++++ docker-compose-local.yml | 104 ++++++++++++++++++++++++++++++++++++++ pycaret/Dockerfile | 2 + pycaret/Dockerfile.worker | 30 +++++++++++ 5 files changed, 227 insertions(+) create mode 100644 docker-compose-local.yml create mode 100644 pycaret/Dockerfile.worker diff --git a/.gitignore b/.gitignore index 59b9508..4d0e8c6 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ po/*~ # RStudio Connect folder rsconnect/ +# Domain documentation (internal docs, not for repo) +../Domain_docs/ \ No newline at end of file diff --git a/docker-compose-full.yml b/docker-compose-full.yml index 3f30847..c432668 100644 --- a/docker-compose-full.yml +++ b/docker-compose-full.yml @@ -34,6 +34,94 @@ services: - logs:/usr/no-code-pycaret/logs networks: - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + + no-code-redis: + image: redis:7-alpine + container_name: no-code-redis + restart: always + ports: + - "6379:6379" + networks: + - nocode-network + volumes: + - redis_data:/data + + celery-worker-1: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-1 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + + celery-worker-2: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-2 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + + celery-worker-3: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-3 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + + celery-worker-4: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-4 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + + celery-worker-5: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-5 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 volumes: datasets: @@ -43,6 +131,7 @@ volumes: outputs: output: logs: + redis_data: networks: nocode-network: diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 0000000..462510a --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,104 @@ +# ============================================================================ +# LOCAL DEVELOPMENT ONLY - DO NOT USE ON SERVER +# ============================================================================ +# This configuration is for local testing with reduced resources: +# - 2 workers (instead of 5) +# - 2 concurrency per worker (instead of 10) +# - Total: 4 concurrent tasks max (suitable for laptops) +# +# For production server, use: docker-compose-full.yml +# ============================================================================ + +services: + no-code-app: + build: + context: .. + dockerfile: no-code-devops/Dockerfile + container_name: no-code-app + restart: always + ports: + - "3838:3838" + volumes: + - datasets:/usr/no-code-app/datasets + - log_files:/usr/no-code-app/.log_files + - models:/usr/no-code-app/models + - recipes:/usr/no-code-app/recipes + - outputs:/usr/no-code-app/outputs + - output:/usr/no-code-app/output + - logs:/usr/no-code-app/logs + - ../no-code-app/.env:/usr/no-code-app/.env:ro + networks: + - nocode-network + environment: + - FASTAPI_BASE=http://no-code-pycaret:8000 + + no-code-pycaret: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile + container_name: no-code-pycaret + restart: always + ports: + - "8000:8000" + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + + no-code-redis: + image: redis:7-alpine + container_name: no-code-redis + restart: always + ports: + - "6379:6379" + networks: + - nocode-network + volumes: + - redis_data:/data + + celery-worker-1: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-1 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=2 + + celery-worker-2: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-2 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=2 + +volumes: + datasets: + log_files: + models: + recipes: + outputs: + output: + logs: + redis_data: + +networks: + nocode-network: + driver: bridge diff --git a/pycaret/Dockerfile b/pycaret/Dockerfile index 0f71427..14e72a1 100644 --- a/pycaret/Dockerfile +++ b/pycaret/Dockerfile @@ -15,6 +15,8 @@ WORKDIR /usr/no-code-pycaret COPY no-code-app/py/main.py . COPY no-code-app/py/requirements.txt . +COPY no-code-app/py/celery_app.py . +COPY no-code-app/py/tasks.py . # ---- Create writable directories ---- RUN mkdir -p /usr/no-code-pycaret/logs \ diff --git a/pycaret/Dockerfile.worker b/pycaret/Dockerfile.worker new file mode 100644 index 0000000..d2a14ac --- /dev/null +++ b/pycaret/Dockerfile.worker @@ -0,0 +1,30 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential gfortran libgomp1 libopenblas-dev \ + libfreetype6-dev libpng-dev fonts-dejavu-core \ + wget \ + && rm -rf /var/lib/apt/lists/* + +RUN rm -rf /usr/no-code-pycaret/* +RUN mkdir /usr/no-code-pycaret + +WORKDIR /usr/no-code-pycaret + +COPY no-code-app/py/main.py . +COPY no-code-app/py/requirements.txt . +COPY no-code-app/py/celery_app.py . +COPY no-code-app/py/tasks.py . + +RUN mkdir -p /usr/no-code-pycaret/logs \ + /usr/no-code-pycaret/models + +VOLUME ["/usr/no-code-pycaret/logs", \ + "/usr/no-code-pycaret/models"] + +RUN pip install --upgrade pip setuptools wheel && \ + pip install -r requirements.txt + +CMD ["celery", "-A", "celery_app", "worker", "--loglevel=info", "--concurrency=2"] From d0556a6307ef92d728ada535cd330d6a85e973b8 Mon Sep 17 00:00:00 2001 From: damiancodes Date: Mon, 16 Feb 2026 18:10:56 +0300 Subject: [PATCH 3/5] Add R Caret worker configuration --- Dockerfile | 4 ++-- docker-compose-full.yml | 34 ++++++++++++++++++++++++++++++++++ docker-compose-local.yml | 2 ++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 58a145b..894a144 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,9 +32,9 @@ RUN Rscript /tmp/packages.R # ---- Install Rautoml from local source ---- COPY Rautoml /tmp/Rautoml -RUN R -e "install.packages(c('caret', 'GGally', 'gtsummary', 'DatabaseConnector', 'naniar', 'haven', 'openxlsx', 'readr', 'readxl', 'recipes', 'rlang', 'rsample', 'shapviz'), repos='https://cloud.r-project.org', dependencies=TRUE)" && \ +RUN R -e "install.packages(c('caret', 'GGally', 'gtsummary', 'DatabaseConnector', 'naniar', 'haven', 'openxlsx', 'readr', 'readxl', 'recipes', 'rlang', 'rsample', 'shapviz', 'ggplot2'), repos='https://cloud.r-project.org', dependencies=TRUE)" && \ R -e "if (!require('gemini.R', quietly=TRUE)) remotes::install_github('abresler/gemini.R', upgrade='never')" && \ - R -e "remotes::install_local('/tmp/Rautoml', dependencies=FALSE, upgrade='never', force=TRUE)" && \ + R -e "remotes::install_local('/tmp/Rautoml', dependencies=TRUE, upgrade='never', force=TRUE)" && \ R -e "library(Rautoml)" || (echo "Rautoml installation failed" && exit 1) # ---- Copy your Shiny app ---- diff --git a/docker-compose-full.yml b/docker-compose-full.yml index c432668..8e19260 100644 --- a/docker-compose-full.yml +++ b/docker-compose-full.yml @@ -20,6 +20,16 @@ services: - nocode-network environment: - FASTAPI_BASE=http://no-code-pycaret:8000 + - REDIS_URL=redis://no-code-redis:6379/0 + - R_CARET_MAX_WORKERS=8 + deploy: + resources: + limits: + memory: 20G + cpus: '8' + reservations: + memory: 10G + cpus: '4' no-code-pycaret: build: @@ -36,6 +46,14 @@ services: - nocode-network environment: - REDIS_URL=redis://no-code-redis:6379/0 + deploy: + resources: + limits: + memory: 4G + cpus: '4' + reservations: + memory: 2G + cpus: '2' no-code-redis: image: redis:7-alpine @@ -47,6 +65,14 @@ services: - nocode-network volumes: - redis_data:/data + deploy: + resources: + limits: + memory: 1G + cpus: '1' + reservations: + memory: 512M + cpus: '0.5' celery-worker-1: build: @@ -62,6 +88,14 @@ services: environment: - REDIS_URL=redis://no-code-redis:6379/0 command: celery -A celery_app worker --loglevel=info --concurrency=10 + deploy: + resources: + limits: + memory: 4G + cpus: '2' + reservations: + memory: 2G + cpus: '1' celery-worker-2: build: diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 462510a..d406718 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -31,6 +31,8 @@ services: - nocode-network environment: - FASTAPI_BASE=http://no-code-pycaret:8000 + - REDIS_URL=redis://no-code-redis:6379/0 + - R_CARET_MAX_WORKERS=4 no-code-pycaret: build: From 6e18110b02c78947d610b72a97204f5f80e1949c Mon Sep 17 00:00:00 2001 From: damiancodes Date: Mon, 16 Feb 2026 19:33:59 +0300 Subject: [PATCH 4/5] Disable Celery workers for R Caret testing --- docker-compose-local.yml | 56 ++++++++++++++++++++-------------------- pycaret/Dockerfile | 2 -- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/docker-compose-local.yml b/docker-compose-local.yml index d406718..d4a8f46 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -61,35 +61,35 @@ services: volumes: - redis_data:/data - celery-worker-1: - build: - context: .. - dockerfile: no-code-devops/pycaret/Dockerfile.worker - container_name: celery-worker-1 - restart: always - volumes: - - models:/usr/no-code-pycaret/models - - logs:/usr/no-code-pycaret/logs - networks: - - nocode-network - environment: - - REDIS_URL=redis://no-code-redis:6379/0 - command: celery -A celery_app worker --loglevel=info --concurrency=2 + # celery-worker-1: + # build: + # context: .. + # dockerfile: no-code-devops/pycaret/Dockerfile.worker + # container_name: celery-worker-1 + # restart: always + # volumes: + # - models:/usr/no-code-pycaret/models + # - logs:/usr/no-code-pycaret/logs + # networks: + # - nocode-network + # environment: + # - REDIS_URL=redis://no-code-redis:6379/0 + # command: celery -A celery_app worker --loglevel=info --concurrency=2 - celery-worker-2: - build: - context: .. - dockerfile: no-code-devops/pycaret/Dockerfile.worker - container_name: celery-worker-2 - restart: always - volumes: - - models:/usr/no-code-pycaret/models - - logs:/usr/no-code-pycaret/logs - networks: - - nocode-network - environment: - - REDIS_URL=redis://no-code-redis:6379/0 - command: celery -A celery_app worker --loglevel=info --concurrency=2 + # celery-worker-2: + # build: + # context: .. + # dockerfile: no-code-devops/pycaret/Dockerfile.worker + # container_name: celery-worker-2 + # restart: always + # volumes: + # - models:/usr/no-code-pycaret/models + # - logs:/usr/no-code-pycaret/logs + # networks: + # - nocode-network + # environment: + # - REDIS_URL=redis://no-code-redis:6379/0 + # command: celery -A celery_app worker --loglevel=info --concurrency=2 volumes: datasets: diff --git a/pycaret/Dockerfile b/pycaret/Dockerfile index 14e72a1..0f71427 100644 --- a/pycaret/Dockerfile +++ b/pycaret/Dockerfile @@ -15,8 +15,6 @@ WORKDIR /usr/no-code-pycaret COPY no-code-app/py/main.py . COPY no-code-app/py/requirements.txt . -COPY no-code-app/py/celery_app.py . -COPY no-code-app/py/tasks.py . # ---- Create writable directories ---- RUN mkdir -p /usr/no-code-pycaret/logs \ From 6cbf9e59ef84c18ac8e6ce947045ce14edff2dad Mon Sep 17 00:00:00 2001 From: damiancodes Date: Mon, 23 Feb 2026 12:38:38 +0300 Subject: [PATCH 5/5] Update resources for 15 workers --- docker-compose-full.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose-full.yml b/docker-compose-full.yml index 8e19260..1a25aa0 100644 --- a/docker-compose-full.yml +++ b/docker-compose-full.yml @@ -21,15 +21,15 @@ services: environment: - FASTAPI_BASE=http://no-code-pycaret:8000 - REDIS_URL=redis://no-code-redis:6379/0 - - R_CARET_MAX_WORKERS=8 + - R_CARET_MAX_WORKERS=15 deploy: resources: limits: - memory: 20G - cpus: '8' + memory: 30G + cpus: '12' reservations: - memory: 10G - cpus: '4' + memory: 15G + cpus: '6' no-code-pycaret: build: