diff --git a/.gitignore b/.gitignore index 59b9508..4d0e8c6 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ po/*~ # RStudio Connect folder rsconnect/ +# Domain documentation (internal docs, not for repo) +../Domain_docs/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d245d40..894a144 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,18 +27,20 @@ ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 RUN R CMD javareconf # ---- Copy and install R packages ---- -COPY packages.R /tmp/packages.R +COPY no-code-devops/packages.R /tmp/packages.R RUN Rscript /tmp/packages.R -# ---- Install Rautoml from GitHub ---- -RUN R -e "remotes::install_github('aphrc-nocode/Rautoml')" +# ---- Install Rautoml from local source ---- +COPY Rautoml /tmp/Rautoml +RUN R -e "install.packages(c('caret', 'GGally', 'gtsummary', 'DatabaseConnector', 'naniar', 'haven', 'openxlsx', 'readr', 'readxl', 'recipes', 'rlang', 'rsample', 'shapviz', 'ggplot2'), repos='https://cloud.r-project.org', dependencies=TRUE)" && \ + R -e "if (!require('gemini.R', quietly=TRUE)) remotes::install_github('abresler/gemini.R', upgrade='never')" && \ + R -e "remotes::install_local('/tmp/Rautoml', dependencies=TRUE, upgrade='never', force=TRUE)" && \ + R -e "library(Rautoml)" || (echo "Rautoml installation failed" && exit 1) -# ---- Clone your Shiny app ---- +# ---- Copy your Shiny app ---- RUN rm -rf /usr/no-code-app/* -RUN git clone https://github.com/aphrc-nocode/no-code-app.git /usr/no-code-app - -COPY .env /usr/no-code-app/ +COPY no-code-app /usr/no-code-app RUN chmod -R 777 /usr/local/lib/R/site-library @@ -54,7 +56,7 @@ RUN mkdir -p /usr/no-code-app/datasets \ && chown -R shiny:shiny /usr/no-code-app # ---- Copy local users.sqlite template ---- -COPY users.sqlite /usr/no-code-app/users_db/users_template.sqlite +COPY no-code-devops/users.sqlite /usr/no-code-app/users_db/users_template.sqlite # ---- Define volumes ---- VOLUME ["/usr/no-code-app/datasets", \ @@ -73,7 +75,7 @@ WORKDIR /usr/no-code-app EXPOSE 3838 # ---- Copy entrypoint script ---- -COPY entrypoint.sh /usr/no-code-app/entrypoint.sh +COPY no-code-devops/entrypoint.sh /usr/no-code-app/entrypoint.sh RUN chmod +x /usr/no-code-app/entrypoint.sh # ---- Run as non-root user ---- diff --git a/docker-compose-full.yml b/docker-compose-full.yml new file mode 100644 index 0000000..1a25aa0 --- /dev/null +++ b/docker-compose-full.yml @@ -0,0 +1,172 @@ +services: + no-code-app: + build: + context: .. + dockerfile: no-code-devops/Dockerfile + container_name: no-code-app + restart: always + ports: + - "3838:3838" + volumes: + - datasets:/usr/no-code-app/datasets + - log_files:/usr/no-code-app/.log_files + - models:/usr/no-code-app/models + - recipes:/usr/no-code-app/recipes + - outputs:/usr/no-code-app/outputs + - output:/usr/no-code-app/output + - logs:/usr/no-code-app/logs + - ../no-code-app/.env:/usr/no-code-app/.env:ro + networks: + - nocode-network + environment: + - FASTAPI_BASE=http://no-code-pycaret:8000 + - REDIS_URL=redis://no-code-redis:6379/0 + - R_CARET_MAX_WORKERS=15 + deploy: + resources: + limits: + memory: 30G + cpus: '12' + reservations: + memory: 15G + cpus: '6' + + no-code-pycaret: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile + container_name: no-code-pycaret + restart: always + ports: + - "8000:8000" + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + deploy: + resources: + limits: + memory: 4G + cpus: '4' + reservations: + memory: 2G + cpus: '2' + + no-code-redis: + image: redis:7-alpine + container_name: no-code-redis + restart: always + ports: + - "6379:6379" + networks: + - nocode-network + volumes: + - redis_data:/data + deploy: + resources: + limits: + memory: 1G + cpus: '1' + reservations: + memory: 512M + cpus: '0.5' + + celery-worker-1: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-1 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + deploy: + resources: + limits: + memory: 4G + cpus: '2' + reservations: + memory: 2G + cpus: '1' + + celery-worker-2: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-2 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + + celery-worker-3: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-3 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + + celery-worker-4: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-4 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + + celery-worker-5: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile.worker + container_name: celery-worker-5 + restart: always + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + command: celery -A celery_app worker --loglevel=info --concurrency=10 + +volumes: + datasets: + log_files: + models: + recipes: + outputs: + output: + logs: + redis_data: + +networks: + nocode-network: + driver: bridge diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 0000000..d4a8f46 --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,106 @@ +# ============================================================================ +# LOCAL DEVELOPMENT ONLY - DO NOT USE ON SERVER +# ============================================================================ +# This configuration is for local testing with reduced resources: +# - 2 workers (instead of 5) +# - 2 concurrency per worker (instead of 10) +# - Total: 4 concurrent tasks max (suitable for laptops) +# +# For production server, use: docker-compose-full.yml +# ============================================================================ + +services: + no-code-app: + build: + context: .. + dockerfile: no-code-devops/Dockerfile + container_name: no-code-app + restart: always + ports: + - "3838:3838" + volumes: + - datasets:/usr/no-code-app/datasets + - log_files:/usr/no-code-app/.log_files + - models:/usr/no-code-app/models + - recipes:/usr/no-code-app/recipes + - outputs:/usr/no-code-app/outputs + - output:/usr/no-code-app/output + - logs:/usr/no-code-app/logs + - ../no-code-app/.env:/usr/no-code-app/.env:ro + networks: + - nocode-network + environment: + - FASTAPI_BASE=http://no-code-pycaret:8000 + - REDIS_URL=redis://no-code-redis:6379/0 + - R_CARET_MAX_WORKERS=4 + + no-code-pycaret: + build: + context: .. + dockerfile: no-code-devops/pycaret/Dockerfile + container_name: no-code-pycaret + restart: always + ports: + - "8000:8000" + volumes: + - models:/usr/no-code-pycaret/models + - logs:/usr/no-code-pycaret/logs + networks: + - nocode-network + environment: + - REDIS_URL=redis://no-code-redis:6379/0 + + no-code-redis: + image: redis:7-alpine + container_name: no-code-redis + restart: always + ports: + - "6379:6379" + networks: + - nocode-network + volumes: + - redis_data:/data + + # celery-worker-1: + # build: + # context: .. + # dockerfile: no-code-devops/pycaret/Dockerfile.worker + # container_name: celery-worker-1 + # restart: always + # volumes: + # - models:/usr/no-code-pycaret/models + # - logs:/usr/no-code-pycaret/logs + # networks: + # - nocode-network + # environment: + # - REDIS_URL=redis://no-code-redis:6379/0 + # command: celery -A celery_app worker --loglevel=info --concurrency=2 + + # celery-worker-2: + # build: + # context: .. + # dockerfile: no-code-devops/pycaret/Dockerfile.worker + # container_name: celery-worker-2 + # restart: always + # volumes: + # - models:/usr/no-code-pycaret/models + # - logs:/usr/no-code-pycaret/logs + # networks: + # - nocode-network + # environment: + # - REDIS_URL=redis://no-code-redis:6379/0 + # command: celery -A celery_app worker --loglevel=info --concurrency=2 + +volumes: + datasets: + log_files: + models: + recipes: + outputs: + output: + logs: + redis_data: + +networks: + nocode-network: + driver: bridge diff --git a/pycaret/Dockerfile b/pycaret/Dockerfile index 117d1a4..0f71427 100644 --- a/pycaret/Dockerfile +++ b/pycaret/Dockerfile @@ -13,8 +13,8 @@ RUN mkdir /usr/no-code-pycaret WORKDIR /usr/no-code-pycaret -RUN wget https://raw.githubusercontent.com/aphrc-nocode/no-code-app/refs/heads/main/py/main.py -RUN wget https://raw.githubusercontent.com/aphrc-nocode/no-code-app/refs/heads/main/py/requirements.txt +COPY no-code-app/py/main.py . +COPY no-code-app/py/requirements.txt . # ---- Create writable directories ---- RUN mkdir -p /usr/no-code-pycaret/logs \ diff --git a/pycaret/Dockerfile.worker b/pycaret/Dockerfile.worker new file mode 100644 index 0000000..d2a14ac --- /dev/null +++ b/pycaret/Dockerfile.worker @@ -0,0 +1,30 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential gfortran libgomp1 libopenblas-dev \ + libfreetype6-dev libpng-dev fonts-dejavu-core \ + wget \ + && rm -rf /var/lib/apt/lists/* + +RUN rm -rf /usr/no-code-pycaret/* +RUN mkdir /usr/no-code-pycaret + +WORKDIR /usr/no-code-pycaret + +COPY no-code-app/py/main.py . +COPY no-code-app/py/requirements.txt . +COPY no-code-app/py/celery_app.py . +COPY no-code-app/py/tasks.py . + +RUN mkdir -p /usr/no-code-pycaret/logs \ + /usr/no-code-pycaret/models + +VOLUME ["/usr/no-code-pycaret/logs", \ + "/usr/no-code-pycaret/models"] + +RUN pip install --upgrade pip setuptools wheel && \ + pip install -r requirements.txt + +CMD ["celery", "-A", "celery_app", "worker", "--loglevel=info", "--concurrency=2"] diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..628a766 --- /dev/null +++ b/start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +echo "Starting No-Code Platform..." +echo "This will build and start both containers:" +echo " - Shiny app on http://localhost:3838" +echo " - Python API on http://localhost:8000" +echo "" + +docker compose -f docker-compose-full.yml up --build