diff --git a/.gitignore b/.gitignore index a42ce01..19020ff 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ testing/ testing* *.pyc null/ +*output*/ +**/*output*/ diff --git a/assets/bindcraft/fast_4stage_multimer.json b/assets/bindcraft/fast_4stage_multimer.json new file mode 100644 index 0000000..66da74d --- /dev/null +++ b/assets/bindcraft/fast_4stage_multimer.json @@ -0,0 +1,67 @@ +{ + "omit_AAs": "C", + "force_reject_AA": false, + "use_multimer_design": true, + "design_algorithm": "4stage", + "sample_models": true, + "rm_template_seq_design": false, + "rm_template_seq_predict": false, + "rm_template_sc_design": false, + "rm_template_sc_predict": false, + "predict_initial_guess": false, + "predict_bigbang": false, + "soft_iterations": 40, + "temporary_iterations": 20, + "hard_iterations": 3, + "greedy_iterations": 8, + "greedy_percentage": 5, + "save_design_animations": false, + "save_design_trajectory_plots": false, + "weights_plddt": 0.1, + "weights_pae_intra": 0.4, + "weights_pae_inter": 0.1, + "weights_con_intra": 1.0, + "weights_con_inter": 1.0, + "intra_contact_distance": 14.0, + "inter_contact_distance": 20.0, + "intra_contact_number": 2, + "inter_contact_number": 2, + "weights_helicity": -0.3, + "random_helicity": false, + "use_i_ptm_loss": true, + "weights_iptm": 0.05, + "use_rg_loss": true, + "weights_rg": 0.3, + "use_termini_distance_loss": false, + "weights_termini_loss": 0.1, + "cyclize_peptide": false, + "enable_mpnn": true, + "mpnn_fix_interface": true, + "num_seqs": 10, + "max_mpnn_sequences": 1, + "sampling_temp": 0.1, + "backbone_noise": 0.00, + "model_path": "v_48_020", + "mpnn_weights": "soluble", + "save_mpnn_fasta": false, + "num_recycles_design": 1, + "num_recycles_validation": 2, + "optimise_beta": true, + "optimise_beta_extra_soft": 0, + "optimise_beta_extra_temp": 0, + "optimise_beta_recycles_design": 2, + "optimise_beta_recycles_valid": 2, + "remove_unrelaxed_trajectory": true, + "remove_unrelaxed_complex": true, + "remove_binder_monomer": true, + "zip_animations": false, + "zip_plots": false, + "save_trajectory_pickle": false, + "max_trajectories": false, + "enable_rejection_check": true, + "acceptance_rate": 0.01, + "start_monitoring": 600, + "af_params_dir": "", + "dssp_path": "", + "dalphaball_path": "" +} diff --git a/assets/samplesheet.csv b/assets/samplesheet.csv index 1553128..170184b 100644 --- a/assets/samplesheet.csv +++ b/assets/samplesheet.csv @@ -1,2 +1,2 @@ id,binder_name,starting_pdb,chains,target_hotspot_residues,min_length,max_length,number_of_final_designs,settings_filters,settings_advanced -s1,PDL1,https://raw.githubusercontent.com/Australian-Structural-Biology-Computing/bindflow/refs/heads/main/assets/bindcraft/PDL1.pdb,A,56,65,150,4,https://raw.githubusercontent.com/Australian-Structural-Biology-Computing/bindflow/refs/heads/main/assets/bindcraft/default_filters.json,https://raw.githubusercontent.com/Australian-Structural-Biology-Computing/bindflow/refs/heads/main/assets/bindcraft/default_4stage_multimer.json +s1,PDL1,https://raw.githubusercontent.com/Australian-Structural-Biology-Computing/bindflow/refs/heads/main/assets/bindcraft/PDL1.pdb,A,56,65,150,4,https://raw.githubusercontent.com/Australian-Structural-Biology-Computing/bindflow/refs/heads/main/assets/bindcraft/default_filters.json,assets/bindcraft/fast_4stage_multimer.json diff --git a/bindflow-launcher.yaml b/bindflow-launcher.yaml new file mode 100644 index 0000000..8ca7ac6 --- /dev/null +++ b/bindflow-launcher.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nextflow-runner + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: nextflow-runner + namespace: default +rules: +- apiGroups: [""] + resources: ["pods", "pods/log", "events"] + verbs: ["get", "list", "watch", "create", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: nextflow-runner + namespace: default +subjects: +- kind: ServiceAccount + name: nextflow-runner + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nextflow-runner +--- +apiVersion: v1 +kind: Pod +metadata: + name: bindflow-launcher + namespace: default +spec: + serviceAccountName: nextflow-runner + restartPolicy: Never + containers: + - name: launcher + image: nextflow/nextflow:25.10.4 + command: ["/bin/bash", "-lc", "sleep infinity"] + env: + - name: NXF_HOME + value: /workspace/.nextflow + volumeMounts: + - name: work + mountPath: /workspace + volumes: + - name: work + persistentVolumeClaim: + claimName: bindflow-rwo diff --git a/conf/k8s_gpu.config b/conf/k8s_gpu.config new file mode 100644 index 0000000..7f2b34a --- /dev/null +++ b/conf/k8s_gpu.config @@ -0,0 +1,64 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Kubernetes + GPU profile for bindflow +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Notes: + - This profile is intended for Kubernetes clusters with NVIDIA device plugin enabled. + - Use an OCI image tag for --bindcraft_container when running on k8s. + - Set --k8s_storage_claim to a RWX PVC shared by all pods. +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Kubernetes + GPU' + config_profile_description = 'Kubernetes executor tuned for GPU-backed BINDCRAFT tasks' +} + +k8s { + namespace = params.k8s_namespace + serviceAccount = params.k8s_service_account + pullPolicy = params.k8s_pull_policy + runAsUser = 1000 +} + +if( params.k8s_storage_claim ) { + k8s.storageClaimName = params.k8s_storage_claim + k8s.storageMountPath = params.k8s_storage_mount +} + +if( params.k8s_workdir ) { + workDir = params.k8s_workdir +} + +process { + executor = 'k8s' + queueSize = params.gpu_parallel_tasks as int + + withName: 'JSONMANAGER' { + cpus = 1 + memory = 2.GB + time = 1.h + } + + withName: 'RANKER' { + cpus = 1 + memory = 2.GB + time = 1.h + } + + withName: 'BINDCRAFT' { + cpus = 8 + memory = 32.GB + time = 24.h + accelerator = params.gpu_per_task as int + maxForks = params.gpu_parallel_tasks as int + } +} + +docker.enabled = false +apptainer.enabled = false +singularity.enabled = false +podman.enabled = false +conda.enabled = false +charliecloud.enabled = false +shifter.enabled = false diff --git a/conf/local_gpu.config b/conf/local_gpu.config new file mode 100644 index 0000000..d8f767d --- /dev/null +++ b/conf/local_gpu.config @@ -0,0 +1,38 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Local GPU profile (no OCI push required) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Uses Singularity + local .img container with NVIDIA passthrough. + Tune parallelism with --gpu_parallel_tasks (recommended: 1 per physical GPU). +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Local GPU (Singularity)' + config_profile_description = 'Run on local host GPUs using Singularity .img containers' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '--nv' +} + +apptainer.enabled = false +docker.enabled = false +podman.enabled = false +conda.enabled = false +charliecloud.enabled = false +shifter.enabled = false + +process { + executor = 'local' + queueSize = params.gpu_parallel_tasks as int + + withName: 'BINDCRAFT' { + cpus = 8 + memory = 32.GB + time = 24.h + maxForks = params.gpu_parallel_tasks as int + } +} diff --git a/modules/local/bindcraft/main.nf b/modules/local/bindcraft/main.nf index e1e6cda..d97f03b 100644 --- a/modules/local/bindcraft/main.nf +++ b/modules/local/bindcraft/main.nf @@ -20,8 +20,17 @@ process BINDCRAFT { script: def version = "1.2.0" def args = task.ext.args ?: '' + def gpuCount = (params.gpu_device_count ?: 1) as Integer + def gpuId = ((task.index ?: 1) - 1) % gpuCount """ + export CUDA_VISIBLE_DEVICES=${gpuId} + export XLA_PYTHON_CLIENT_PREALLOCATE=false + export XLA_PYTHON_CLIENT_ALLOCATOR=platform + export XLA_PYTHON_CLIENT_MEM_FRACTION=0.60 + export TF_FORCE_GPU_ALLOW_GROWTH=true + echo "BINDCRAFT task ${task.index ?: 1} using CUDA_VISIBLE_DEVICES=\$CUDA_VISIBLE_DEVICES" + /app/run_bindcraft.sh \\ --settings ${target_file} \\ --filters ${filters} \\ diff --git a/nextflow.config b/nextflow.config index 81f9c31..83b6c1c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -18,6 +18,15 @@ params { batches = 1 quote_char = "\"" bindcraft_container = null + gpu_device_count = 1 + gpu_per_task = 1 + gpu_parallel_tasks = 1 + k8s_namespace = null + k8s_service_account = null + k8s_storage_claim = null + k8s_storage_mount = '/workspace' + k8s_workdir = null + k8s_pull_policy = 'IfNotPresent' // MultiQC options multiqc_config = null @@ -99,6 +108,7 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + singularity.runOptions = '--nv' conda.enabled = false docker.enabled = false podman.enabled = false @@ -136,6 +146,7 @@ profiles { apptainer { apptainer.enabled = true apptainer.autoMounts = true + apptainer.runOptions = '--nv' conda.enabled = false docker.enabled = false singularity.enabled = false @@ -154,6 +165,8 @@ profiles { test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } gadi { includeConfig 'conf/gadi.config' } + local_gpu { includeConfig 'conf/local_gpu.config' } + k8s_gpu { includeConfig 'conf/k8s_gpu.config' } } // Load nf-core custom profiles from different Institutions @@ -185,15 +198,8 @@ env { JULIA_DEPOT_PATH = "/usr/local/share/julia" } -// Set bash options -process.shell = """\ -bash - -set -e # Exit if a tool returns a non-zero status/exit code -set -u # Treat unset variables and parameters as an error -set -o pipefail # Returns the status of the last command to exit with a non-zero status or zero if all successfully execute -set -C # No clobber - prevent output redirection from overwriting files. -""" +// Set bash options as argv tokens (Nextflow does not accept newlines here) +process.shell = ['/bin/bash', '-euo', 'pipefail', '-C'] // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 1e207a1..0b773df 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -52,9 +52,7 @@ "properties": { "bindcraft_container": { "type": "string", - "format": "file-path", - "exists": true, - "description": "Path to bindcraft container to be used during execution.", + "description": "Bindcraft container reference. Use a local image path for Singularity/Apptainer or an OCI image URI (e.g. ghcr.io/org/image:tag) for Docker/Kubernetes.", "help_text": "", "fa_icon": "fas fa-file-csv" }, @@ -98,6 +96,52 @@ "description": ".", "help_text": "", "fa_icon": "fas fa-file-json" + }, + "gpu_per_task": { + "type": "integer", + "default": 1, + "description": "Number of GPUs requested per BINDCRAFT task in k8s_gpu profile.", + "help_text": "" + }, + "gpu_device_count": { + "type": "integer", + "default": 1, + "description": "Number of visible GPUs used for local task-to-GPU mapping (CUDA_VISIBLE_DEVICES).", + "help_text": "" + }, + "gpu_parallel_tasks": { + "type": "integer", + "default": 1, + "description": "Maximum number of concurrent BINDCRAFT GPU tasks in k8s_gpu profile.", + "help_text": "" + }, + "k8s_namespace": { + "type": ["string", "null"], + "description": "Kubernetes namespace for workflow pods.", + "help_text": "" + }, + "k8s_service_account": { + "type": ["string", "null"], + "description": "Kubernetes service account used by workflow pods.", + "help_text": "" + }, + "k8s_storage_claim": { + "type": ["string", "null"], + "description": "RWX PVC claim name to mount as shared pipeline storage in k8s_gpu profile.", + "help_text": "" + }, + "k8s_storage_mount": { + "type": "string", + "default": "/workspace", + "description": "Mount path inside pods for the shared PVC.", + "help_text": "" + }, + "k8s_pull_policy": { + "type": "string", + "default": "IfNotPresent", + "description": "Image pull policy for Kubernetes pods.", + "enum": ["Always", "IfNotPresent", "Never"], + "help_text": "" } }