From 0527f66018d6e43939cf33a4c1ab4faa832030db Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Thu, 4 Oct 2018 11:16:50 +0200 Subject: [PATCH 01/58] Moved files around --- .../all-worker-types/main.go | 0 .../generic-worker/artifacts.go | 0 .../generic-worker/artifacts_test.go | 0 aws.go => cmd/generic-worker/aws.go | 0 .../generic-worker/aws_helper_test.go | 0 aws_test.go => cmd/generic-worker/aws_test.go | 0 .../generic-worker/chain_of_trust.go | 0 .../chain_of_trust_all-unix-style.go | 0 .../generic-worker/chain_of_trust_windows.go | 0 .../generic-worker/config_test.go | 0 .../diskspace_all-unix-style.go | 0 .../generic-worker/diskspace_windows.go | 0 feature.go => cmd/generic-worker/feature.go | 0 .../generic-worker/garbagecollector.go | 0 .../generated_all-unix-style.go | 0 .../generic-worker/generated_windows.go | 0 .../helper_all-unix-style_test.go | 0 .../generic-worker/helper_test.go | 0 .../generic-worker/helper_windows_test.go | 0 .../generic-worker/intermittent_task_test.go | 0 livelog.go => cmd/generic-worker/livelog.go | 0 main.go => cmd/generic-worker/main.go | 0 .../generic-worker/main_test.go | 0 model.go => cmd/generic-worker/model.go | 0 mounts.go => cmd/generic-worker/mounts.go | 0 .../generic-worker/mounts_test.go | 0 .../generic-worker/mounts_windows_test.go | 0 openpgp.go => cmd/generic-worker/openpgp.go | 0 .../generic-worker/os_groups.go | 0 .../os_groups_all-unix-style.go | 0 .../os_groups_all-unix-style_test.go | 0 .../generic-worker/os_groups_windows.go | 0 .../generic-worker/os_groups_windows_test.go | 0 .../generic-worker/payload_test.go | 0 .../generic-worker/plat_all-unix-style.go | 0 .../plat_all-unix-style_test.go | 0 .../generic-worker/plat_darwin.go | 0 .../generic-worker/plat_windows.go | 0 .../generic-worker/plat_windows_test.go | 0 .../generic-worker/rdp_windows.go | 0 .../runasadministrator_windows.go | 0 .../runasadministrator_windows_test.go | 0 runtime.go => cmd/generic-worker/runtime.go | 0 sentry.go => cmd/generic-worker/sentry.go | 0 .../generic-worker/supersede.go | 0 .../generic-worker/supersede_test.go | 0 .../generic-worker/taskcluster_proxy.go | 0 .../generic-worker/taskcluster_proxy_test.go | 0 .../generic-worker/taskstatus.go | 0 .../generic-worker/taskstatus_test.go | 0 .../user_creation_windows_test.go | 0 {gw-codegen => cmd/gw-codegen}/main.go | 0 {gw-workers => cmd/gw-workers}/main.go | 0 cmd/inspect-worker-types/main.go | 98 +++++++++ cmd/list-worker-types/main.go | 195 ++++++++++++++++++ cmd/list-worker-types/types.go | 177 ++++++++++++++++ {occ-workers => cmd/occ-workers}/main.go | 0 .../refresh-gw-configs/main.go | 0 .../testdata}/SampleArtifacts/%%%/v/X | 0 .../testdata}/SampleArtifacts/_/X.txt | 0 .../testdata}/SampleArtifacts/b/c/d.jpg | 0 .../testdata}/config/bool-as-string.json | 0 .../testdata}/config/invalid-ip.json | 0 .../testdata}/config/invalid-json.json | 0 {testdata => cmd/testdata}/config/noip.json | 0 {testdata => cmd/testdata}/config/valid.json | 0 .../config/worker-type-metadata.json | 0 {testdata => cmd/testdata}/curlget.go | 0 .../testdata}/machine-configuration.json | 0 .../testdata}/mouse_and_screen_resolution.py | 0 {testdata => cmd/testdata}/move-file.go | 0 .../testdata}/private-opengpg-key | 0 {testdata => cmd/testdata}/public-openpgp-key | 0 {testdata => cmd/testdata}/resolvetask.go | 0 {testdata => cmd/testdata}/run-after-user.bat | 0 .../testdata}/spawn-orphan-process.go | 0 .../tasks/KTBKfEgxR5GdfIIREQIvFQ.json | 0 .../tasks/LK1Rz2UtT16d-HBSqyCtuA.json | 0 {testdata => cmd/testdata}/tasks/README.md | 0 .../tasks/VESwp9JaRo-XkFN_bemBhw.json | 0 {worker_types => cmd}/transform-occ/main.go | 0 .../transform-occ/main_test.go | 0 .../update-ssl-creds}/main.go | 0 .../update-worker-type}/update-worker-type.go | 0 {yamltojson => cmd/yamltojson}/main.go | 0 {fileutil => lib/fileutil}/fileutil.go | 0 {gwconfig => lib/gwconfig}/config.go | 0 {gwconfig => lib/gwconfig}/mergeconfig.go | 0 {livelog => lib/livelog}/livelog.go | 0 {livelog => lib/livelog}/livelog_test.go | 0 {process => lib/process}/logininfo_windows.go | 0 {process => lib/process}/misc_windows.go | 0 {process => lib/process}/process.go | 0 .../process}/process_all-unix-style.go | 0 {process => lib/process}/process_windows.go | 0 {runtime => lib/runtime}/runtime_windows.go | 0 {tcproxy => lib/tcproxy}/tcproxy.go | 0 {tcproxy => lib/tcproxy}/tcproxy_test.go | 0 {win32 => lib/win32}/win32_windows.go | 0 {win32 => lib/win32}/win32_windows_386.go | 0 {win32 => lib/win32}/win32_windows_amd64.go | 0 {win32 => lib/win32}/win32_windows_test.go | 0 {win32 => lib/win32}/win32acl_windows.go | 0 {win32 => lib/win32}/wrap_dll_windows.go | 0 .../scripts}/gecko-try.sh | 0 .../scripts}/gecko.patch | 0 .../scripts}/lib/tooltool.py | 0 .../scripts}/nss-try.sh | 0 .../scripts}/nss.patch | 0 .../scripts}/waitforOCC.go | 0 .../worker_types}/README.md | 0 .../ami-test-win2012r2/ami-base-name | 0 .../worker_types}/ami-test-win2012r2/userdata | 0 .../ami-test-win7sp1/ami-base-name | 0 .../worker_types}/ami-test-win7sp1/userdata | 0 .../worker_types}/cleanup_ec2.sh | 0 .../worker_types}/delete.sh | 0 .../worker_types}/find_old_aws_objects.sh | 0 .../gecko-1-b-win2012-beta/ami-base-name | 0 .../gecko-1-b-win2012-beta/userdata | 0 .../gecko-1-b-win2012/ami-base-name | 0 .../worker_types}/gecko-1-b-win2012/userdata | 0 .../gecko-2-b-win2012/ami-base-name | 0 .../worker_types}/gecko-2-b-win2012/userdata | 0 .../gecko-3-b-win2012/ami-base-name | 0 .../worker_types}/gecko-3-b-win2012/userdata | 0 .../gecko-t-win10-64-beta/ami-base-name | 0 .../gecko-t-win10-64-beta/userdata | 0 .../gecko-t-win10-64-cu/ami-base-name | 0 .../gecko-t-win10-64-cu/userdata | 0 .../gecko-t-win10-64-gpu-b/ami-base-name | 0 .../gecko-t-win10-64-gpu-b/userdata | 0 .../gecko-t-win10-64-gpu/ami-base-name | 0 .../gecko-t-win10-64-gpu/userdata | 0 .../gecko-t-win10-64-hw/ami-base-name | 0 .../gecko-t-win10-64/ami-base-name | 0 .../worker_types}/gecko-t-win10-64/userdata | 0 .../gecko-t-win7-32-beta/ami-base-name | 0 .../gecko-t-win7-32-beta/userdata | 0 .../gecko-t-win7-32-cu/ami-base-name | 0 .../worker_types}/gecko-t-win7-32-cu/userdata | 0 .../gecko-t-win7-32-gpu-b/ami-base-name | 0 .../gecko-t-win7-32-gpu-b/userdata | 0 .../gecko-t-win7-32-gpu/ami-base-name | 0 .../gecko-t-win7-32-gpu/userdata | 0 .../gecko-t-win7-32-hw/ami-base-name | 0 .../gecko-t-win7-32/ami-base-name | 0 .../worker_types}/gecko-t-win7-32/userdata | 0 .../worker_types}/generate_occ_userdata.sh | 0 .../nss-win2012r2-new/ami-base-name | 0 .../worker_types}/nss-win2012r2-new/userdata | 0 .../worker_types}/nss-win2012r2/ami-base-name | 0 .../worker_types}/nss-win2012r2/userdata | 0 .../worker_types}/process_region.sh | 0 .../worker_types}/set_occ_ami_base_names.sh | 0 .../worker_types}/transform.sh | 0 .../worker_types}/update.sh | 0 .../worker_types}/win2012r2-cu/ami-base-name | 0 .../worker_types}/win2012r2-cu/userdata | 0 .../worker_types}/win2012r2/ami-base-name | 0 .../worker_types}/win2012r2/userdata | 0 .../worker_types}/win2016/ami-base-name | 0 .../worker_types}/win2016/userdata | 0 .../worker_types}/worker_type.sh | 0 .../all-unix-style.yml | 0 windows.yml => schemas/windows.yml | 0 build.cmd => scripts/build.cmd | 0 build.sh => scripts/build.sh | 0 clean-builds.sh => scripts/clean-builds.sh | 0 gotestcover.sh => scripts/gotestcover.sh | 0 .../publish-payload-schema.sh | 0 .../refresh_readme.sh | 0 release.sh => scripts/release.sh | 0 .../tutorial-audio.sh | 0 174 files changed, 470 insertions(+) rename {worker_types => cmd}/all-worker-types/main.go (100%) rename artifacts.go => cmd/generic-worker/artifacts.go (100%) rename artifacts_test.go => cmd/generic-worker/artifacts_test.go (100%) rename aws.go => cmd/generic-worker/aws.go (100%) rename aws_helper_test.go => cmd/generic-worker/aws_helper_test.go (100%) rename aws_test.go => cmd/generic-worker/aws_test.go (100%) rename chain_of_trust.go => cmd/generic-worker/chain_of_trust.go (100%) rename chain_of_trust_all-unix-style.go => cmd/generic-worker/chain_of_trust_all-unix-style.go (100%) rename chain_of_trust_windows.go => cmd/generic-worker/chain_of_trust_windows.go (100%) rename config_test.go => cmd/generic-worker/config_test.go (100%) rename diskspace_all-unix-style.go => cmd/generic-worker/diskspace_all-unix-style.go (100%) rename diskspace_windows.go => cmd/generic-worker/diskspace_windows.go (100%) rename feature.go => cmd/generic-worker/feature.go (100%) rename garbagecollector.go => cmd/generic-worker/garbagecollector.go (100%) rename generated_all-unix-style.go => cmd/generic-worker/generated_all-unix-style.go (100%) rename generated_windows.go => cmd/generic-worker/generated_windows.go (100%) rename helper_all-unix-style_test.go => cmd/generic-worker/helper_all-unix-style_test.go (100%) rename helper_test.go => cmd/generic-worker/helper_test.go (100%) rename helper_windows_test.go => cmd/generic-worker/helper_windows_test.go (100%) rename intermittent_task_test.go => cmd/generic-worker/intermittent_task_test.go (100%) rename livelog.go => cmd/generic-worker/livelog.go (100%) rename main.go => cmd/generic-worker/main.go (100%) rename main_test.go => cmd/generic-worker/main_test.go (100%) rename model.go => cmd/generic-worker/model.go (100%) rename mounts.go => cmd/generic-worker/mounts.go (100%) rename mounts_test.go => cmd/generic-worker/mounts_test.go (100%) rename mounts_windows_test.go => cmd/generic-worker/mounts_windows_test.go (100%) rename openpgp.go => cmd/generic-worker/openpgp.go (100%) rename os_groups.go => cmd/generic-worker/os_groups.go (100%) rename os_groups_all-unix-style.go => cmd/generic-worker/os_groups_all-unix-style.go (100%) rename os_groups_all-unix-style_test.go => cmd/generic-worker/os_groups_all-unix-style_test.go (100%) rename os_groups_windows.go => cmd/generic-worker/os_groups_windows.go (100%) rename os_groups_windows_test.go => cmd/generic-worker/os_groups_windows_test.go (100%) rename payload_test.go => cmd/generic-worker/payload_test.go (100%) rename plat_all-unix-style.go => cmd/generic-worker/plat_all-unix-style.go (100%) rename plat_all-unix-style_test.go => cmd/generic-worker/plat_all-unix-style_test.go (100%) rename plat_darwin.go => cmd/generic-worker/plat_darwin.go (100%) rename plat_windows.go => cmd/generic-worker/plat_windows.go (100%) rename plat_windows_test.go => cmd/generic-worker/plat_windows_test.go (100%) rename rdp_windows.go => cmd/generic-worker/rdp_windows.go (100%) rename runasadministrator_windows.go => cmd/generic-worker/runasadministrator_windows.go (100%) rename runasadministrator_windows_test.go => cmd/generic-worker/runasadministrator_windows_test.go (100%) rename runtime.go => cmd/generic-worker/runtime.go (100%) rename sentry.go => cmd/generic-worker/sentry.go (100%) rename supersede.go => cmd/generic-worker/supersede.go (100%) rename supersede_test.go => cmd/generic-worker/supersede_test.go (100%) rename taskcluster_proxy.go => cmd/generic-worker/taskcluster_proxy.go (100%) rename taskcluster_proxy_test.go => cmd/generic-worker/taskcluster_proxy_test.go (100%) rename taskstatus.go => cmd/generic-worker/taskstatus.go (100%) rename taskstatus_test.go => cmd/generic-worker/taskstatus_test.go (100%) rename user_creation_windows_test.go => cmd/generic-worker/user_creation_windows_test.go (100%) rename {gw-codegen => cmd/gw-codegen}/main.go (100%) rename {gw-workers => cmd/gw-workers}/main.go (100%) create mode 100644 cmd/inspect-worker-types/main.go create mode 100644 cmd/list-worker-types/main.go create mode 100644 cmd/list-worker-types/types.go rename {occ-workers => cmd/occ-workers}/main.go (100%) rename {worker_types => cmd}/refresh-gw-configs/main.go (100%) rename {testdata => cmd/testdata}/SampleArtifacts/%%%/v/X (100%) rename {testdata => cmd/testdata}/SampleArtifacts/_/X.txt (100%) rename {testdata => cmd/testdata}/SampleArtifacts/b/c/d.jpg (100%) rename {testdata => cmd/testdata}/config/bool-as-string.json (100%) rename {testdata => cmd/testdata}/config/invalid-ip.json (100%) rename {testdata => cmd/testdata}/config/invalid-json.json (100%) rename {testdata => cmd/testdata}/config/noip.json (100%) rename {testdata => cmd/testdata}/config/valid.json (100%) rename {testdata => cmd/testdata}/config/worker-type-metadata.json (100%) rename {testdata => cmd/testdata}/curlget.go (100%) rename {testdata => cmd/testdata}/machine-configuration.json (100%) rename {testdata => cmd/testdata}/mouse_and_screen_resolution.py (100%) rename {testdata => cmd/testdata}/move-file.go (100%) rename {testdata => cmd/testdata}/private-opengpg-key (100%) rename {testdata => cmd/testdata}/public-openpgp-key (100%) rename {testdata => cmd/testdata}/resolvetask.go (100%) rename {testdata => cmd/testdata}/run-after-user.bat (100%) rename {testdata => cmd/testdata}/spawn-orphan-process.go (100%) rename {testdata => cmd/testdata}/tasks/KTBKfEgxR5GdfIIREQIvFQ.json (100%) rename {testdata => cmd/testdata}/tasks/LK1Rz2UtT16d-HBSqyCtuA.json (100%) rename {testdata => cmd/testdata}/tasks/README.md (100%) rename {testdata => cmd/testdata}/tasks/VESwp9JaRo-XkFN_bemBhw.json (100%) rename {worker_types => cmd}/transform-occ/main.go (100%) rename {worker_types => cmd}/transform-occ/main_test.go (100%) rename {update-ssl-creds => cmd/update-ssl-creds}/main.go (100%) rename {update-worker-type => cmd/update-worker-type}/update-worker-type.go (100%) rename {yamltojson => cmd/yamltojson}/main.go (100%) rename {fileutil => lib/fileutil}/fileutil.go (100%) rename {gwconfig => lib/gwconfig}/config.go (100%) rename {gwconfig => lib/gwconfig}/mergeconfig.go (100%) rename {livelog => lib/livelog}/livelog.go (100%) rename {livelog => lib/livelog}/livelog_test.go (100%) rename {process => lib/process}/logininfo_windows.go (100%) rename {process => lib/process}/misc_windows.go (100%) rename {process => lib/process}/process.go (100%) rename {process => lib/process}/process_all-unix-style.go (100%) rename {process => lib/process}/process_windows.go (100%) rename {runtime => lib/runtime}/runtime_windows.go (100%) rename {tcproxy => lib/tcproxy}/tcproxy.go (100%) rename {tcproxy => lib/tcproxy}/tcproxy_test.go (100%) rename {win32 => lib/win32}/win32_windows.go (100%) rename {win32 => lib/win32}/win32_windows_386.go (100%) rename {win32 => lib/win32}/win32_windows_amd64.go (100%) rename {win32 => lib/win32}/win32_windows_test.go (100%) rename {win32 => lib/win32}/win32acl_windows.go (100%) rename {win32 => lib/win32}/wrap_dll_windows.go (100%) rename {mozilla-try-scripts => mozilla/scripts}/gecko-try.sh (100%) rename {mozilla-try-scripts => mozilla/scripts}/gecko.patch (100%) rename {mozilla-try-scripts => mozilla/scripts}/lib/tooltool.py (100%) rename {mozilla-try-scripts => mozilla/scripts}/nss-try.sh (100%) rename {mozilla-try-scripts => mozilla/scripts}/nss.patch (100%) rename {mozilla-try-scripts => mozilla/scripts}/waitforOCC.go (100%) rename {worker_types => mozilla/worker_types}/README.md (100%) rename {worker_types => mozilla/worker_types}/ami-test-win2012r2/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/ami-test-win2012r2/userdata (100%) rename {worker_types => mozilla/worker_types}/ami-test-win7sp1/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/ami-test-win7sp1/userdata (100%) rename {worker_types => mozilla/worker_types}/cleanup_ec2.sh (100%) rename {worker_types => mozilla/worker_types}/delete.sh (100%) rename {worker_types => mozilla/worker_types}/find_old_aws_objects.sh (100%) rename {worker_types => mozilla/worker_types}/gecko-1-b-win2012-beta/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-1-b-win2012-beta/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-1-b-win2012/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-1-b-win2012/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-2-b-win2012/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-2-b-win2012/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-3-b-win2012/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-3-b-win2012/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-beta/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-beta/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-cu/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-cu/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-gpu-b/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-gpu-b/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-gpu/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-gpu/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64-hw/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win10-64/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-beta/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-beta/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-cu/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-cu/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-gpu-b/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-gpu-b/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-gpu/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-gpu/userdata (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32-hw/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/gecko-t-win7-32/userdata (100%) rename {worker_types => mozilla/worker_types}/generate_occ_userdata.sh (100%) rename {worker_types => mozilla/worker_types}/nss-win2012r2-new/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/nss-win2012r2-new/userdata (100%) rename {worker_types => mozilla/worker_types}/nss-win2012r2/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/nss-win2012r2/userdata (100%) rename {worker_types => mozilla/worker_types}/process_region.sh (100%) rename {worker_types => mozilla/worker_types}/set_occ_ami_base_names.sh (100%) rename {worker_types => mozilla/worker_types}/transform.sh (100%) rename {worker_types => mozilla/worker_types}/update.sh (100%) rename {worker_types => mozilla/worker_types}/win2012r2-cu/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/win2012r2-cu/userdata (100%) rename {worker_types => mozilla/worker_types}/win2012r2/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/win2012r2/userdata (100%) rename {worker_types => mozilla/worker_types}/win2016/ami-base-name (100%) rename {worker_types => mozilla/worker_types}/win2016/userdata (100%) rename {worker_types => mozilla/worker_types}/worker_type.sh (100%) rename all-unix-style.yml => schemas/all-unix-style.yml (100%) rename windows.yml => schemas/windows.yml (100%) rename build.cmd => scripts/build.cmd (100%) rename build.sh => scripts/build.sh (100%) rename clean-builds.sh => scripts/clean-builds.sh (100%) rename gotestcover.sh => scripts/gotestcover.sh (100%) rename publish-payload-schema.sh => scripts/publish-payload-schema.sh (100%) rename refresh_readme.sh => scripts/refresh_readme.sh (100%) rename release.sh => scripts/release.sh (100%) rename tutorial-audio.sh => scripts/tutorial-audio.sh (100%) diff --git a/worker_types/all-worker-types/main.go b/cmd/all-worker-types/main.go similarity index 100% rename from worker_types/all-worker-types/main.go rename to cmd/all-worker-types/main.go diff --git a/artifacts.go b/cmd/generic-worker/artifacts.go similarity index 100% rename from artifacts.go rename to cmd/generic-worker/artifacts.go diff --git a/artifacts_test.go b/cmd/generic-worker/artifacts_test.go similarity index 100% rename from artifacts_test.go rename to cmd/generic-worker/artifacts_test.go diff --git a/aws.go b/cmd/generic-worker/aws.go similarity index 100% rename from aws.go rename to cmd/generic-worker/aws.go diff --git a/aws_helper_test.go b/cmd/generic-worker/aws_helper_test.go similarity index 100% rename from aws_helper_test.go rename to cmd/generic-worker/aws_helper_test.go diff --git a/aws_test.go b/cmd/generic-worker/aws_test.go similarity index 100% rename from aws_test.go rename to cmd/generic-worker/aws_test.go diff --git a/chain_of_trust.go b/cmd/generic-worker/chain_of_trust.go similarity index 100% rename from chain_of_trust.go rename to cmd/generic-worker/chain_of_trust.go diff --git a/chain_of_trust_all-unix-style.go b/cmd/generic-worker/chain_of_trust_all-unix-style.go similarity index 100% rename from chain_of_trust_all-unix-style.go rename to cmd/generic-worker/chain_of_trust_all-unix-style.go diff --git a/chain_of_trust_windows.go b/cmd/generic-worker/chain_of_trust_windows.go similarity index 100% rename from chain_of_trust_windows.go rename to cmd/generic-worker/chain_of_trust_windows.go diff --git a/config_test.go b/cmd/generic-worker/config_test.go similarity index 100% rename from config_test.go rename to cmd/generic-worker/config_test.go diff --git a/diskspace_all-unix-style.go b/cmd/generic-worker/diskspace_all-unix-style.go similarity index 100% rename from diskspace_all-unix-style.go rename to cmd/generic-worker/diskspace_all-unix-style.go diff --git a/diskspace_windows.go b/cmd/generic-worker/diskspace_windows.go similarity index 100% rename from diskspace_windows.go rename to cmd/generic-worker/diskspace_windows.go diff --git a/feature.go b/cmd/generic-worker/feature.go similarity index 100% rename from feature.go rename to cmd/generic-worker/feature.go diff --git a/garbagecollector.go b/cmd/generic-worker/garbagecollector.go similarity index 100% rename from garbagecollector.go rename to cmd/generic-worker/garbagecollector.go diff --git a/generated_all-unix-style.go b/cmd/generic-worker/generated_all-unix-style.go similarity index 100% rename from generated_all-unix-style.go rename to cmd/generic-worker/generated_all-unix-style.go diff --git a/generated_windows.go b/cmd/generic-worker/generated_windows.go similarity index 100% rename from generated_windows.go rename to cmd/generic-worker/generated_windows.go diff --git a/helper_all-unix-style_test.go b/cmd/generic-worker/helper_all-unix-style_test.go similarity index 100% rename from helper_all-unix-style_test.go rename to cmd/generic-worker/helper_all-unix-style_test.go diff --git a/helper_test.go b/cmd/generic-worker/helper_test.go similarity index 100% rename from helper_test.go rename to cmd/generic-worker/helper_test.go diff --git a/helper_windows_test.go b/cmd/generic-worker/helper_windows_test.go similarity index 100% rename from helper_windows_test.go rename to cmd/generic-worker/helper_windows_test.go diff --git a/intermittent_task_test.go b/cmd/generic-worker/intermittent_task_test.go similarity index 100% rename from intermittent_task_test.go rename to cmd/generic-worker/intermittent_task_test.go diff --git a/livelog.go b/cmd/generic-worker/livelog.go similarity index 100% rename from livelog.go rename to cmd/generic-worker/livelog.go diff --git a/main.go b/cmd/generic-worker/main.go similarity index 100% rename from main.go rename to cmd/generic-worker/main.go diff --git a/main_test.go b/cmd/generic-worker/main_test.go similarity index 100% rename from main_test.go rename to cmd/generic-worker/main_test.go diff --git a/model.go b/cmd/generic-worker/model.go similarity index 100% rename from model.go rename to cmd/generic-worker/model.go diff --git a/mounts.go b/cmd/generic-worker/mounts.go similarity index 100% rename from mounts.go rename to cmd/generic-worker/mounts.go diff --git a/mounts_test.go b/cmd/generic-worker/mounts_test.go similarity index 100% rename from mounts_test.go rename to cmd/generic-worker/mounts_test.go diff --git a/mounts_windows_test.go b/cmd/generic-worker/mounts_windows_test.go similarity index 100% rename from mounts_windows_test.go rename to cmd/generic-worker/mounts_windows_test.go diff --git a/openpgp.go b/cmd/generic-worker/openpgp.go similarity index 100% rename from openpgp.go rename to cmd/generic-worker/openpgp.go diff --git a/os_groups.go b/cmd/generic-worker/os_groups.go similarity index 100% rename from os_groups.go rename to cmd/generic-worker/os_groups.go diff --git a/os_groups_all-unix-style.go b/cmd/generic-worker/os_groups_all-unix-style.go similarity index 100% rename from os_groups_all-unix-style.go rename to cmd/generic-worker/os_groups_all-unix-style.go diff --git a/os_groups_all-unix-style_test.go b/cmd/generic-worker/os_groups_all-unix-style_test.go similarity index 100% rename from os_groups_all-unix-style_test.go rename to cmd/generic-worker/os_groups_all-unix-style_test.go diff --git a/os_groups_windows.go b/cmd/generic-worker/os_groups_windows.go similarity index 100% rename from os_groups_windows.go rename to cmd/generic-worker/os_groups_windows.go diff --git a/os_groups_windows_test.go b/cmd/generic-worker/os_groups_windows_test.go similarity index 100% rename from os_groups_windows_test.go rename to cmd/generic-worker/os_groups_windows_test.go diff --git a/payload_test.go b/cmd/generic-worker/payload_test.go similarity index 100% rename from payload_test.go rename to cmd/generic-worker/payload_test.go diff --git a/plat_all-unix-style.go b/cmd/generic-worker/plat_all-unix-style.go similarity index 100% rename from plat_all-unix-style.go rename to cmd/generic-worker/plat_all-unix-style.go diff --git a/plat_all-unix-style_test.go b/cmd/generic-worker/plat_all-unix-style_test.go similarity index 100% rename from plat_all-unix-style_test.go rename to cmd/generic-worker/plat_all-unix-style_test.go diff --git a/plat_darwin.go b/cmd/generic-worker/plat_darwin.go similarity index 100% rename from plat_darwin.go rename to cmd/generic-worker/plat_darwin.go diff --git a/plat_windows.go b/cmd/generic-worker/plat_windows.go similarity index 100% rename from plat_windows.go rename to cmd/generic-worker/plat_windows.go diff --git a/plat_windows_test.go b/cmd/generic-worker/plat_windows_test.go similarity index 100% rename from plat_windows_test.go rename to cmd/generic-worker/plat_windows_test.go diff --git a/rdp_windows.go b/cmd/generic-worker/rdp_windows.go similarity index 100% rename from rdp_windows.go rename to cmd/generic-worker/rdp_windows.go diff --git a/runasadministrator_windows.go b/cmd/generic-worker/runasadministrator_windows.go similarity index 100% rename from runasadministrator_windows.go rename to cmd/generic-worker/runasadministrator_windows.go diff --git a/runasadministrator_windows_test.go b/cmd/generic-worker/runasadministrator_windows_test.go similarity index 100% rename from runasadministrator_windows_test.go rename to cmd/generic-worker/runasadministrator_windows_test.go diff --git a/runtime.go b/cmd/generic-worker/runtime.go similarity index 100% rename from runtime.go rename to cmd/generic-worker/runtime.go diff --git a/sentry.go b/cmd/generic-worker/sentry.go similarity index 100% rename from sentry.go rename to cmd/generic-worker/sentry.go diff --git a/supersede.go b/cmd/generic-worker/supersede.go similarity index 100% rename from supersede.go rename to cmd/generic-worker/supersede.go diff --git a/supersede_test.go b/cmd/generic-worker/supersede_test.go similarity index 100% rename from supersede_test.go rename to cmd/generic-worker/supersede_test.go diff --git a/taskcluster_proxy.go b/cmd/generic-worker/taskcluster_proxy.go similarity index 100% rename from taskcluster_proxy.go rename to cmd/generic-worker/taskcluster_proxy.go diff --git a/taskcluster_proxy_test.go b/cmd/generic-worker/taskcluster_proxy_test.go similarity index 100% rename from taskcluster_proxy_test.go rename to cmd/generic-worker/taskcluster_proxy_test.go diff --git a/taskstatus.go b/cmd/generic-worker/taskstatus.go similarity index 100% rename from taskstatus.go rename to cmd/generic-worker/taskstatus.go diff --git a/taskstatus_test.go b/cmd/generic-worker/taskstatus_test.go similarity index 100% rename from taskstatus_test.go rename to cmd/generic-worker/taskstatus_test.go diff --git a/user_creation_windows_test.go b/cmd/generic-worker/user_creation_windows_test.go similarity index 100% rename from user_creation_windows_test.go rename to cmd/generic-worker/user_creation_windows_test.go diff --git a/gw-codegen/main.go b/cmd/gw-codegen/main.go similarity index 100% rename from gw-codegen/main.go rename to cmd/gw-codegen/main.go diff --git a/gw-workers/main.go b/cmd/gw-workers/main.go similarity index 100% rename from gw-workers/main.go rename to cmd/gw-workers/main.go diff --git a/cmd/inspect-worker-types/main.go b/cmd/inspect-worker-types/main.go new file mode 100644 index 00000000..e1f61648 --- /dev/null +++ b/cmd/inspect-worker-types/main.go @@ -0,0 +1,98 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "regexp" + "strings" + "time" + + "github.com/taskcluster/httpbackoff" + "github.com/taskcluster/taskcluster-client-go/tcqueue" +) + +func main() { + taskGroupID := os.Args[1] + fmt.Printf("Task group: %v\n\n", taskGroupID) + queue := tcqueue.NewFromEnv() + query := func(ct string) *tcqueue.ListTaskGroupResponse { + lgtr, err := queue.ListTaskGroup(taskGroupID, ct, "") + if err != nil { + log.Fatal(1, err) + } + for _, t := range lgtr.Tasks { + show(queue, t) + } + return lgtr + } + ct := query("").ContinuationToken + for ct != "" { + ct = query(ct).ContinuationToken + } +} + +func show(queue *tcqueue.Queue, t tcqueue.TaskDefinitionAndStatus) { + name := t.Task.ProvisionerID + "/" + t.Task.WorkerType + ": " + fmt.Print(name[:75]) + if t.Status.State == "pending" { + fmt.Printf("not yet determined - task %v still pending...\n", t.Status.TaskID) + return + } + var resp *http.Response + artifactFound := "" + for _, artifact := range []string{ + "public/logs/live_backing.log", + "public/logs/chain_of_trust.log", + } { + backingLogURL, err := queue.GetLatestArtifact_SignedURL(t.Status.TaskID, artifact, time.Hour) + if err != nil { + log.Fatal(2, err) + } + resp, _, err = httpbackoff.Get(backingLogURL.String()) + if err == nil { + artifactFound = artifact + break + } + switch e := err.(type) { + case httpbackoff.BadHttpResponseCode: + if e.HttpResponseCode == 404 { + continue + } + } + log.Fatal(3, err) + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Print("*** ") + } + logContent := string(data) + switch true { + case strings.Contains(logContent, "Task not successful due to following exception"): + fmt.Println("generic-worker - unknown version") + // case strings.Contains(logContent, "Rejecting Schema: http://schemas.taskcluster.net/docker-worker/v1/payload.json"): + //fmt.Println("worker: docker-worker - unknown version") + case strings.Contains(logContent, "Worker Node Type:"): + fmt.Println("docker-worker - unknown version") + case strings.Contains(logContent, `"release": "https://github.com/taskcluster/generic-worker/releases/tag/v`): + re := regexp.MustCompile(`"https://github.com/taskcluster/generic-worker/releases/tag/v([^"]*)"`) + match := re.FindStringSubmatch(logContent) + fmt.Printf("generic-worker %v\n", match[1]) + case strings.Contains(logContent, `not allowed at task.payload.features`): + fmt.Println("taskcluster-worker - unknown version") + case strings.Contains(logContent, `raise TaskVerificationError`): + fmt.Println("scriptworker - unknown version") + case strings.Contains(logContent, `KeyError: 'artifacts_deps'`): + fmt.Println("some kind of scriptworker - unknown version") + case artifactFound == "": + fmt.Println("No artifacts found!") + case artifactFound == "public/logs/chain_of_trust.log": + fmt.Println("scriptworker chain of trust - unknown version") + default: + fmt.Println("UNKNOWN") + log.Fatal(5, logContent) + } +} diff --git a/cmd/list-worker-types/main.go b/cmd/list-worker-types/main.go new file mode 100644 index 00000000..a4a10d9c --- /dev/null +++ b/cmd/list-worker-types/main.go @@ -0,0 +1,195 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "sort" + "strings" + "sync" + "time" + + "github.com/taskcluster/slugid-go/slugid" + tcclient "github.com/taskcluster/taskcluster-client-go" + "github.com/taskcluster/taskcluster-client-go/tcawsprovisioner" + "github.com/taskcluster/taskcluster-client-go/tcqueue" +) + +type Queue tcqueue.Queue +type Provisioner tcawsprovisioner.AwsProvisioner + +func main() { + taskIDs := map[string]string{} + myQueue := tcqueue.NewFromEnv() + taskGroupID := slugid.Nice() + created := time.Now() + for _, wt := range AllWorkerTypes() { + fmt.Println(wt) + x := strings.Split(wt, "/") + provisionerID := x[0] + workerType := x[1] + taskID := slugid.Nice() + taskIDs[wt] = taskID + payload := GenericWorkerPayload{ + MaxRunTime: 3600, + Command: []string{ + `echo`, + }, + } + payloadJSON := mustCompileToRawMessage(payload) + taskDef := &tcqueue.TaskDefinitionRequest{ + Created: tcclient.Time(created), + Deadline: tcclient.Time(created.Add(time.Hour * 3)), + Dependencies: []string{}, + Expires: tcclient.Time(created.Add(time.Hour * 24 * 30)), + Extra: json.RawMessage("{}"), + Metadata: struct { + Description string `json:"description"` + Name string `json:"name"` + Owner string `json:"owner"` + Source string `json:"source"` + }{ + Description: "Checking worker version on " + provisionerID + "/" + workerType, + Name: "Checking worker version on " + provisionerID + "/" + workerType, + Owner: "pmoore@mozilla.com", + Source: "https://github.com/petemoore", + }, + Payload: *payloadJSON, + Priority: "highest", + ProvisionerID: provisionerID, + Requires: "all-completed", + Retries: 5, + Routes: []string{}, + SchedulerID: "-", + Scopes: []string{}, + Tags: map[string]string{}, + TaskGroupID: taskGroupID, + WorkerType: workerType, + } + tsr, err := myQueue.CreateTask(taskID, taskDef) + fatalOnError(err) + + respJSON, err := json.MarshalIndent(tsr, "", " ") + fatalOnError(err) + + fmt.Println(string(respJSON)) + } + + fmt.Println("Task Group ID: " + taskGroupID) +} + +func mustCompileToRawMessage(data interface{}) *json.RawMessage { + bytes, err := json.Marshal(data) + fatalOnError(err) + var JSON json.RawMessage + err = json.Unmarshal(bytes, &JSON) + fatalOnError(err) + return &JSON +} + +func fatalOnError(err error) { + if err != nil { + log.Fatalf("Error:\n%v", err) + } +} + +func NewProvisioner() *Provisioner { + p := tcawsprovisioner.NewFromEnv() + P := Provisioner(*p) + return &P +} + +func NewQueue() *Queue { + q := tcqueue.NewFromEnv() + Q := Queue(*q) + return &Q +} + +func (p *Provisioner) AllWorkerTypes() []string { + prov := tcawsprovisioner.AwsProvisioner(*p) + wt, err := prov.ListWorkerTypes() + if err != nil { + panic(err) + } + return []string(*wt) +} + +func AllWorkerTypes() []string { + uniqueWorkerTypes := map[string]bool{} + q := NewQueue() + provisioners := q.AllProvisionerIDs() + workerTypes := make([][]string, len(provisioners), len(provisioners)) + var wg sync.WaitGroup + for i, p := range provisioners { + if p == "test-provisioner" || p == "no-provisioning-nope" || p == "dummy-test-provisioner" || p == "test-dummy-provisioner" { + continue + } + wg.Add(1) + go func(p string, i int) { + defer wg.Done() + provWorkerTypes := q.ProvisionerWorkerTypes(p) + workerTypes[i] = make([]string, len(provWorkerTypes), len(provWorkerTypes)) + for j, wt := range provWorkerTypes { + workerTypes[i][j] = p + "/" + wt + } + }(p, i) + } + wg.Wait() + for _, p := range workerTypes { + for _, wt := range p { + uniqueWorkerTypes[wt] = true + } + } + + // Now merge in known worker types according to AWS provisioner + p := NewProvisioner() + provisionerWorkerTypes := p.AllWorkerTypes() + for _, wt := range provisionerWorkerTypes { + uniqueWorkerTypes["aws-provisioner-v1/"+wt] = true + } + + keys := make([]string, 0, len(uniqueWorkerTypes)) + for key := range uniqueWorkerTypes { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func (q *Queue) AllProvisionerIDs() []string { + Q := tcqueue.Queue(*q) + provisioners := []string{} + var r *tcqueue.ListProvisionersResponse + ct := "" + for r == nil || r.ContinuationToken != "" { + var err error + r, err = (&Q).ListProvisioners(ct, "") + if err != nil { + panic(err) + } + ct = r.ContinuationToken + for _, p := range r.Provisioners { + provisioners = append(provisioners, p.ProvisionerID) + } + } + return provisioners +} + +func (q *Queue) ProvisionerWorkerTypes(provisionerID string) []string { + Q := tcqueue.Queue(*q) + workerTypes := []string{} + var r *tcqueue.ListWorkerTypesResponse + ct := "" + for r == nil || r.ContinuationToken != "" { + var err error + r, err = (&Q).ListWorkerTypes(provisionerID, ct, "") + if err != nil { + panic(err) + } + ct = r.ContinuationToken + for _, p := range r.WorkerTypes { + workerTypes = append(workerTypes, p.WorkerType) + } + } + return workerTypes +} diff --git a/cmd/list-worker-types/types.go b/cmd/list-worker-types/types.go new file mode 100644 index 00000000..2d1a5e2a --- /dev/null +++ b/cmd/list-worker-types/types.go @@ -0,0 +1,177 @@ +package main + +import ( + "encoding/json" + + tcclient "github.com/taskcluster/taskcluster-client-go" +) + +// ********************************************************* +// These type definitions are copied from: +// https://github.com/taskcluster/generic-worker/blob/ec86473df8dba68631a50af98e5af7d44d7e1717/generated_windows.go#L40-L201 +// ********************************************************* + +type ( + // This schema defines the structure of the `payload` property referred to in a + // Taskcluster Task definition. + GenericWorkerPayload struct { + + // Artifacts to be published. + // + // Since: generic-worker 1.0.0 + Artifacts []struct { + + // Explicitly set the value of the HTTP `Content-Type` response header when the artifact(s) + // is/are served over HTTP(S). If not provided (this property is optional) the worker will + // guess the content type of artifacts based on the filename extension of the file storing + // the artifact content. It does this by looking at the system filename-to-mimetype mappings + // defined in the Windows registry. Note, setting `contentType` on a directory artifact will + // apply the same contentType to all files contained in the directory. + // + // See [mime.TypeByExtension](https://godoc.org/mime#TypeByExtension). + // + // Since: generic-worker 10.4.0 + ContentType string `json:"contentType,omitempty"` + + // Date when artifact should expire must be in the future, no earlier than task deadline, but + // no later than task expiry. If not set, defaults to task expiry. + // + // Since: generic-worker 1.0.0 + Expires tcclient.Time `json:"expires,omitempty"` + + // Name of the artifact, as it will be published. If not set, `path` will be used. + // Conventionally (although not enforced) path elements are forward slash separated. Example: + // `public/build/a/house`. Note, no scopes are required to read artifacts beginning `public/`. + // Artifact names not beginning `public/` are scope-protected (caller requires scopes to + // download the artifact). See the Queue documentation for more information. + // + // Since: generic-worker 8.1.0 + Name string `json:"name,omitempty"` + + // Relative path of the file/directory from the task directory. Note this is not an absolute + // path as is typically used in docker-worker, since the absolute task directory name is not + // known when the task is submitted. Example: `dist\regedit.exe`. It doesn't matter if + // forward slashes or backslashes are used. + // + // Since: generic-worker 1.0.0 + Path string `json:"path"` + + // Artifacts can be either an individual `file` or a `directory` containing + // potentially multiple files with recursively included subdirectories. + // + // Since: generic-worker 1.0.0 + // + // Possible values: + // * "file" + // * "directory" + Type string `json:"type"` + } `json:"artifacts,omitempty"` + + // One entry per command (consider each entry to be interpreted as a full line of + // a Windows™ .bat file). For example: + // ``` + // [ + // "set", + // "echo hello world > hello_world.txt", + // "set GOPATH=C:\\Go" + // ] + // ``` + // + // Since: generic-worker 0.0.1 + Command []string `json:"command"` + + // Env vars must be string to __string__ mappings (not number or boolean). For example: + // ``` + // { + // "PATH": "C:\\Windows\\system32;C:\\Windows", + // "GOOS": "windows", + // "FOO_ENABLE": "true", + // "BAR_TOTAL": "3" + // } + // ``` + // + // Since: generic-worker 0.0.1 + Env map[string]string `json:"env,omitempty"` + + // Feature flags enable additional functionality. + // + // Since: generic-worker 5.3.0 + Features struct { + + // An artifact named `public/chainOfTrust.json.asc` should be generated + // which will include information for downstream tasks to build a level + // of trust for the artifacts produced by the task and the environment + // it ran in. + // + // Since: generic-worker 5.3.0 + ChainOfTrust bool `json:"chainOfTrust,omitempty"` + + // The taskcluster proxy provides an easy and safe way to make authenticated + // taskcluster requests within the scope(s) of a particular task. See + // [the github project](https://github.com/taskcluster/taskcluster-proxy) for more information. + // + // Since: generic-worker 10.6.0 + TaskclusterProxy bool `json:"taskclusterProxy,omitempty"` + } `json:"features,omitempty"` + + // Maximum time the task container can run in seconds. + // + // Since: generic-worker 0.0.1 + // + // Mininum: 1 + // Maximum: 86400 + MaxRunTime int64 `json:"maxRunTime"` + + // Directories and/or files to be mounted. + // + // Since: generic-worker 5.4.0 + Mounts []Mount `json:"mounts,omitempty"` + + // A list of OS Groups that the task user should be a member of. Requires + // scope `generic-worker:os-group:` for each group listed. + // + // Since: generic-worker 6.0.0 + OSGroups []string `json:"osGroups,omitempty"` + + // Specifies an artifact name for publishing RDP connection information. + // + // Since this is potentially sensitive data, care should be taken to publish + // to a suitably locked down path, such as + // `login-identity//rdpinfo.json` which is only readable for + // the given login identity (for example + // `login-identity/mozilla-ldap/pmoore@mozilla.com/rdpInfo.txt`). See the + // [artifact namespace guide](https://docs.taskcluster.net/manual/design/namespaces#artifacts) for more information. + // + // Use of this feature requires scope + // `generic-worker:allow-rdp:/` which must be + // declared as a task scope. + // + // The RDP connection data is published during task startup so that a user + // may interact with the running task. + // + // The task environment will be retained for 12 hours after the task + // completes, to enable an interactive user to perform investigative tasks. + // After these 12 hours, the worker will delete the task's Windows user + // account, and then continue with other tasks. + // + // No guarantees are given about the resolution status of the interactive + // task, since the task is inherently non-reproducible and no automation + // should rely on this value. + // + // Since: generic-worker 10.5.0 + RdpInfo string `json:"rdpInfo,omitempty"` + + // URL of a service that can indicate tasks superseding this one; the current `taskId` + // will be appended as a query argument `taskId`. The service should return an object with + // a `supersedes` key containing a list of `taskId`s, including the supplied `taskId`. The + // tasks should be ordered such that each task supersedes all tasks appearing later in the + // list. + // + // See [superseding](https://docs.taskcluster.net/reference/platform/taskcluster-queue/docs/superseding) for more detail. + // + // Since: generic-worker 10.2.2 + SupersederURL string `json:"supersederUrl,omitempty"` + } + + Mount json.RawMessage +) diff --git a/occ-workers/main.go b/cmd/occ-workers/main.go similarity index 100% rename from occ-workers/main.go rename to cmd/occ-workers/main.go diff --git a/worker_types/refresh-gw-configs/main.go b/cmd/refresh-gw-configs/main.go similarity index 100% rename from worker_types/refresh-gw-configs/main.go rename to cmd/refresh-gw-configs/main.go diff --git a/testdata/SampleArtifacts/%%%/v/X b/cmd/testdata/SampleArtifacts/%%%/v/X similarity index 100% rename from testdata/SampleArtifacts/%%%/v/X rename to cmd/testdata/SampleArtifacts/%%%/v/X diff --git a/testdata/SampleArtifacts/_/X.txt b/cmd/testdata/SampleArtifacts/_/X.txt similarity index 100% rename from testdata/SampleArtifacts/_/X.txt rename to cmd/testdata/SampleArtifacts/_/X.txt diff --git a/testdata/SampleArtifacts/b/c/d.jpg b/cmd/testdata/SampleArtifacts/b/c/d.jpg similarity index 100% rename from testdata/SampleArtifacts/b/c/d.jpg rename to cmd/testdata/SampleArtifacts/b/c/d.jpg diff --git a/testdata/config/bool-as-string.json b/cmd/testdata/config/bool-as-string.json similarity index 100% rename from testdata/config/bool-as-string.json rename to cmd/testdata/config/bool-as-string.json diff --git a/testdata/config/invalid-ip.json b/cmd/testdata/config/invalid-ip.json similarity index 100% rename from testdata/config/invalid-ip.json rename to cmd/testdata/config/invalid-ip.json diff --git a/testdata/config/invalid-json.json b/cmd/testdata/config/invalid-json.json similarity index 100% rename from testdata/config/invalid-json.json rename to cmd/testdata/config/invalid-json.json diff --git a/testdata/config/noip.json b/cmd/testdata/config/noip.json similarity index 100% rename from testdata/config/noip.json rename to cmd/testdata/config/noip.json diff --git a/testdata/config/valid.json b/cmd/testdata/config/valid.json similarity index 100% rename from testdata/config/valid.json rename to cmd/testdata/config/valid.json diff --git a/testdata/config/worker-type-metadata.json b/cmd/testdata/config/worker-type-metadata.json similarity index 100% rename from testdata/config/worker-type-metadata.json rename to cmd/testdata/config/worker-type-metadata.json diff --git a/testdata/curlget.go b/cmd/testdata/curlget.go similarity index 100% rename from testdata/curlget.go rename to cmd/testdata/curlget.go diff --git a/testdata/machine-configuration.json b/cmd/testdata/machine-configuration.json similarity index 100% rename from testdata/machine-configuration.json rename to cmd/testdata/machine-configuration.json diff --git a/testdata/mouse_and_screen_resolution.py b/cmd/testdata/mouse_and_screen_resolution.py similarity index 100% rename from testdata/mouse_and_screen_resolution.py rename to cmd/testdata/mouse_and_screen_resolution.py diff --git a/testdata/move-file.go b/cmd/testdata/move-file.go similarity index 100% rename from testdata/move-file.go rename to cmd/testdata/move-file.go diff --git a/testdata/private-opengpg-key b/cmd/testdata/private-opengpg-key similarity index 100% rename from testdata/private-opengpg-key rename to cmd/testdata/private-opengpg-key diff --git a/testdata/public-openpgp-key b/cmd/testdata/public-openpgp-key similarity index 100% rename from testdata/public-openpgp-key rename to cmd/testdata/public-openpgp-key diff --git a/testdata/resolvetask.go b/cmd/testdata/resolvetask.go similarity index 100% rename from testdata/resolvetask.go rename to cmd/testdata/resolvetask.go diff --git a/testdata/run-after-user.bat b/cmd/testdata/run-after-user.bat similarity index 100% rename from testdata/run-after-user.bat rename to cmd/testdata/run-after-user.bat diff --git a/testdata/spawn-orphan-process.go b/cmd/testdata/spawn-orphan-process.go similarity index 100% rename from testdata/spawn-orphan-process.go rename to cmd/testdata/spawn-orphan-process.go diff --git a/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json b/cmd/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json similarity index 100% rename from testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json rename to cmd/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json diff --git a/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json b/cmd/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json similarity index 100% rename from testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json rename to cmd/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json diff --git a/testdata/tasks/README.md b/cmd/testdata/tasks/README.md similarity index 100% rename from testdata/tasks/README.md rename to cmd/testdata/tasks/README.md diff --git a/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json b/cmd/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json similarity index 100% rename from testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json rename to cmd/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json diff --git a/worker_types/transform-occ/main.go b/cmd/transform-occ/main.go similarity index 100% rename from worker_types/transform-occ/main.go rename to cmd/transform-occ/main.go diff --git a/worker_types/transform-occ/main_test.go b/cmd/transform-occ/main_test.go similarity index 100% rename from worker_types/transform-occ/main_test.go rename to cmd/transform-occ/main_test.go diff --git a/update-ssl-creds/main.go b/cmd/update-ssl-creds/main.go similarity index 100% rename from update-ssl-creds/main.go rename to cmd/update-ssl-creds/main.go diff --git a/update-worker-type/update-worker-type.go b/cmd/update-worker-type/update-worker-type.go similarity index 100% rename from update-worker-type/update-worker-type.go rename to cmd/update-worker-type/update-worker-type.go diff --git a/yamltojson/main.go b/cmd/yamltojson/main.go similarity index 100% rename from yamltojson/main.go rename to cmd/yamltojson/main.go diff --git a/fileutil/fileutil.go b/lib/fileutil/fileutil.go similarity index 100% rename from fileutil/fileutil.go rename to lib/fileutil/fileutil.go diff --git a/gwconfig/config.go b/lib/gwconfig/config.go similarity index 100% rename from gwconfig/config.go rename to lib/gwconfig/config.go diff --git a/gwconfig/mergeconfig.go b/lib/gwconfig/mergeconfig.go similarity index 100% rename from gwconfig/mergeconfig.go rename to lib/gwconfig/mergeconfig.go diff --git a/livelog/livelog.go b/lib/livelog/livelog.go similarity index 100% rename from livelog/livelog.go rename to lib/livelog/livelog.go diff --git a/livelog/livelog_test.go b/lib/livelog/livelog_test.go similarity index 100% rename from livelog/livelog_test.go rename to lib/livelog/livelog_test.go diff --git a/process/logininfo_windows.go b/lib/process/logininfo_windows.go similarity index 100% rename from process/logininfo_windows.go rename to lib/process/logininfo_windows.go diff --git a/process/misc_windows.go b/lib/process/misc_windows.go similarity index 100% rename from process/misc_windows.go rename to lib/process/misc_windows.go diff --git a/process/process.go b/lib/process/process.go similarity index 100% rename from process/process.go rename to lib/process/process.go diff --git a/process/process_all-unix-style.go b/lib/process/process_all-unix-style.go similarity index 100% rename from process/process_all-unix-style.go rename to lib/process/process_all-unix-style.go diff --git a/process/process_windows.go b/lib/process/process_windows.go similarity index 100% rename from process/process_windows.go rename to lib/process/process_windows.go diff --git a/runtime/runtime_windows.go b/lib/runtime/runtime_windows.go similarity index 100% rename from runtime/runtime_windows.go rename to lib/runtime/runtime_windows.go diff --git a/tcproxy/tcproxy.go b/lib/tcproxy/tcproxy.go similarity index 100% rename from tcproxy/tcproxy.go rename to lib/tcproxy/tcproxy.go diff --git a/tcproxy/tcproxy_test.go b/lib/tcproxy/tcproxy_test.go similarity index 100% rename from tcproxy/tcproxy_test.go rename to lib/tcproxy/tcproxy_test.go diff --git a/win32/win32_windows.go b/lib/win32/win32_windows.go similarity index 100% rename from win32/win32_windows.go rename to lib/win32/win32_windows.go diff --git a/win32/win32_windows_386.go b/lib/win32/win32_windows_386.go similarity index 100% rename from win32/win32_windows_386.go rename to lib/win32/win32_windows_386.go diff --git a/win32/win32_windows_amd64.go b/lib/win32/win32_windows_amd64.go similarity index 100% rename from win32/win32_windows_amd64.go rename to lib/win32/win32_windows_amd64.go diff --git a/win32/win32_windows_test.go b/lib/win32/win32_windows_test.go similarity index 100% rename from win32/win32_windows_test.go rename to lib/win32/win32_windows_test.go diff --git a/win32/win32acl_windows.go b/lib/win32/win32acl_windows.go similarity index 100% rename from win32/win32acl_windows.go rename to lib/win32/win32acl_windows.go diff --git a/win32/wrap_dll_windows.go b/lib/win32/wrap_dll_windows.go similarity index 100% rename from win32/wrap_dll_windows.go rename to lib/win32/wrap_dll_windows.go diff --git a/mozilla-try-scripts/gecko-try.sh b/mozilla/scripts/gecko-try.sh similarity index 100% rename from mozilla-try-scripts/gecko-try.sh rename to mozilla/scripts/gecko-try.sh diff --git a/mozilla-try-scripts/gecko.patch b/mozilla/scripts/gecko.patch similarity index 100% rename from mozilla-try-scripts/gecko.patch rename to mozilla/scripts/gecko.patch diff --git a/mozilla-try-scripts/lib/tooltool.py b/mozilla/scripts/lib/tooltool.py similarity index 100% rename from mozilla-try-scripts/lib/tooltool.py rename to mozilla/scripts/lib/tooltool.py diff --git a/mozilla-try-scripts/nss-try.sh b/mozilla/scripts/nss-try.sh similarity index 100% rename from mozilla-try-scripts/nss-try.sh rename to mozilla/scripts/nss-try.sh diff --git a/mozilla-try-scripts/nss.patch b/mozilla/scripts/nss.patch similarity index 100% rename from mozilla-try-scripts/nss.patch rename to mozilla/scripts/nss.patch diff --git a/mozilla-try-scripts/waitforOCC.go b/mozilla/scripts/waitforOCC.go similarity index 100% rename from mozilla-try-scripts/waitforOCC.go rename to mozilla/scripts/waitforOCC.go diff --git a/worker_types/README.md b/mozilla/worker_types/README.md similarity index 100% rename from worker_types/README.md rename to mozilla/worker_types/README.md diff --git a/worker_types/ami-test-win2012r2/ami-base-name b/mozilla/worker_types/ami-test-win2012r2/ami-base-name similarity index 100% rename from worker_types/ami-test-win2012r2/ami-base-name rename to mozilla/worker_types/ami-test-win2012r2/ami-base-name diff --git a/worker_types/ami-test-win2012r2/userdata b/mozilla/worker_types/ami-test-win2012r2/userdata similarity index 100% rename from worker_types/ami-test-win2012r2/userdata rename to mozilla/worker_types/ami-test-win2012r2/userdata diff --git a/worker_types/ami-test-win7sp1/ami-base-name b/mozilla/worker_types/ami-test-win7sp1/ami-base-name similarity index 100% rename from worker_types/ami-test-win7sp1/ami-base-name rename to mozilla/worker_types/ami-test-win7sp1/ami-base-name diff --git a/worker_types/ami-test-win7sp1/userdata b/mozilla/worker_types/ami-test-win7sp1/userdata similarity index 100% rename from worker_types/ami-test-win7sp1/userdata rename to mozilla/worker_types/ami-test-win7sp1/userdata diff --git a/worker_types/cleanup_ec2.sh b/mozilla/worker_types/cleanup_ec2.sh similarity index 100% rename from worker_types/cleanup_ec2.sh rename to mozilla/worker_types/cleanup_ec2.sh diff --git a/worker_types/delete.sh b/mozilla/worker_types/delete.sh similarity index 100% rename from worker_types/delete.sh rename to mozilla/worker_types/delete.sh diff --git a/worker_types/find_old_aws_objects.sh b/mozilla/worker_types/find_old_aws_objects.sh similarity index 100% rename from worker_types/find_old_aws_objects.sh rename to mozilla/worker_types/find_old_aws_objects.sh diff --git a/worker_types/gecko-1-b-win2012-beta/ami-base-name b/mozilla/worker_types/gecko-1-b-win2012-beta/ami-base-name similarity index 100% rename from worker_types/gecko-1-b-win2012-beta/ami-base-name rename to mozilla/worker_types/gecko-1-b-win2012-beta/ami-base-name diff --git a/worker_types/gecko-1-b-win2012-beta/userdata b/mozilla/worker_types/gecko-1-b-win2012-beta/userdata similarity index 100% rename from worker_types/gecko-1-b-win2012-beta/userdata rename to mozilla/worker_types/gecko-1-b-win2012-beta/userdata diff --git a/worker_types/gecko-1-b-win2012/ami-base-name b/mozilla/worker_types/gecko-1-b-win2012/ami-base-name similarity index 100% rename from worker_types/gecko-1-b-win2012/ami-base-name rename to mozilla/worker_types/gecko-1-b-win2012/ami-base-name diff --git a/worker_types/gecko-1-b-win2012/userdata b/mozilla/worker_types/gecko-1-b-win2012/userdata similarity index 100% rename from worker_types/gecko-1-b-win2012/userdata rename to mozilla/worker_types/gecko-1-b-win2012/userdata diff --git a/worker_types/gecko-2-b-win2012/ami-base-name b/mozilla/worker_types/gecko-2-b-win2012/ami-base-name similarity index 100% rename from worker_types/gecko-2-b-win2012/ami-base-name rename to mozilla/worker_types/gecko-2-b-win2012/ami-base-name diff --git a/worker_types/gecko-2-b-win2012/userdata b/mozilla/worker_types/gecko-2-b-win2012/userdata similarity index 100% rename from worker_types/gecko-2-b-win2012/userdata rename to mozilla/worker_types/gecko-2-b-win2012/userdata diff --git a/worker_types/gecko-3-b-win2012/ami-base-name b/mozilla/worker_types/gecko-3-b-win2012/ami-base-name similarity index 100% rename from worker_types/gecko-3-b-win2012/ami-base-name rename to mozilla/worker_types/gecko-3-b-win2012/ami-base-name diff --git a/worker_types/gecko-3-b-win2012/userdata b/mozilla/worker_types/gecko-3-b-win2012/userdata similarity index 100% rename from worker_types/gecko-3-b-win2012/userdata rename to mozilla/worker_types/gecko-3-b-win2012/userdata diff --git a/worker_types/gecko-t-win10-64-beta/ami-base-name b/mozilla/worker_types/gecko-t-win10-64-beta/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-beta/ami-base-name rename to mozilla/worker_types/gecko-t-win10-64-beta/ami-base-name diff --git a/worker_types/gecko-t-win10-64-beta/userdata b/mozilla/worker_types/gecko-t-win10-64-beta/userdata similarity index 100% rename from worker_types/gecko-t-win10-64-beta/userdata rename to mozilla/worker_types/gecko-t-win10-64-beta/userdata diff --git a/worker_types/gecko-t-win10-64-cu/ami-base-name b/mozilla/worker_types/gecko-t-win10-64-cu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-cu/ami-base-name rename to mozilla/worker_types/gecko-t-win10-64-cu/ami-base-name diff --git a/worker_types/gecko-t-win10-64-cu/userdata b/mozilla/worker_types/gecko-t-win10-64-cu/userdata similarity index 100% rename from worker_types/gecko-t-win10-64-cu/userdata rename to mozilla/worker_types/gecko-t-win10-64-cu/userdata diff --git a/worker_types/gecko-t-win10-64-gpu-b/ami-base-name b/mozilla/worker_types/gecko-t-win10-64-gpu-b/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-gpu-b/ami-base-name rename to mozilla/worker_types/gecko-t-win10-64-gpu-b/ami-base-name diff --git a/worker_types/gecko-t-win10-64-gpu-b/userdata b/mozilla/worker_types/gecko-t-win10-64-gpu-b/userdata similarity index 100% rename from worker_types/gecko-t-win10-64-gpu-b/userdata rename to mozilla/worker_types/gecko-t-win10-64-gpu-b/userdata diff --git a/worker_types/gecko-t-win10-64-gpu/ami-base-name b/mozilla/worker_types/gecko-t-win10-64-gpu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-gpu/ami-base-name rename to mozilla/worker_types/gecko-t-win10-64-gpu/ami-base-name diff --git a/worker_types/gecko-t-win10-64-gpu/userdata b/mozilla/worker_types/gecko-t-win10-64-gpu/userdata similarity index 100% rename from worker_types/gecko-t-win10-64-gpu/userdata rename to mozilla/worker_types/gecko-t-win10-64-gpu/userdata diff --git a/worker_types/gecko-t-win10-64-hw/ami-base-name b/mozilla/worker_types/gecko-t-win10-64-hw/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-hw/ami-base-name rename to mozilla/worker_types/gecko-t-win10-64-hw/ami-base-name diff --git a/worker_types/gecko-t-win10-64/ami-base-name b/mozilla/worker_types/gecko-t-win10-64/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64/ami-base-name rename to mozilla/worker_types/gecko-t-win10-64/ami-base-name diff --git a/worker_types/gecko-t-win10-64/userdata b/mozilla/worker_types/gecko-t-win10-64/userdata similarity index 100% rename from worker_types/gecko-t-win10-64/userdata rename to mozilla/worker_types/gecko-t-win10-64/userdata diff --git a/worker_types/gecko-t-win7-32-beta/ami-base-name b/mozilla/worker_types/gecko-t-win7-32-beta/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-beta/ami-base-name rename to mozilla/worker_types/gecko-t-win7-32-beta/ami-base-name diff --git a/worker_types/gecko-t-win7-32-beta/userdata b/mozilla/worker_types/gecko-t-win7-32-beta/userdata similarity index 100% rename from worker_types/gecko-t-win7-32-beta/userdata rename to mozilla/worker_types/gecko-t-win7-32-beta/userdata diff --git a/worker_types/gecko-t-win7-32-cu/ami-base-name b/mozilla/worker_types/gecko-t-win7-32-cu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-cu/ami-base-name rename to mozilla/worker_types/gecko-t-win7-32-cu/ami-base-name diff --git a/worker_types/gecko-t-win7-32-cu/userdata b/mozilla/worker_types/gecko-t-win7-32-cu/userdata similarity index 100% rename from worker_types/gecko-t-win7-32-cu/userdata rename to mozilla/worker_types/gecko-t-win7-32-cu/userdata diff --git a/worker_types/gecko-t-win7-32-gpu-b/ami-base-name b/mozilla/worker_types/gecko-t-win7-32-gpu-b/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-gpu-b/ami-base-name rename to mozilla/worker_types/gecko-t-win7-32-gpu-b/ami-base-name diff --git a/worker_types/gecko-t-win7-32-gpu-b/userdata b/mozilla/worker_types/gecko-t-win7-32-gpu-b/userdata similarity index 100% rename from worker_types/gecko-t-win7-32-gpu-b/userdata rename to mozilla/worker_types/gecko-t-win7-32-gpu-b/userdata diff --git a/worker_types/gecko-t-win7-32-gpu/ami-base-name b/mozilla/worker_types/gecko-t-win7-32-gpu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-gpu/ami-base-name rename to mozilla/worker_types/gecko-t-win7-32-gpu/ami-base-name diff --git a/worker_types/gecko-t-win7-32-gpu/userdata b/mozilla/worker_types/gecko-t-win7-32-gpu/userdata similarity index 100% rename from worker_types/gecko-t-win7-32-gpu/userdata rename to mozilla/worker_types/gecko-t-win7-32-gpu/userdata diff --git a/worker_types/gecko-t-win7-32-hw/ami-base-name b/mozilla/worker_types/gecko-t-win7-32-hw/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-hw/ami-base-name rename to mozilla/worker_types/gecko-t-win7-32-hw/ami-base-name diff --git a/worker_types/gecko-t-win7-32/ami-base-name b/mozilla/worker_types/gecko-t-win7-32/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32/ami-base-name rename to mozilla/worker_types/gecko-t-win7-32/ami-base-name diff --git a/worker_types/gecko-t-win7-32/userdata b/mozilla/worker_types/gecko-t-win7-32/userdata similarity index 100% rename from worker_types/gecko-t-win7-32/userdata rename to mozilla/worker_types/gecko-t-win7-32/userdata diff --git a/worker_types/generate_occ_userdata.sh b/mozilla/worker_types/generate_occ_userdata.sh similarity index 100% rename from worker_types/generate_occ_userdata.sh rename to mozilla/worker_types/generate_occ_userdata.sh diff --git a/worker_types/nss-win2012r2-new/ami-base-name b/mozilla/worker_types/nss-win2012r2-new/ami-base-name similarity index 100% rename from worker_types/nss-win2012r2-new/ami-base-name rename to mozilla/worker_types/nss-win2012r2-new/ami-base-name diff --git a/worker_types/nss-win2012r2-new/userdata b/mozilla/worker_types/nss-win2012r2-new/userdata similarity index 100% rename from worker_types/nss-win2012r2-new/userdata rename to mozilla/worker_types/nss-win2012r2-new/userdata diff --git a/worker_types/nss-win2012r2/ami-base-name b/mozilla/worker_types/nss-win2012r2/ami-base-name similarity index 100% rename from worker_types/nss-win2012r2/ami-base-name rename to mozilla/worker_types/nss-win2012r2/ami-base-name diff --git a/worker_types/nss-win2012r2/userdata b/mozilla/worker_types/nss-win2012r2/userdata similarity index 100% rename from worker_types/nss-win2012r2/userdata rename to mozilla/worker_types/nss-win2012r2/userdata diff --git a/worker_types/process_region.sh b/mozilla/worker_types/process_region.sh similarity index 100% rename from worker_types/process_region.sh rename to mozilla/worker_types/process_region.sh diff --git a/worker_types/set_occ_ami_base_names.sh b/mozilla/worker_types/set_occ_ami_base_names.sh similarity index 100% rename from worker_types/set_occ_ami_base_names.sh rename to mozilla/worker_types/set_occ_ami_base_names.sh diff --git a/worker_types/transform.sh b/mozilla/worker_types/transform.sh similarity index 100% rename from worker_types/transform.sh rename to mozilla/worker_types/transform.sh diff --git a/worker_types/update.sh b/mozilla/worker_types/update.sh similarity index 100% rename from worker_types/update.sh rename to mozilla/worker_types/update.sh diff --git a/worker_types/win2012r2-cu/ami-base-name b/mozilla/worker_types/win2012r2-cu/ami-base-name similarity index 100% rename from worker_types/win2012r2-cu/ami-base-name rename to mozilla/worker_types/win2012r2-cu/ami-base-name diff --git a/worker_types/win2012r2-cu/userdata b/mozilla/worker_types/win2012r2-cu/userdata similarity index 100% rename from worker_types/win2012r2-cu/userdata rename to mozilla/worker_types/win2012r2-cu/userdata diff --git a/worker_types/win2012r2/ami-base-name b/mozilla/worker_types/win2012r2/ami-base-name similarity index 100% rename from worker_types/win2012r2/ami-base-name rename to mozilla/worker_types/win2012r2/ami-base-name diff --git a/worker_types/win2012r2/userdata b/mozilla/worker_types/win2012r2/userdata similarity index 100% rename from worker_types/win2012r2/userdata rename to mozilla/worker_types/win2012r2/userdata diff --git a/worker_types/win2016/ami-base-name b/mozilla/worker_types/win2016/ami-base-name similarity index 100% rename from worker_types/win2016/ami-base-name rename to mozilla/worker_types/win2016/ami-base-name diff --git a/worker_types/win2016/userdata b/mozilla/worker_types/win2016/userdata similarity index 100% rename from worker_types/win2016/userdata rename to mozilla/worker_types/win2016/userdata diff --git a/worker_types/worker_type.sh b/mozilla/worker_types/worker_type.sh similarity index 100% rename from worker_types/worker_type.sh rename to mozilla/worker_types/worker_type.sh diff --git a/all-unix-style.yml b/schemas/all-unix-style.yml similarity index 100% rename from all-unix-style.yml rename to schemas/all-unix-style.yml diff --git a/windows.yml b/schemas/windows.yml similarity index 100% rename from windows.yml rename to schemas/windows.yml diff --git a/build.cmd b/scripts/build.cmd similarity index 100% rename from build.cmd rename to scripts/build.cmd diff --git a/build.sh b/scripts/build.sh similarity index 100% rename from build.sh rename to scripts/build.sh diff --git a/clean-builds.sh b/scripts/clean-builds.sh similarity index 100% rename from clean-builds.sh rename to scripts/clean-builds.sh diff --git a/gotestcover.sh b/scripts/gotestcover.sh similarity index 100% rename from gotestcover.sh rename to scripts/gotestcover.sh diff --git a/publish-payload-schema.sh b/scripts/publish-payload-schema.sh similarity index 100% rename from publish-payload-schema.sh rename to scripts/publish-payload-schema.sh diff --git a/refresh_readme.sh b/scripts/refresh_readme.sh similarity index 100% rename from refresh_readme.sh rename to scripts/refresh_readme.sh diff --git a/release.sh b/scripts/release.sh similarity index 100% rename from release.sh rename to scripts/release.sh diff --git a/tutorial-audio.sh b/scripts/tutorial-audio.sh similarity index 100% rename from tutorial-audio.sh rename to scripts/tutorial-audio.sh From 276ef87b245a11c7c251ad5b890ef253ce61a894 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Thu, 4 Oct 2018 12:02:25 +0200 Subject: [PATCH 02/58] fixes --- cmd/inspect-worker-types/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/inspect-worker-types/main.go b/cmd/inspect-worker-types/main.go index e1f61648..7a8eea6d 100644 --- a/cmd/inspect-worker-types/main.go +++ b/cmd/inspect-worker-types/main.go @@ -47,11 +47,12 @@ func show(queue *tcqueue.Queue, t tcqueue.TaskDefinitionAndStatus) { "public/logs/live_backing.log", "public/logs/chain_of_trust.log", } { - backingLogURL, err := queue.GetLatestArtifact_SignedURL(t.Status.TaskID, artifact, time.Hour) + logURL, err := queue.GetLatestArtifact_SignedURL(t.Status.TaskID, artifact, time.Hour) if err != nil { log.Fatal(2, err) } - resp, _, err = httpbackoff.Get(backingLogURL.String()) + // log.Printf("URL: %v", logURL) + resp, _, err = httpbackoff.Get(logURL.String()) if err == nil { artifactFound = artifact break @@ -91,6 +92,8 @@ func show(queue *tcqueue.Queue, t tcqueue.TaskDefinitionAndStatus) { fmt.Println("No artifacts found!") case artifactFound == "public/logs/chain_of_trust.log": fmt.Println("scriptworker chain of trust - unknown version") + case strings.Contains(logContent, `os.environ.get('GITHUB_HEAD_REPO_URL', decision_json['payload']['env']['GITHUB_HEAD_REPO_URL'])`): + fmt.Println("scriptworker - deepspeech - unknown version") default: fmt.Println("UNKNOWN") log.Fatal(5, logContent) From fce7966a7f4d90fc184dbe2ec5fadac6481e4c12 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 14:12:14 +0200 Subject: [PATCH 03/58] Moved stuff around --- .../cmd/aws-worker-types}/main.go | 0 {cmd => aws/cmd}/gw-workers/main.go | 0 {cmd => aws/cmd}/refresh-gw-configs/main.go | 0 {cmd => aws/cmd}/update-ssl-creds/main.go | 0 .../cmd/update-worker-type/main.go | 0 .../scripts}/cleanup_ec2.sh | 0 .../scripts}/find_old_aws_objects.sh | 0 .../update-workertypes}/README.md | 0 .../update-workertypes}/delete.sh | 0 .../update-workertypes}/process_region.sh | 0 .../update-workertypes}/update.sh | 0 .../update-workertypes}/worker_type.sh | 0 .../generate_occ_userdata.sh | 0 .../OpenCloudConfig}/occ-workers/main.go | 0 .../OpenCloudConfig}/transform-occ/main.go | 0 .../transform-occ/main_test.go | 0 .../transform.sh | 0 .../waitforOCC.go | 0 mozilla/{scripts => gecko}/gecko-try.sh | 0 mozilla/{scripts => gecko}/gecko.patch | 0 mozilla/{scripts => gecko}/lib/tooltool.py | 0 mozilla/{scripts => nss}/nss-try.sh | 0 mozilla/{scripts => nss}/nss.patch | 0 .../ami-test-win2012r2/ami-base-name | 0 .../ami-test-win2012r2/userdata | 0 .../ami-test-win7sp1/ami-base-name | 0 .../ami-test-win7sp1/userdata | 0 .../gecko-1-b-win2012-beta/ami-base-name | 0 .../gecko-1-b-win2012-beta/userdata | 0 .../gecko-1-b-win2012/ami-base-name | 0 .../gecko-1-b-win2012/userdata | 0 .../gecko-2-b-win2012/ami-base-name | 0 .../gecko-2-b-win2012/userdata | 0 .../gecko-3-b-win2012/ami-base-name | 0 .../gecko-3-b-win2012/userdata | 0 .../gecko-t-win10-64-beta/ami-base-name | 0 .../gecko-t-win10-64-beta/userdata | 0 .../gecko-t-win10-64-cu/ami-base-name | 0 .../gecko-t-win10-64-cu/userdata | 0 .../gecko-t-win10-64-gpu-b/ami-base-name | 0 .../gecko-t-win10-64-gpu-b/userdata | 0 .../gecko-t-win10-64-gpu/ami-base-name | 0 .../gecko-t-win10-64-gpu/userdata | 0 .../gecko-t-win10-64-hw/ami-base-name | 0 .../gecko-t-win10-64/ami-base-name | 0 .../gecko-t-win10-64/userdata | 0 .../gecko-t-win7-32-beta/ami-base-name | 0 .../gecko-t-win7-32-beta/userdata | 0 .../gecko-t-win7-32-cu/ami-base-name | 0 .../gecko-t-win7-32-cu/userdata | 0 .../gecko-t-win7-32-gpu-b/ami-base-name | 0 .../gecko-t-win7-32-gpu-b/userdata | 0 .../gecko-t-win7-32-gpu/ami-base-name | 0 .../gecko-t-win7-32-gpu/userdata | 0 .../gecko-t-win7-32-hw/ami-base-name | 0 .../gecko-t-win7-32/ami-base-name | 0 .../gecko-t-win7-32/userdata | 0 .../nss-win2012r2-new/ami-base-name | 0 .../nss-win2012r2-new/userdata | 0 .../nss-win2012r2/ami-base-name | 0 .../nss-win2012r2/userdata | 0 .../win2012r2-cu/ami-base-name | 0 .../aws-provisioner-v1}/win2012r2-cu/userdata | 0 .../win2012r2/ami-base-name | 0 .../aws-provisioner-v1}/win2012r2/userdata | 0 .../aws-provisioner-v1}/win2016/ami-base-name | 0 .../aws-provisioner-v1}/win2016/userdata | 0 .../worker_types/set_occ_ami_base_names.sh | 70 ------------------- .../SampleArtifacts/%%%/v/X | 0 .../SampleArtifacts/_/X.txt | 0 .../SampleArtifacts/b/c/d.jpg | 0 .../config/bool-as-string.json | 0 .../config/invalid-ip.json | 0 .../config/invalid-json.json | 0 {cmd/testdata => testdata}/config/noip.json | 0 {cmd/testdata => testdata}/config/valid.json | 0 .../config/worker-type-metadata.json | 0 {cmd/testdata => testdata}/curlget.go | 0 .../machine-configuration.json | 0 .../mouse_and_screen_resolution.py | 0 {cmd/testdata => testdata}/move-file.go | 0 .../testdata => testdata}/private-opengpg-key | 0 {cmd/testdata => testdata}/public-openpgp-key | 0 {cmd/testdata => testdata}/resolvetask.go | 0 {cmd/testdata => testdata}/run-after-user.bat | 0 .../spawn-orphan-process.go | 0 .../tasks/KTBKfEgxR5GdfIIREQIvFQ.json | 0 .../tasks/LK1Rz2UtT16d-HBSqyCtuA.json | 0 {cmd/testdata => testdata}/tasks/README.md | 0 .../tasks/VESwp9JaRo-XkFN_bemBhw.json | 0 90 files changed, 70 deletions(-) rename {cmd/all-worker-types => aws/cmd/aws-worker-types}/main.go (100%) rename {cmd => aws/cmd}/gw-workers/main.go (100%) rename {cmd => aws/cmd}/refresh-gw-configs/main.go (100%) rename {cmd => aws/cmd}/update-ssl-creds/main.go (100%) rename cmd/update-worker-type/update-worker-type.go => aws/cmd/update-worker-type/main.go (100%) rename {mozilla/worker_types => aws/scripts}/cleanup_ec2.sh (100%) rename {mozilla/worker_types => aws/scripts}/find_old_aws_objects.sh (100%) rename {mozilla/worker_types => aws/update-workertypes}/README.md (100%) rename {mozilla/worker_types => aws/update-workertypes}/delete.sh (100%) rename {mozilla/worker_types => aws/update-workertypes}/process_region.sh (100%) rename {mozilla/worker_types => aws/update-workertypes}/update.sh (100%) rename {mozilla/worker_types => aws/update-workertypes}/worker_type.sh (100%) rename mozilla/{worker_types => OpenCloudConfig}/generate_occ_userdata.sh (100%) rename {cmd => mozilla/OpenCloudConfig}/occ-workers/main.go (100%) rename {cmd => mozilla/OpenCloudConfig}/transform-occ/main.go (100%) rename {cmd => mozilla/OpenCloudConfig}/transform-occ/main_test.go (100%) rename mozilla/{worker_types => OpenCloudConfig}/transform.sh (100%) rename mozilla/{scripts => OpenCloudConfig}/waitforOCC.go (100%) rename mozilla/{scripts => gecko}/gecko-try.sh (100%) rename mozilla/{scripts => gecko}/gecko.patch (100%) rename mozilla/{scripts => gecko}/lib/tooltool.py (100%) rename mozilla/{scripts => nss}/nss-try.sh (100%) rename mozilla/{scripts => nss}/nss.patch (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/ami-test-win2012r2/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/ami-test-win2012r2/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/ami-test-win7sp1/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/ami-test-win7sp1/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-1-b-win2012-beta/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-1-b-win2012-beta/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-1-b-win2012/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-1-b-win2012/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-2-b-win2012/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-2-b-win2012/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-3-b-win2012/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-3-b-win2012/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-beta/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-beta/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-cu/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-cu/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-gpu-b/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-gpu-b/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-gpu/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-gpu/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64-hw/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win10-64/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-beta/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-beta/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-cu/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-cu/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-gpu-b/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-gpu-b/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-gpu/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-gpu/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32-hw/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/gecko-t-win7-32/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/nss-win2012r2-new/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/nss-win2012r2-new/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/nss-win2012r2/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/nss-win2012r2/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/win2012r2-cu/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/win2012r2-cu/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/win2012r2/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/win2012r2/userdata (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/win2016/ami-base-name (100%) rename mozilla/{worker_types => worker-type-definitions/aws-provisioner-v1}/win2016/userdata (100%) delete mode 100755 mozilla/worker_types/set_occ_ami_base_names.sh rename {cmd/testdata => testdata}/SampleArtifacts/%%%/v/X (100%) rename {cmd/testdata => testdata}/SampleArtifacts/_/X.txt (100%) rename {cmd/testdata => testdata}/SampleArtifacts/b/c/d.jpg (100%) rename {cmd/testdata => testdata}/config/bool-as-string.json (100%) rename {cmd/testdata => testdata}/config/invalid-ip.json (100%) rename {cmd/testdata => testdata}/config/invalid-json.json (100%) rename {cmd/testdata => testdata}/config/noip.json (100%) rename {cmd/testdata => testdata}/config/valid.json (100%) rename {cmd/testdata => testdata}/config/worker-type-metadata.json (100%) rename {cmd/testdata => testdata}/curlget.go (100%) rename {cmd/testdata => testdata}/machine-configuration.json (100%) rename {cmd/testdata => testdata}/mouse_and_screen_resolution.py (100%) rename {cmd/testdata => testdata}/move-file.go (100%) rename {cmd/testdata => testdata}/private-opengpg-key (100%) rename {cmd/testdata => testdata}/public-openpgp-key (100%) rename {cmd/testdata => testdata}/resolvetask.go (100%) rename {cmd/testdata => testdata}/run-after-user.bat (100%) rename {cmd/testdata => testdata}/spawn-orphan-process.go (100%) rename {cmd/testdata => testdata}/tasks/KTBKfEgxR5GdfIIREQIvFQ.json (100%) rename {cmd/testdata => testdata}/tasks/LK1Rz2UtT16d-HBSqyCtuA.json (100%) rename {cmd/testdata => testdata}/tasks/README.md (100%) rename {cmd/testdata => testdata}/tasks/VESwp9JaRo-XkFN_bemBhw.json (100%) diff --git a/cmd/all-worker-types/main.go b/aws/cmd/aws-worker-types/main.go similarity index 100% rename from cmd/all-worker-types/main.go rename to aws/cmd/aws-worker-types/main.go diff --git a/cmd/gw-workers/main.go b/aws/cmd/gw-workers/main.go similarity index 100% rename from cmd/gw-workers/main.go rename to aws/cmd/gw-workers/main.go diff --git a/cmd/refresh-gw-configs/main.go b/aws/cmd/refresh-gw-configs/main.go similarity index 100% rename from cmd/refresh-gw-configs/main.go rename to aws/cmd/refresh-gw-configs/main.go diff --git a/cmd/update-ssl-creds/main.go b/aws/cmd/update-ssl-creds/main.go similarity index 100% rename from cmd/update-ssl-creds/main.go rename to aws/cmd/update-ssl-creds/main.go diff --git a/cmd/update-worker-type/update-worker-type.go b/aws/cmd/update-worker-type/main.go similarity index 100% rename from cmd/update-worker-type/update-worker-type.go rename to aws/cmd/update-worker-type/main.go diff --git a/mozilla/worker_types/cleanup_ec2.sh b/aws/scripts/cleanup_ec2.sh similarity index 100% rename from mozilla/worker_types/cleanup_ec2.sh rename to aws/scripts/cleanup_ec2.sh diff --git a/mozilla/worker_types/find_old_aws_objects.sh b/aws/scripts/find_old_aws_objects.sh similarity index 100% rename from mozilla/worker_types/find_old_aws_objects.sh rename to aws/scripts/find_old_aws_objects.sh diff --git a/mozilla/worker_types/README.md b/aws/update-workertypes/README.md similarity index 100% rename from mozilla/worker_types/README.md rename to aws/update-workertypes/README.md diff --git a/mozilla/worker_types/delete.sh b/aws/update-workertypes/delete.sh similarity index 100% rename from mozilla/worker_types/delete.sh rename to aws/update-workertypes/delete.sh diff --git a/mozilla/worker_types/process_region.sh b/aws/update-workertypes/process_region.sh similarity index 100% rename from mozilla/worker_types/process_region.sh rename to aws/update-workertypes/process_region.sh diff --git a/mozilla/worker_types/update.sh b/aws/update-workertypes/update.sh similarity index 100% rename from mozilla/worker_types/update.sh rename to aws/update-workertypes/update.sh diff --git a/mozilla/worker_types/worker_type.sh b/aws/update-workertypes/worker_type.sh similarity index 100% rename from mozilla/worker_types/worker_type.sh rename to aws/update-workertypes/worker_type.sh diff --git a/mozilla/worker_types/generate_occ_userdata.sh b/mozilla/OpenCloudConfig/generate_occ_userdata.sh similarity index 100% rename from mozilla/worker_types/generate_occ_userdata.sh rename to mozilla/OpenCloudConfig/generate_occ_userdata.sh diff --git a/cmd/occ-workers/main.go b/mozilla/OpenCloudConfig/occ-workers/main.go similarity index 100% rename from cmd/occ-workers/main.go rename to mozilla/OpenCloudConfig/occ-workers/main.go diff --git a/cmd/transform-occ/main.go b/mozilla/OpenCloudConfig/transform-occ/main.go similarity index 100% rename from cmd/transform-occ/main.go rename to mozilla/OpenCloudConfig/transform-occ/main.go diff --git a/cmd/transform-occ/main_test.go b/mozilla/OpenCloudConfig/transform-occ/main_test.go similarity index 100% rename from cmd/transform-occ/main_test.go rename to mozilla/OpenCloudConfig/transform-occ/main_test.go diff --git a/mozilla/worker_types/transform.sh b/mozilla/OpenCloudConfig/transform.sh similarity index 100% rename from mozilla/worker_types/transform.sh rename to mozilla/OpenCloudConfig/transform.sh diff --git a/mozilla/scripts/waitforOCC.go b/mozilla/OpenCloudConfig/waitforOCC.go similarity index 100% rename from mozilla/scripts/waitforOCC.go rename to mozilla/OpenCloudConfig/waitforOCC.go diff --git a/mozilla/scripts/gecko-try.sh b/mozilla/gecko/gecko-try.sh similarity index 100% rename from mozilla/scripts/gecko-try.sh rename to mozilla/gecko/gecko-try.sh diff --git a/mozilla/scripts/gecko.patch b/mozilla/gecko/gecko.patch similarity index 100% rename from mozilla/scripts/gecko.patch rename to mozilla/gecko/gecko.patch diff --git a/mozilla/scripts/lib/tooltool.py b/mozilla/gecko/lib/tooltool.py similarity index 100% rename from mozilla/scripts/lib/tooltool.py rename to mozilla/gecko/lib/tooltool.py diff --git a/mozilla/scripts/nss-try.sh b/mozilla/nss/nss-try.sh similarity index 100% rename from mozilla/scripts/nss-try.sh rename to mozilla/nss/nss-try.sh diff --git a/mozilla/scripts/nss.patch b/mozilla/nss/nss.patch similarity index 100% rename from mozilla/scripts/nss.patch rename to mozilla/nss/nss.patch diff --git a/mozilla/worker_types/ami-test-win2012r2/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name similarity index 100% rename from mozilla/worker_types/ami-test-win2012r2/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name diff --git a/mozilla/worker_types/ami-test-win2012r2/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata similarity index 100% rename from mozilla/worker_types/ami-test-win2012r2/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata diff --git a/mozilla/worker_types/ami-test-win7sp1/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name similarity index 100% rename from mozilla/worker_types/ami-test-win7sp1/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name diff --git a/mozilla/worker_types/ami-test-win7sp1/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata similarity index 100% rename from mozilla/worker_types/ami-test-win7sp1/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata diff --git a/mozilla/worker_types/gecko-1-b-win2012-beta/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-1-b-win2012-beta/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name diff --git a/mozilla/worker_types/gecko-1-b-win2012-beta/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata similarity index 100% rename from mozilla/worker_types/gecko-1-b-win2012-beta/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata diff --git a/mozilla/worker_types/gecko-1-b-win2012/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-1-b-win2012/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name diff --git a/mozilla/worker_types/gecko-1-b-win2012/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata similarity index 100% rename from mozilla/worker_types/gecko-1-b-win2012/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata diff --git a/mozilla/worker_types/gecko-2-b-win2012/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-2-b-win2012/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name diff --git a/mozilla/worker_types/gecko-2-b-win2012/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata similarity index 100% rename from mozilla/worker_types/gecko-2-b-win2012/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata diff --git a/mozilla/worker_types/gecko-3-b-win2012/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-3-b-win2012/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name diff --git a/mozilla/worker_types/gecko-3-b-win2012/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata similarity index 100% rename from mozilla/worker_types/gecko-3-b-win2012/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata diff --git a/mozilla/worker_types/gecko-t-win10-64-beta/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-beta/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win10-64-beta/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-beta/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata diff --git a/mozilla/worker_types/gecko-t-win10-64-cu/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-cu/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win10-64-cu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-cu/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata diff --git a/mozilla/worker_types/gecko-t-win10-64-gpu-b/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-gpu-b/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win10-64-gpu-b/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-gpu-b/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata diff --git a/mozilla/worker_types/gecko-t-win10-64-gpu/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-gpu/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win10-64-gpu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-gpu/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata diff --git a/mozilla/worker_types/gecko-t-win10-64-hw/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64-hw/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win10-64/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win10-64/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win10-64/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata diff --git a/mozilla/worker_types/gecko-t-win7-32-beta/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-beta/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win7-32-beta/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-beta/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata diff --git a/mozilla/worker_types/gecko-t-win7-32-cu/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-cu/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win7-32-cu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-cu/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata diff --git a/mozilla/worker_types/gecko-t-win7-32-gpu-b/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-gpu-b/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win7-32-gpu-b/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-gpu-b/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata diff --git a/mozilla/worker_types/gecko-t-win7-32-gpu/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-gpu/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win7-32-gpu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-gpu/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata diff --git a/mozilla/worker_types/gecko-t-win7-32-hw/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32-hw/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win7-32/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name diff --git a/mozilla/worker_types/gecko-t-win7-32/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata similarity index 100% rename from mozilla/worker_types/gecko-t-win7-32/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata diff --git a/mozilla/worker_types/nss-win2012r2-new/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name similarity index 100% rename from mozilla/worker_types/nss-win2012r2-new/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name diff --git a/mozilla/worker_types/nss-win2012r2-new/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata similarity index 100% rename from mozilla/worker_types/nss-win2012r2-new/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata diff --git a/mozilla/worker_types/nss-win2012r2/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name similarity index 100% rename from mozilla/worker_types/nss-win2012r2/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name diff --git a/mozilla/worker_types/nss-win2012r2/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/userdata similarity index 100% rename from mozilla/worker_types/nss-win2012r2/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/userdata diff --git a/mozilla/worker_types/win2012r2-cu/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name similarity index 100% rename from mozilla/worker_types/win2012r2-cu/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name diff --git a/mozilla/worker_types/win2012r2-cu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/userdata similarity index 100% rename from mozilla/worker_types/win2012r2-cu/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/userdata diff --git a/mozilla/worker_types/win2012r2/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/ami-base-name similarity index 100% rename from mozilla/worker_types/win2012r2/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/ami-base-name diff --git a/mozilla/worker_types/win2012r2/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/userdata similarity index 100% rename from mozilla/worker_types/win2012r2/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/userdata diff --git a/mozilla/worker_types/win2016/ami-base-name b/mozilla/worker-type-definitions/aws-provisioner-v1/win2016/ami-base-name similarity index 100% rename from mozilla/worker_types/win2016/ami-base-name rename to mozilla/worker-type-definitions/aws-provisioner-v1/win2016/ami-base-name diff --git a/mozilla/worker_types/win2016/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/win2016/userdata similarity index 100% rename from mozilla/worker_types/win2016/userdata rename to mozilla/worker-type-definitions/aws-provisioner-v1/win2016/userdata diff --git a/mozilla/worker_types/set_occ_ami_base_names.sh b/mozilla/worker_types/set_occ_ami_base_names.sh deleted file mode 100755 index 38b5a183..00000000 --- a/mozilla/worker_types/set_occ_ami_base_names.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -cd "$(dirname "${0}")" -aws_region=us-west-2 - -curl -L 'https://github.com/mozilla-releng/OpenCloudConfig/tree/master/userdata/Manifest' 2>/dev/null | sed -n 's/.*\(gecko[^.]*\)\.json.*/\1/p' | sort -u | while read tc_worker_type -do - case "${tc_worker_type}" in - gecko-t-win7-32-gpu*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win7-32-base-20170905'} - aws_instance_type=${aws_instance_type:='g2.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko test worker for Windows 7 32 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=root - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":30,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdc","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-t-win7-32*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win7-32-base-20170905'} - aws_instance_type=${aws_instance_type:='c4.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko test worker for Windows 7 32 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=root - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":30,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdc","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-t-win10-64-gpu*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win10-64-gpu-base-20170921'} - aws_instance_type=${aws_instance_type:='g2.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko tester for Windows 10 64 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=Administrator - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-t-win10-64*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win10-64-base-20170905'} - aws_instance_type=${aws_instance_type:='c4.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko tester for Windows 10 64 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=Administrator - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-[123]-b-win2012*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-b-win2012-base-*'} - aws_instance_type=${aws_instance_type:='c4.4xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko builder for Windows; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=Administrator - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":40,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - *) - echo "ERROR: unknown worker type: '${tc_worker_type}'" - exit 67 - ;; - esac - echo "${aws_base_ami_search_term}" > "${tc_worker_type}/ami-base-name" -done diff --git a/cmd/testdata/SampleArtifacts/%%%/v/X b/testdata/SampleArtifacts/%%%/v/X similarity index 100% rename from cmd/testdata/SampleArtifacts/%%%/v/X rename to testdata/SampleArtifacts/%%%/v/X diff --git a/cmd/testdata/SampleArtifacts/_/X.txt b/testdata/SampleArtifacts/_/X.txt similarity index 100% rename from cmd/testdata/SampleArtifacts/_/X.txt rename to testdata/SampleArtifacts/_/X.txt diff --git a/cmd/testdata/SampleArtifacts/b/c/d.jpg b/testdata/SampleArtifacts/b/c/d.jpg similarity index 100% rename from cmd/testdata/SampleArtifacts/b/c/d.jpg rename to testdata/SampleArtifacts/b/c/d.jpg diff --git a/cmd/testdata/config/bool-as-string.json b/testdata/config/bool-as-string.json similarity index 100% rename from cmd/testdata/config/bool-as-string.json rename to testdata/config/bool-as-string.json diff --git a/cmd/testdata/config/invalid-ip.json b/testdata/config/invalid-ip.json similarity index 100% rename from cmd/testdata/config/invalid-ip.json rename to testdata/config/invalid-ip.json diff --git a/cmd/testdata/config/invalid-json.json b/testdata/config/invalid-json.json similarity index 100% rename from cmd/testdata/config/invalid-json.json rename to testdata/config/invalid-json.json diff --git a/cmd/testdata/config/noip.json b/testdata/config/noip.json similarity index 100% rename from cmd/testdata/config/noip.json rename to testdata/config/noip.json diff --git a/cmd/testdata/config/valid.json b/testdata/config/valid.json similarity index 100% rename from cmd/testdata/config/valid.json rename to testdata/config/valid.json diff --git a/cmd/testdata/config/worker-type-metadata.json b/testdata/config/worker-type-metadata.json similarity index 100% rename from cmd/testdata/config/worker-type-metadata.json rename to testdata/config/worker-type-metadata.json diff --git a/cmd/testdata/curlget.go b/testdata/curlget.go similarity index 100% rename from cmd/testdata/curlget.go rename to testdata/curlget.go diff --git a/cmd/testdata/machine-configuration.json b/testdata/machine-configuration.json similarity index 100% rename from cmd/testdata/machine-configuration.json rename to testdata/machine-configuration.json diff --git a/cmd/testdata/mouse_and_screen_resolution.py b/testdata/mouse_and_screen_resolution.py similarity index 100% rename from cmd/testdata/mouse_and_screen_resolution.py rename to testdata/mouse_and_screen_resolution.py diff --git a/cmd/testdata/move-file.go b/testdata/move-file.go similarity index 100% rename from cmd/testdata/move-file.go rename to testdata/move-file.go diff --git a/cmd/testdata/private-opengpg-key b/testdata/private-opengpg-key similarity index 100% rename from cmd/testdata/private-opengpg-key rename to testdata/private-opengpg-key diff --git a/cmd/testdata/public-openpgp-key b/testdata/public-openpgp-key similarity index 100% rename from cmd/testdata/public-openpgp-key rename to testdata/public-openpgp-key diff --git a/cmd/testdata/resolvetask.go b/testdata/resolvetask.go similarity index 100% rename from cmd/testdata/resolvetask.go rename to testdata/resolvetask.go diff --git a/cmd/testdata/run-after-user.bat b/testdata/run-after-user.bat similarity index 100% rename from cmd/testdata/run-after-user.bat rename to testdata/run-after-user.bat diff --git a/cmd/testdata/spawn-orphan-process.go b/testdata/spawn-orphan-process.go similarity index 100% rename from cmd/testdata/spawn-orphan-process.go rename to testdata/spawn-orphan-process.go diff --git a/cmd/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json b/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json similarity index 100% rename from cmd/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json rename to testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json diff --git a/cmd/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json b/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json similarity index 100% rename from cmd/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json rename to testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json diff --git a/cmd/testdata/tasks/README.md b/testdata/tasks/README.md similarity index 100% rename from cmd/testdata/tasks/README.md rename to testdata/tasks/README.md diff --git a/cmd/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json b/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json similarity index 100% rename from cmd/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json rename to testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json From cb38464825534e4def5f107c0b814eb0d3729d20 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 14:12:46 +0200 Subject: [PATCH 04/58] cleanup --- aws/scripts/cleanup_ec2.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/scripts/cleanup_ec2.sh b/aws/scripts/cleanup_ec2.sh index 68f7363c..9eee6e4d 100755 --- a/aws/scripts/cleanup_ec2.sh +++ b/aws/scripts/cleanup_ec2.sh @@ -1,8 +1,8 @@ -#!/bin/bash +#!/bin/bash -e cd "$(dirname "${0}")" -go run all-worker-types/main.go -ALL_REFERENCED_AMIS="$(cat worker_type_definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u)" +go get github.com/taskcluster/generic-worker/cmd/all-worker-types +"$GOPATH/bin/all-worker-types" > /dev/null +cat worker_type_definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u rm -r worker_type_definitions -echo "All referenced amis: ${ALL_REFERENCED_AMIS}" From 0033a62f962460170d45a9650f7cd601e51249bf Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 14:35:24 +0200 Subject: [PATCH 05/58] Fixed git import paths --- cmd/generic-worker/aws.go | 2 +- cmd/generic-worker/chain_of_trust.go | 2 +- cmd/generic-worker/chain_of_trust_all-unix-style.go | 2 +- cmd/generic-worker/chain_of_trust_windows.go | 2 +- cmd/generic-worker/config_test.go | 2 +- cmd/generic-worker/helper_test.go | 2 +- cmd/generic-worker/livelog.go | 4 ++-- cmd/generic-worker/main.go | 4 ++-- cmd/generic-worker/model.go | 2 +- cmd/generic-worker/mounts.go | 2 +- cmd/generic-worker/mounts_test.go | 2 +- cmd/generic-worker/plat_all-unix-style.go | 2 +- cmd/generic-worker/plat_windows.go | 6 +++--- cmd/generic-worker/rdp_windows.go | 2 +- cmd/generic-worker/runasadministrator_windows.go | 2 +- cmd/generic-worker/supersede.go | 2 +- cmd/generic-worker/taskcluster_proxy.go | 2 +- lib/gwconfig/config.go | 2 +- lib/process/logininfo_windows.go | 2 +- lib/process/process_windows.go | 4 ++-- lib/win32/win32_windows_test.go | 2 +- .../OpenCloudConfig}/refresh-gw-configs/main.go | 2 +- 22 files changed, 27 insertions(+), 27 deletions(-) rename {aws/cmd => mozilla/OpenCloudConfig}/refresh-gw-configs/main.go (98%) diff --git a/cmd/generic-worker/aws.go b/cmd/generic-worker/aws.go index 83b2c296..34d16d0d 100644 --- a/cmd/generic-worker/aws.go +++ b/cmd/generic-worker/aws.go @@ -16,7 +16,7 @@ import ( "strings" "time" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/taskcluster-client-go/tcawsprovisioner" ) diff --git a/cmd/generic-worker/chain_of_trust.go b/cmd/generic-worker/chain_of_trust.go index d6ec565a..bac5e46c 100644 --- a/cmd/generic-worker/chain_of_trust.go +++ b/cmd/generic-worker/chain_of_trust.go @@ -12,7 +12,7 @@ import ( "golang.org/x/crypto/openpgp/clearsign" "golang.org/x/crypto/openpgp/packet" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/taskcluster-base-go/scopes" "github.com/taskcluster/taskcluster-client-go/tcqueue" ) diff --git a/cmd/generic-worker/chain_of_trust_all-unix-style.go b/cmd/generic-worker/chain_of_trust_all-unix-style.go index ee371da3..48d26af5 100644 --- a/cmd/generic-worker/chain_of_trust_all-unix-style.go +++ b/cmd/generic-worker/chain_of_trust_all-unix-style.go @@ -8,7 +8,7 @@ import ( "os/user" "strconv" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" ) func (cot *ChainOfTrustTaskFeature) ensureTaskUserCantReadPrivateCotKey() error { diff --git a/cmd/generic-worker/chain_of_trust_windows.go b/cmd/generic-worker/chain_of_trust_windows.go index 155513cd..7704559a 100644 --- a/cmd/generic-worker/chain_of_trust_windows.go +++ b/cmd/generic-worker/chain_of_trust_windows.go @@ -7,7 +7,7 @@ import ( "golang.org/x/sys/windows" acl "github.com/hectane/go-acl" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" ) func (cot *ChainOfTrustTaskFeature) ensureTaskUserCantReadPrivateCotKey() error { diff --git a/cmd/generic-worker/config_test.go b/cmd/generic-worker/config_test.go index 786c9e38..8f529407 100644 --- a/cmd/generic-worker/config_test.go +++ b/cmd/generic-worker/config_test.go @@ -7,7 +7,7 @@ import ( "runtime" "testing" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" ) func TestMissingIPConfig(t *testing.T) { diff --git a/cmd/generic-worker/helper_test.go b/cmd/generic-worker/helper_test.go index 52313664..4350271a 100644 --- a/cmd/generic-worker/helper_test.go +++ b/cmd/generic-worker/helper_test.go @@ -19,7 +19,7 @@ import ( "testing" "time" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/slugid-go/slugid" tcclient "github.com/taskcluster/taskcluster-client-go" diff --git a/cmd/generic-worker/livelog.go b/cmd/generic-worker/livelog.go index 0447f826..aa6115bc 100644 --- a/cmd/generic-worker/livelog.go +++ b/cmd/generic-worker/livelog.go @@ -9,8 +9,8 @@ import ( "strconv" "time" - "github.com/taskcluster/generic-worker/livelog" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/livelog" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/stateless-dns-go/hostname" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" diff --git a/cmd/generic-worker/main.go b/cmd/generic-worker/main.go index a459f975..26a63acc 100644 --- a/cmd/generic-worker/main.go +++ b/cmd/generic-worker/main.go @@ -19,8 +19,8 @@ import ( "time" docopt "github.com/docopt/docopt-go" - "github.com/taskcluster/generic-worker/gwconfig" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/gwconfig" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" "github.com/taskcluster/taskcluster-client-go/tcauth" diff --git a/cmd/generic-worker/model.go b/cmd/generic-worker/model.go index 558fd98d..e52672d1 100644 --- a/cmd/generic-worker/model.go +++ b/cmd/generic-worker/model.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/taskcluster-client-go/tcqueue" ) diff --git a/cmd/generic-worker/mounts.go b/cmd/generic-worker/mounts.go index 56d21027..c423c7b7 100644 --- a/cmd/generic-worker/mounts.go +++ b/cmd/generic-worker/mounts.go @@ -15,7 +15,7 @@ import ( "time" "github.com/mholt/archiver" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/slugid-go/slugid" "github.com/taskcluster/taskcluster-base-go/scopes" diff --git a/cmd/generic-worker/mounts_test.go b/cmd/generic-worker/mounts_test.go index d56c968d..04d7e153 100644 --- a/cmd/generic-worker/mounts_test.go +++ b/cmd/generic-worker/mounts_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" ) func TestMounts(t *testing.T) { diff --git a/cmd/generic-worker/plat_all-unix-style.go b/cmd/generic-worker/plat_all-unix-style.go index be0abc97..4bde3373 100644 --- a/cmd/generic-worker/plat_all-unix-style.go +++ b/cmd/generic-worker/plat_all-unix-style.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/shell" ) diff --git a/cmd/generic-worker/plat_windows.go b/cmd/generic-worker/plat_windows.go index 25839b70..533cac82 100644 --- a/cmd/generic-worker/plat_windows.go +++ b/cmd/generic-worker/plat_windows.go @@ -16,9 +16,9 @@ import ( "unsafe" "github.com/dchest/uniuri" - "github.com/taskcluster/generic-worker/process" - "github.com/taskcluster/generic-worker/runtime" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/process" + "github.com/taskcluster/generic-worker/lib/runtime" + "github.com/taskcluster/generic-worker/lib/win32" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" ) diff --git a/cmd/generic-worker/rdp_windows.go b/cmd/generic-worker/rdp_windows.go index c6c385bf..d0b06178 100644 --- a/cmd/generic-worker/rdp_windows.go +++ b/cmd/generic-worker/rdp_windows.go @@ -5,7 +5,7 @@ import ( "path/filepath" "time" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" ) diff --git a/cmd/generic-worker/runasadministrator_windows.go b/cmd/generic-worker/runasadministrator_windows.go index 423dec88..fefd5ac0 100644 --- a/cmd/generic-worker/runasadministrator_windows.go +++ b/cmd/generic-worker/runasadministrator_windows.go @@ -3,7 +3,7 @@ package main import ( "fmt" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/win32" "github.com/taskcluster/taskcluster-base-go/scopes" ) diff --git a/cmd/generic-worker/supersede.go b/cmd/generic-worker/supersede.go index af676244..6d718330 100644 --- a/cmd/generic-worker/supersede.go +++ b/cmd/generic-worker/supersede.go @@ -5,7 +5,7 @@ import ( "fmt" "path/filepath" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/taskcluster-base-go/scopes" ) diff --git a/cmd/generic-worker/taskcluster_proxy.go b/cmd/generic-worker/taskcluster_proxy.go index b4d6ac09..e3cb8bad 100644 --- a/cmd/generic-worker/taskcluster_proxy.go +++ b/cmd/generic-worker/taskcluster_proxy.go @@ -7,7 +7,7 @@ import ( "log" "net/http" - "github.com/taskcluster/generic-worker/tcproxy" + "github.com/taskcluster/generic-worker/lib/tcproxy" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" ) diff --git a/lib/gwconfig/config.go b/lib/gwconfig/config.go index c246a333..6157d91b 100644 --- a/lib/gwconfig/config.go +++ b/lib/gwconfig/config.go @@ -8,7 +8,7 @@ import ( "reflect" "runtime" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" ) type ( diff --git a/lib/process/logininfo_windows.go b/lib/process/logininfo_windows.go index f70464b0..344c1b54 100644 --- a/lib/process/logininfo_windows.go +++ b/lib/process/logininfo_windows.go @@ -7,7 +7,7 @@ import ( "time" "unsafe" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/win32" ) // LoginInfo represents a logged in user session diff --git a/lib/process/process_windows.go b/lib/process/process_windows.go index ea9a52dc..5b6f729b 100644 --- a/lib/process/process_windows.go +++ b/lib/process/process_windows.go @@ -10,8 +10,8 @@ import ( "syscall" "time" - "github.com/taskcluster/generic-worker/runtime" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/runtime" + "github.com/taskcluster/generic-worker/lib/win32" ) type Command struct { diff --git a/lib/win32/win32_windows_test.go b/lib/win32/win32_windows_test.go index c7ef4ed7..9c816ed6 100644 --- a/lib/win32/win32_windows_test.go +++ b/lib/win32/win32_windows_test.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/win32" ) func ExampleMergeEnvLists() { diff --git a/aws/cmd/refresh-gw-configs/main.go b/mozilla/OpenCloudConfig/refresh-gw-configs/main.go similarity index 98% rename from aws/cmd/refresh-gw-configs/main.go rename to mozilla/OpenCloudConfig/refresh-gw-configs/main.go index c608f158..50538860 100644 --- a/aws/cmd/refresh-gw-configs/main.go +++ b/mozilla/OpenCloudConfig/refresh-gw-configs/main.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" "github.com/taskcluster/taskcluster-client-go/tcawsprovisioner" ) From bbbdaf0a0208e5a7c50b098216bb131a572a88a4 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 14:40:31 +0200 Subject: [PATCH 06/58] Fixed .taskcluster.yml --- .taskcluster.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 1ef60ad2..8534b03c 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -86,9 +86,9 @@ tasks: - 'git clean -fdx' - 'git checkout -B tmp -t "X%TASK_ID%"' - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen + - cd cmd\gw-codegen - go get -v -u - - cd .. + - cd ..\.. - go generate - | :: this counts the number of lines returned by git status @@ -168,9 +168,9 @@ tasks: - 'git clean -fdx' - 'git checkout -B tmp -t "X%TASK_ID%"' - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen + - cd cmd\gw-codegen - go get -v -u - - cd .. + - cd ..\.. - go generate - set revision={{ event.head.sha }} - go get -ldflags "-X main.revision=%revision%" -v -u -t ./... @@ -239,9 +239,9 @@ tasks: - 'git clean -fdx' - 'git checkout -B tmp -t "X%TASK_ID%"' - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen + - cd cmd\gw-codegen - go get -v -u - - cd .. + - cd ..\.. - go generate - | :: this counts the number of lines returned by git status @@ -324,9 +324,9 @@ tasks: git clean -fdx git checkout -B tmp -t "X${TASK_ID}" go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen + cd cmd/gw-codegen go get -v -u - cd .. + cd ../.. go generate go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # output of wc command can contain spaces on darwin, so no quotes around expression @@ -397,9 +397,9 @@ tasks: # git clean -fdx # git checkout -B tmp -t "X${TASK_ID}" # go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - # cd gw-codegen + # cd cmd/gw-codegen # go get -v -u - # cd .. + # cd ../.. # go generate # go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # test "$(git status --porcelain | wc -l)" == 0 @@ -464,9 +464,9 @@ tasks: git clean -fdx git checkout -B tmp -t "X${TASK_ID}" go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen + cd cmd/gw-codegen go get -v -u - cd .. + cd ../.. go generate go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... test "$(git status --porcelain | wc -l)" == 0 From 39b6e7c20da05181cf5ab2c08940f7e5b3846937 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 14:51:19 +0200 Subject: [PATCH 07/58] fixes --- .taskcluster.yml | 12 ++++++------ .travis.yml | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 8534b03c..646c5d07 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -89,7 +89,7 @@ tasks: - cd cmd\gw-codegen - go get -v -u - cd ..\.. - - go generate + - go generate ./... - | :: this counts the number of lines returned by git status :: dump temp file a directory higher, otherwise git status reports the tmp1.txt file! @@ -171,7 +171,7 @@ tasks: - cd cmd\gw-codegen - go get -v -u - cd ..\.. - - go generate + - go generate ./... - set revision={{ event.head.sha }} - go get -ldflags "-X main.revision=%revision%" -v -u -t ./... - set GORACE=history_size=7 @@ -242,7 +242,7 @@ tasks: - cd cmd\gw-codegen - go get -v -u - cd ..\.. - - go generate + - go generate ./... - | :: this counts the number of lines returned by git status :: dump temp file a directory higher, otherwise git status reports the tmp1.txt file! @@ -327,7 +327,7 @@ tasks: cd cmd/gw-codegen go get -v -u cd ../.. - go generate + go generate ./... go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # output of wc command can contain spaces on darwin, so no quotes around expression test $(git status --porcelain | wc -l) == 0 @@ -400,7 +400,7 @@ tasks: # cd cmd/gw-codegen # go get -v -u # cd ../.. - # go generate + # go generate ./... # go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # test "$(git status --porcelain | wc -l)" == 0 # GORACE=history_size=7 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v ./... @@ -467,7 +467,7 @@ tasks: cd cmd/gw-codegen go get -v -u cd ../.. - go generate + go generate ./... go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... test "$(git status --porcelain | wc -l)" == 0 GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... diff --git a/.travis.yml b/.travis.yml index 656cca7c..7ee78980 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ env: - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=386" - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=amd64" - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=arm" - # currently broken + # currently broken due to upstream issue # - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=arm64" - "CGO_ENABLED=0 GIMME_OS=darwin GIMME_ARCH=amd64" - "CGO_ENABLED=0 GIMME_OS=windows GIMME_ARCH=amd64" @@ -16,12 +16,12 @@ install: - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go get github.com/mattn/goveralls github.com/taskcluster/taskcluster-proxy github.com/taskcluster/livelog github.com/gordonklaus/ineffassign" script: - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go get ./gw-codegen" - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go generate" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go get ./cmd/gw-codegen" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go generate ./..." - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || git status" - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || test $(git status --porcelain | wc -l) == 0" - "go get -ldflags \"-X main.revision=$(git rev-parse HEAD)\" -v -t ./..." - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || GORACE=history_size=7 travis_wait 30 ./gotestcover.sh coverage.report" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || GORACE=history_size=7 travis_wait 30 scripts/gotestcover.sh coverage.report" - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || ${GOPATH}/bin/ineffassign ." after_script: From e65ef8eeb0c2473b6d1681cc3304df2d76a97392 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 14:55:53 +0200 Subject: [PATCH 08/58] fixed build scripts --- scripts/build.cmd | 10 +++++----- scripts/build.sh | 10 ++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/scripts/build.cmd b/scripts/build.cmd index 5e449666..1feda1b7 100644 --- a/scripts/build.cmd +++ b/scripts/build.cmd @@ -2,14 +2,14 @@ @echo on -:: cd to dir containing this script -pushd %~dp0 +:: cd to parent dir of the dir of this script +pushd %~dp0\.. go get github.com/taskcluster/livelog github.com/gordonklaus/ineffassign || exit /b %ERRORLEVEL% -cd gw-codegen +cd cmd\gw-codegen go get -v || exit /b %ERRORLEVEL% -cd .. -go generate || exit /b %ERRORLEVEL% +cd ..\.. +go generate ./... || exit /b %ERRORLEVEL% go get -v -t ./... || exit /b %ERRORLEVEL% :: this counts the number of lines returned by git status diff --git a/scripts/build.sh b/scripts/build.sh index 36ef5ac1..8780affd 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -cd "$(dirname "${0}")" +cd "$(dirname "${0}")"/.. # Support go 1 release 1.9 or higher. Let's not move this to 1.10 until # https://bugzil.la/1441889 is resolved, and travis-ci.org works correctly with @@ -42,7 +42,7 @@ done echo "${OUTPUT_ALL_PLATFORMS}" echo "${OUTPUT_TEST}" -go get github.com/taskcluster/generic-worker/gw-codegen +go get github.com/taskcluster/generic-worker/cmd/gw-codegen export PATH="$(go env GOPATH)/bin:${PATH}" go generate ./... @@ -51,14 +51,12 @@ function install { GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v ./... # GOOS="${1}" GOARCH="${2}" go vet ./... # note, this just builds tests, it doesn't run them! - GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker - GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/livelog + GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/... else CGO_ENABLED=0 go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v ./... go vet ./... # note, this just builds tests, it doesn't run them! - CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker - CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/livelog + CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/... fi } From 807f1110a0133377cd89693307a3acc212e1d95c Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 14:59:43 +0200 Subject: [PATCH 09/58] fixed path for code generation --- cmd/generic-worker/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/generic-worker/main.go b/cmd/generic-worker/main.go index 26a63acc..78808ffb 100644 --- a/cmd/generic-worker/main.go +++ b/cmd/generic-worker/main.go @@ -1,5 +1,5 @@ -//go:generate gw-codegen all-unix-style.yml generated_all-unix-style.go !windows -//go:generate gw-codegen windows.yml generated_windows.go +//go:generate gw-codegen ../../schemas/all-unix-style.yml generated_all-unix-style.go !windows +//go:generate gw-codegen ../../schemas/windows.yml generated_windows.go package main From 033f337a193ad75e6b5377a780423da534cb41e6 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 15:06:42 +0200 Subject: [PATCH 10/58] test all packages --- scripts/build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 8780affd..541cc407 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -51,12 +51,16 @@ function install { GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v ./... # GOOS="${1}" GOARCH="${2}" go vet ./... # note, this just builds tests, it doesn't run them! - GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/... + go list ./... | while read PACKAGE; do + GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c "${PACKAGE}" + done else CGO_ENABLED=0 go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v ./... go vet ./... # note, this just builds tests, it doesn't run them! - CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/... + go list ./... | while read PACKAGE; do + CGO_ENABLED=0 go test -c "${PACKAGE}" + done fi } From 8c6be8545838bff482fdb01d0f51388ef056a19e Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 15:16:08 +0200 Subject: [PATCH 11/58] moved testdata into generic-worker package folder --- {testdata => cmd/generic-worker/testdata}/SampleArtifacts/%%%/v/X | 0 {testdata => cmd/generic-worker/testdata}/SampleArtifacts/_/X.txt | 0 .../generic-worker/testdata}/SampleArtifacts/b/c/d.jpg | 0 .../generic-worker/testdata}/config/bool-as-string.json | 0 {testdata => cmd/generic-worker/testdata}/config/invalid-ip.json | 0 .../generic-worker/testdata}/config/invalid-json.json | 0 {testdata => cmd/generic-worker/testdata}/config/noip.json | 0 {testdata => cmd/generic-worker/testdata}/config/valid.json | 0 .../generic-worker/testdata}/config/worker-type-metadata.json | 0 {testdata => cmd/generic-worker/testdata}/curlget.go | 0 .../generic-worker/testdata}/machine-configuration.json | 0 .../generic-worker/testdata}/mouse_and_screen_resolution.py | 0 {testdata => cmd/generic-worker/testdata}/move-file.go | 0 {testdata => cmd/generic-worker/testdata}/private-opengpg-key | 0 {testdata => cmd/generic-worker/testdata}/public-openpgp-key | 0 {testdata => cmd/generic-worker/testdata}/resolvetask.go | 0 {testdata => cmd/generic-worker/testdata}/run-after-user.bat | 0 {testdata => cmd/generic-worker/testdata}/spawn-orphan-process.go | 0 .../generic-worker/testdata}/tasks/KTBKfEgxR5GdfIIREQIvFQ.json | 0 .../generic-worker/testdata}/tasks/LK1Rz2UtT16d-HBSqyCtuA.json | 0 {testdata => cmd/generic-worker/testdata}/tasks/README.md | 0 .../generic-worker/testdata}/tasks/VESwp9JaRo-XkFN_bemBhw.json | 0 22 files changed, 0 insertions(+), 0 deletions(-) rename {testdata => cmd/generic-worker/testdata}/SampleArtifacts/%%%/v/X (100%) rename {testdata => cmd/generic-worker/testdata}/SampleArtifacts/_/X.txt (100%) rename {testdata => cmd/generic-worker/testdata}/SampleArtifacts/b/c/d.jpg (100%) rename {testdata => cmd/generic-worker/testdata}/config/bool-as-string.json (100%) rename {testdata => cmd/generic-worker/testdata}/config/invalid-ip.json (100%) rename {testdata => cmd/generic-worker/testdata}/config/invalid-json.json (100%) rename {testdata => cmd/generic-worker/testdata}/config/noip.json (100%) rename {testdata => cmd/generic-worker/testdata}/config/valid.json (100%) rename {testdata => cmd/generic-worker/testdata}/config/worker-type-metadata.json (100%) rename {testdata => cmd/generic-worker/testdata}/curlget.go (100%) rename {testdata => cmd/generic-worker/testdata}/machine-configuration.json (100%) rename {testdata => cmd/generic-worker/testdata}/mouse_and_screen_resolution.py (100%) rename {testdata => cmd/generic-worker/testdata}/move-file.go (100%) rename {testdata => cmd/generic-worker/testdata}/private-opengpg-key (100%) rename {testdata => cmd/generic-worker/testdata}/public-openpgp-key (100%) rename {testdata => cmd/generic-worker/testdata}/resolvetask.go (100%) rename {testdata => cmd/generic-worker/testdata}/run-after-user.bat (100%) rename {testdata => cmd/generic-worker/testdata}/spawn-orphan-process.go (100%) rename {testdata => cmd/generic-worker/testdata}/tasks/KTBKfEgxR5GdfIIREQIvFQ.json (100%) rename {testdata => cmd/generic-worker/testdata}/tasks/LK1Rz2UtT16d-HBSqyCtuA.json (100%) rename {testdata => cmd/generic-worker/testdata}/tasks/README.md (100%) rename {testdata => cmd/generic-worker/testdata}/tasks/VESwp9JaRo-XkFN_bemBhw.json (100%) diff --git a/testdata/SampleArtifacts/%%%/v/X b/cmd/generic-worker/testdata/SampleArtifacts/%%%/v/X similarity index 100% rename from testdata/SampleArtifacts/%%%/v/X rename to cmd/generic-worker/testdata/SampleArtifacts/%%%/v/X diff --git a/testdata/SampleArtifacts/_/X.txt b/cmd/generic-worker/testdata/SampleArtifacts/_/X.txt similarity index 100% rename from testdata/SampleArtifacts/_/X.txt rename to cmd/generic-worker/testdata/SampleArtifacts/_/X.txt diff --git a/testdata/SampleArtifacts/b/c/d.jpg b/cmd/generic-worker/testdata/SampleArtifacts/b/c/d.jpg similarity index 100% rename from testdata/SampleArtifacts/b/c/d.jpg rename to cmd/generic-worker/testdata/SampleArtifacts/b/c/d.jpg diff --git a/testdata/config/bool-as-string.json b/cmd/generic-worker/testdata/config/bool-as-string.json similarity index 100% rename from testdata/config/bool-as-string.json rename to cmd/generic-worker/testdata/config/bool-as-string.json diff --git a/testdata/config/invalid-ip.json b/cmd/generic-worker/testdata/config/invalid-ip.json similarity index 100% rename from testdata/config/invalid-ip.json rename to cmd/generic-worker/testdata/config/invalid-ip.json diff --git a/testdata/config/invalid-json.json b/cmd/generic-worker/testdata/config/invalid-json.json similarity index 100% rename from testdata/config/invalid-json.json rename to cmd/generic-worker/testdata/config/invalid-json.json diff --git a/testdata/config/noip.json b/cmd/generic-worker/testdata/config/noip.json similarity index 100% rename from testdata/config/noip.json rename to cmd/generic-worker/testdata/config/noip.json diff --git a/testdata/config/valid.json b/cmd/generic-worker/testdata/config/valid.json similarity index 100% rename from testdata/config/valid.json rename to cmd/generic-worker/testdata/config/valid.json diff --git a/testdata/config/worker-type-metadata.json b/cmd/generic-worker/testdata/config/worker-type-metadata.json similarity index 100% rename from testdata/config/worker-type-metadata.json rename to cmd/generic-worker/testdata/config/worker-type-metadata.json diff --git a/testdata/curlget.go b/cmd/generic-worker/testdata/curlget.go similarity index 100% rename from testdata/curlget.go rename to cmd/generic-worker/testdata/curlget.go diff --git a/testdata/machine-configuration.json b/cmd/generic-worker/testdata/machine-configuration.json similarity index 100% rename from testdata/machine-configuration.json rename to cmd/generic-worker/testdata/machine-configuration.json diff --git a/testdata/mouse_and_screen_resolution.py b/cmd/generic-worker/testdata/mouse_and_screen_resolution.py similarity index 100% rename from testdata/mouse_and_screen_resolution.py rename to cmd/generic-worker/testdata/mouse_and_screen_resolution.py diff --git a/testdata/move-file.go b/cmd/generic-worker/testdata/move-file.go similarity index 100% rename from testdata/move-file.go rename to cmd/generic-worker/testdata/move-file.go diff --git a/testdata/private-opengpg-key b/cmd/generic-worker/testdata/private-opengpg-key similarity index 100% rename from testdata/private-opengpg-key rename to cmd/generic-worker/testdata/private-opengpg-key diff --git a/testdata/public-openpgp-key b/cmd/generic-worker/testdata/public-openpgp-key similarity index 100% rename from testdata/public-openpgp-key rename to cmd/generic-worker/testdata/public-openpgp-key diff --git a/testdata/resolvetask.go b/cmd/generic-worker/testdata/resolvetask.go similarity index 100% rename from testdata/resolvetask.go rename to cmd/generic-worker/testdata/resolvetask.go diff --git a/testdata/run-after-user.bat b/cmd/generic-worker/testdata/run-after-user.bat similarity index 100% rename from testdata/run-after-user.bat rename to cmd/generic-worker/testdata/run-after-user.bat diff --git a/testdata/spawn-orphan-process.go b/cmd/generic-worker/testdata/spawn-orphan-process.go similarity index 100% rename from testdata/spawn-orphan-process.go rename to cmd/generic-worker/testdata/spawn-orphan-process.go diff --git a/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json b/cmd/generic-worker/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json similarity index 100% rename from testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json rename to cmd/generic-worker/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json diff --git a/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json b/cmd/generic-worker/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json similarity index 100% rename from testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json rename to cmd/generic-worker/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json diff --git a/testdata/tasks/README.md b/cmd/generic-worker/testdata/tasks/README.md similarity index 100% rename from testdata/tasks/README.md rename to cmd/generic-worker/testdata/tasks/README.md diff --git a/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json b/cmd/generic-worker/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json similarity index 100% rename from testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json rename to cmd/generic-worker/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json From 1e5b30f17b576d3637603b3e55961d48de4a4078 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 15:19:15 +0200 Subject: [PATCH 12/58] fix gotestcover.sh to search for packages from root directory of project, not from script directory --- scripts/gotestcover.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gotestcover.sh b/scripts/gotestcover.sh index 995172bd..502b7df0 100755 --- a/scripts/gotestcover.sh +++ b/scripts/gotestcover.sh @@ -5,7 +5,7 @@ if [ -z "${REPORT}" ]; then echo "Specify a report, e.g. '${0}' myreport.txt" >&2 exit 64 fi -cd "$(dirname "${0}")" +cd "$(dirname "${0}")"/.. TEMP_SINGLE_REPORT="$(mktemp -t coverage.tmp.XXXXXXXXXX)" echo "mode: atomic" > "${REPORT}" HEAD_REV="$(git rev-parse HEAD)" From 7c4b4ca5af98dfadb3fcf6bd87213a73685a975f Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 15:43:24 +0200 Subject: [PATCH 13/58] updates --- .taskcluster.yml | 18 +++++++------- .travis.yml | 4 ++-- README.md | 4 ++-- aws/scripts/cleanup_ec2.sh | 2 +- .../README.md | 0 .../delete.sh | 0 .../process_region.sh | 0 .../update.sh | 0 .../worker_type.sh | 13 +++++----- scripts/build.sh | 2 +- scripts/clean-builds.sh | 24 +++++++++---------- scripts/publish-payload-schema.sh | 4 ++-- 12 files changed, 36 insertions(+), 35 deletions(-) rename aws/{update-workertypes => update-worker-types}/README.md (100%) rename aws/{update-workertypes => update-worker-types}/delete.sh (100%) rename aws/{update-workertypes => update-worker-types}/process_region.sh (100%) rename aws/{update-workertypes => update-worker-types}/update.sh (100%) rename aws/{update-workertypes => update-worker-types}/worker_type.sh (80%) diff --git a/.taskcluster.yml b/.taskcluster.yml index 646c5d07..7d749a72 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -310,12 +310,12 @@ tasks: export CGO_ENABLED=0 export GOROOT="$(pwd)/go1.10.3/go" export GOPATH="$(pwd)/gopath1.10.3" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + export PATH="$(go env GOPATH)/bin:${GOROOT}/bin:${PATH}" go version go env source "${GW_CREDS_BOOTSTRAP}" - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" + mkdir -p "$(go env GOPATH)/src/github.com/taskcluster" + cd "$(go env GOPATH)/src/github.com/taskcluster" if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi cd 'generic-worker' git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" @@ -382,13 +382,13 @@ tasks: # export CGO_ENABLED=0 # export GOROOT="$(pwd)/go1.10.3/go" # export GOPATH="$(pwd)/gopath1.10.3" - # export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + # export PATH="$(go env GOPATH)/bin:${GOROOT}/bin:${PATH}" # export CGO_ENABLED=0 # go version # go env # source "${GW_CREDS_BOOTSTRAP}" - # mkdir -p "${GOPATH}/src/github.com/taskcluster" - # cd "${GOPATH}/src/github.com/taskcluster" + # mkdir -p "$(go env GOPATH)/src/github.com/taskcluster" + # cd "$(go env GOPATH)/src/github.com/taskcluster" # if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi # cd 'generic-worker' # git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" @@ -454,8 +454,8 @@ tasks: go env curl -s http://taskcluster/secrets/v1/secret/repo:github.com/taskcluster/generic-worker | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -d > ~/env_private.sh source ~/env_private.sh - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" + mkdir -p "$(go env GOPATH)/src/github.com/taskcluster" + cd "$(go env GOPATH)/src/github.com/taskcluster" if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi cd 'generic-worker' git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" @@ -471,7 +471,7 @@ tasks: go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... test "$(git status --porcelain | wc -l)" == 0 GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... - "${GOPATH}/bin/ineffassign" . + "$(go env GOPATH)/bin/ineffassign" . artifacts: public/build/generic-worker-linux-amd64: path: "/go/bin/generic-worker" diff --git a/.travis.yml b/.travis.yml index 7ee78980..1307a78a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,10 @@ script: - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || test $(git status --porcelain | wc -l) == 0" - "go get -ldflags \"-X main.revision=$(git rev-parse HEAD)\" -v -t ./..." - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || GORACE=history_size=7 travis_wait 30 scripts/gotestcover.sh coverage.report" - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || ${GOPATH}/bin/ineffassign ." + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || $(go env GOPATH)/bin/ineffassign ." after_script: - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || ${GOPATH}/bin/goveralls -coverprofile=coverage.report -service=travis-ci" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || $(go env GOPATH)/bin/goveralls -coverprofile=coverage.report -service=travis-ci" before_deploy: - "source .travis_rename_releases.sh" diff --git a/README.md b/README.md index c6b36ef0..997e6b47 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,11 @@ If you prefer not to use a prepackaged binary, or want to have the latest unreleased version from the development head: -* Head over to https://golang.org/dl/ and follow the instructions for your platform. __Note, go 1.8 or higher is required__. Be sure to set your GOPATH to something appropriate. +* Head over to https://golang.org/dl/ and follow the instructions for your platform. __Note, go 1.8 or higher is required__. * Run `go get github.com/taskcluster/generic-worker` * Run `go get github.com/taskcluster/livelog` -All being well, the binaries will be built under `${GOPATH}/bin`. +All being well, the binaries will be built under `$(go env GOPATH)/bin`. # Acquire taskcluster credentials for running tests diff --git a/aws/scripts/cleanup_ec2.sh b/aws/scripts/cleanup_ec2.sh index 9eee6e4d..2454a0f3 100755 --- a/aws/scripts/cleanup_ec2.sh +++ b/aws/scripts/cleanup_ec2.sh @@ -3,6 +3,6 @@ cd "$(dirname "${0}")" go get github.com/taskcluster/generic-worker/cmd/all-worker-types -"$GOPATH/bin/all-worker-types" > /dev/null +"$(go env GOPATH)/bin/all-worker-types" > /dev/null cat worker_type_definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u rm -r worker_type_definitions diff --git a/aws/update-workertypes/README.md b/aws/update-worker-types/README.md similarity index 100% rename from aws/update-workertypes/README.md rename to aws/update-worker-types/README.md diff --git a/aws/update-workertypes/delete.sh b/aws/update-worker-types/delete.sh similarity index 100% rename from aws/update-workertypes/delete.sh rename to aws/update-worker-types/delete.sh diff --git a/aws/update-workertypes/process_region.sh b/aws/update-worker-types/process_region.sh similarity index 100% rename from aws/update-workertypes/process_region.sh rename to aws/update-worker-types/process_region.sh diff --git a/aws/update-workertypes/update.sh b/aws/update-worker-types/update.sh similarity index 100% rename from aws/update-workertypes/update.sh rename to aws/update-worker-types/update.sh diff --git a/aws/update-workertypes/worker_type.sh b/aws/update-worker-types/worker_type.sh similarity index 80% rename from aws/update-workertypes/worker_type.sh rename to aws/update-worker-types/worker_type.sh index 4e511631..4f378ffe 100755 --- a/aws/update-workertypes/worker_type.sh +++ b/aws/update-worker-types/worker_type.sh @@ -6,7 +6,6 @@ ############ newly generated AMIs). # TODO: [pmoore] submit a task after updating worker type -# TODO: [pmoore] publish ssh key to secret store after generating it echo "$(date): Checking inputs..." @@ -18,14 +17,16 @@ fi export WORKER_TYPE="${1}" export ACTION="${2}" -if [ ! -d "$(dirname "${0}")/${WORKER_TYPE}" ]; then - echo "ERROR: No directory for worker type: '$(dirname "${0}")/${WORKER_TYPE}'" +WORKER_TYPES_DIR=${WORKER_TYPES_DIR:-$(dirname "${0}")} + +if [ ! -d "${WORKER_TYPES_DIR}/${WORKER_TYPE}" ]; then + echo "ERROR: No directory for worker type: '${WORKER_TYPES_DIR}/${WORKER_TYPE}'" exit 65 fi echo "$(date): Starting"'!' -# cd into directory containing script... +# cd into worker type directory containing script... cd "$(dirname "${0}")/${WORKER_TYPE}" # needed to not confuse the script later @@ -34,7 +35,7 @@ rm -f *.latest-ami # generate a random slugid for aws client token... go get github.com/taskcluster/slugid-go/slug go install github.com/taskcluster/generic-worker/update-worker-type -export SLUGID=$("${GOPATH}/bin/slug") +export SLUGID=$("$(go env GOPATH)/bin/slug") # aws ec2 describe-regions --query '{A:Regions[*].RegionName}' --output text | grep -v sa-east-1 | while read x REGION; do # (skip sa-east-1 since it doesn't support all the APIs we use in this script) @@ -42,7 +43,7 @@ export SLUGID=$("${GOPATH}/bin/slug") echo us-west-1 118 us-west-2 199 us-east-1 100 | xargs -P32 -n2 ../process_region.sh if [ "${ACTION}" == "update" ]; then - "${GOPATH}/bin/update-worker-type" . + "$(go env GOPATH)/bin/update-worker-type" . echo echo "The worker type has been proactively updated("'!'"):" echo diff --git a/scripts/build.sh b/scripts/build.sh index 541cc407..e2b18fdf 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -104,7 +104,7 @@ fi ##install plan9 amd64 #install solaris amd64 -find "${GOPATH}/bin" -name 'generic-worker*' +find "$(go env GOPATH)/bin" -name 'generic-worker*' CGO_ENABLED=0 go get github.com/taskcluster/livelog # capital X here ... we only want to delete things that are ignored! diff --git a/scripts/clean-builds.sh b/scripts/clean-builds.sh index 83fb824f..d1d78660 100755 --- a/scripts/clean-builds.sh +++ b/scripts/clean-builds.sh @@ -33,19 +33,19 @@ export GOARCH="${MY_GOARCH}" go version go env export GOPATH="$(mktemp -d -t generic-worker.XXXXXXXXXX)" -export PATH="${GOPATH}/bin:${PATH}" -rm -rf "${GOPATH}" +export PATH="$(go env GOPATH)/bin:${PATH}" +rm -rf "$(go env GOPATH)" go get -d -v 'github.com/taskcluster/generic-worker' -"${GOPATH}/src/github.com/taskcluster/generic-worker/build.sh" -a -t +"$(go env GOPATH)/src/github.com/taskcluster/generic-worker/build.sh" -a -t -mv "${GOPATH}/bin/darwin_386/generic-worker" target/generic-worker-darwin-386 -mv "${GOPATH}/bin/generic-worker" target/generic-worker-darwin-amd64 -mv "${GOPATH}/bin/linux_386/generic-worker" target/generic-worker-linux-386 -mv "${GOPATH}/bin/linux_amd64/generic-worker" target/generic-worker-linux-amd64 -mv "${GOPATH}/bin/linux_arm/generic-worker" target/generic-worker-linux-arm -mv "${GOPATH}/bin/linux_arm64/generic-worker" target/generic-worker-linux-arm64 -mv "${GOPATH}/bin/windows_386/generic-worker.exe" target/generic-worker-windows-386 -mv "${GOPATH}/bin/windows_amd64/generic-worker.exe" target/generic-worker-windows-amd64 +mv "$(go env GOPATH)/bin/darwin_386/generic-worker" target/generic-worker-darwin-386 +mv "$(go env GOPATH)/bin/generic-worker" target/generic-worker-darwin-amd64 +mv "$(go env GOPATH)/bin/linux_386/generic-worker" target/generic-worker-linux-386 +mv "$(go env GOPATH)/bin/linux_amd64/generic-worker" target/generic-worker-linux-amd64 +mv "$(go env GOPATH)/bin/linux_arm/generic-worker" target/generic-worker-linux-arm +mv "$(go env GOPATH)/bin/linux_arm64/generic-worker" target/generic-worker-linux-arm64 +mv "$(go env GOPATH)/bin/windows_386/generic-worker.exe" target/generic-worker-windows-386 +mv "$(go env GOPATH)/bin/windows_amd64/generic-worker.exe" target/generic-worker-windows-amd64 echo echo "Release binaries available in directory '$(pwd)/target'" @@ -56,5 +56,5 @@ echo echo rm -rf "${GO_DOWNLOAD_DIR}" -rm -rf "${GOPATH}" +rm -rf "$(go env GOPATH)" rm target/go.tar.gz diff --git a/scripts/publish-payload-schema.sh b/scripts/publish-payload-schema.sh index e05f7f53..ec81deee 100755 --- a/scripts/publish-payload-schema.sh +++ b/scripts/publish-payload-schema.sh @@ -1,9 +1,9 @@ #!/bin/bash -e cd "$(dirname "${0}")" -go get github.com/taskcluster/generic-worker/yamltojson +go get github.com/taskcluster/generic-worker/cmd/yamltojson jsonFile="$(mktemp -t jsonFile.XXXXXX)" -cat windows.yml | yamltojson > "${jsonFile}" +cat windows.yml | "$(go env GOPATH)/bin/yamltojson" > "${jsonFile}" aws s3 cp "${jsonFile}" s3://schemas.taskcluster.net/generic-worker/v1/payload.json rm "${jsonFile}" curl https://schemas.taskcluster.net/generic-worker/v1/payload.json From c0c052c87415a0e434e25ffc1ba2956640fed5bc Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 15:45:55 +0200 Subject: [PATCH 14/58] fixes --- aws/update-worker-types/worker_type.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/update-worker-types/worker_type.sh b/aws/update-worker-types/worker_type.sh index 4f378ffe..222fad73 100755 --- a/aws/update-worker-types/worker_type.sh +++ b/aws/update-worker-types/worker_type.sh @@ -34,7 +34,7 @@ rm -f *.latest-ami # generate a random slugid for aws client token... go get github.com/taskcluster/slugid-go/slug -go install github.com/taskcluster/generic-worker/update-worker-type +go install github.com/taskcluster/generic-worker/aws/cmd/update-worker-type export SLUGID=$("$(go env GOPATH)/bin/slug") # aws ec2 describe-regions --query '{A:Regions[*].RegionName}' --output text | grep -v sa-east-1 | while read x REGION; do From 9a1f5ce82b7a5311bb778281005407472a3afed3 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 17:02:13 +0200 Subject: [PATCH 15/58] fixed ldflags parameter when building --- .taskcluster.yml | 18 +++++++++--------- cmd/generic-worker/main_test.go | 2 +- scripts/build.cmd | 2 +- scripts/build.sh | 2 +- scripts/gotestcover.sh | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 7d749a72..ad900476 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -105,9 +105,9 @@ tasks: - set CGO_ENABLED=1 - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe @@ -176,9 +176,9 @@ tasks: - go get -ldflags "-X main.revision=%revision%" -v -u -t ./... - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v' - ineffassign . artifacts: - name: public/build/generic-worker-windows-386.exe @@ -258,9 +258,9 @@ tasks: - set CGO_ENABLED=1 - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe @@ -331,7 +331,7 @@ tasks: go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # output of wc command can contain spaces on darwin, so no quotes around expression test $(git status --porcelain | wc -l) == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -race -v ./... + GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -v ./... ineffassign . artifacts: - name: public/build/generic-worker-darwin-amd64 @@ -403,7 +403,7 @@ tasks: # go generate ./... # go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # test "$(git status --porcelain | wc -l)" == 0 - # GORACE=history_size=7 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v ./... + # GORACE=history_size=7 go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v ./... # ineffassign . # artifacts: # - name: public/build/generic-worker-linux-armv6l @@ -470,7 +470,7 @@ tasks: go generate ./... go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... test "$(git status --porcelain | wc -l)" == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... + GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... "$(go env GOPATH)/bin/ineffassign" . artifacts: public/build/generic-worker-linux-amd64: diff --git a/cmd/generic-worker/main_test.go b/cmd/generic-worker/main_test.go index 762e242b..9d4c768e 100644 --- a/cmd/generic-worker/main_test.go +++ b/cmd/generic-worker/main_test.go @@ -110,7 +110,7 @@ func TestIdleWithoutCrash(t *testing.T) { func TestRevisionNumberStored(t *testing.T) { if !regexp.MustCompile("^[0-9a-f]{40}$").MatchString(revision) { t.Fatalf("Git revision could not be determined - got '%v' but expected to match regular expression '^[0-9a-f](40)$'\n"+ - "Did you specify `-ldflags \"-X github.com/taskcluster/generic-worker.revision=\"` in your go test command?\n"+ + "Did you specify `-ldflags \"-X github.com/taskcluster/cmd/generic-worker.revision=\"` in your go test command?\n"+ "Try using build.sh / build.cmd in root directory of generic-worker source code repository.", revision) } t.Logf("Git revision successfully retrieved: %v", revision) diff --git a/scripts/build.cmd b/scripts/build.cmd index 1feda1b7..23031c4a 100644 --- a/scripts/build.cmd +++ b/scripts/build.cmd @@ -23,5 +23,5 @@ git rev-parse HEAD > revision.txt set /p REVISION=< revision.txt del revision.txt set GORACE=history_size=7 -go test -ldflags "-X github.com/taskcluster/generic-worker.revision=%REVISION%" ./... || exit /b %ERRORLEVEL% +go test -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%REVISION%" ./... || exit /b %ERRORLEVEL% ineffassign . || exit /b %ERRORLEVEL% diff --git a/scripts/build.sh b/scripts/build.sh index e2b18fdf..83e1f0f2 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -112,7 +112,7 @@ git clean -fdX if $TEST; then go get github.com/taskcluster/taskcluster-proxy - CGO_ENABLED=1 GORACE="history_size=7" go test -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -race -timeout 1h ./... + CGO_ENABLED=1 GORACE="history_size=7" go test -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -timeout 1h ./... fi go vet ./... go get github.com/golang/lint/golint diff --git a/scripts/gotestcover.sh b/scripts/gotestcover.sh index 502b7df0..85c03d12 100755 --- a/scripts/gotestcover.sh +++ b/scripts/gotestcover.sh @@ -16,7 +16,7 @@ go list ./... > "${PACKAGE_LIST}" while read package do - CGO_ENABLED=1 go test -ldflags "-X github.com/taskcluster/generic-worker.revision=${HEAD_REV}" -race -timeout 1h -covermode=atomic "-coverprofile=${TEMP_SINGLE_REPORT}" "${package}" + CGO_ENABLED=1 go test -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=${HEAD_REV}" -race -timeout 1h -covermode=atomic "-coverprofile=${TEMP_SINGLE_REPORT}" "${package}" if [ -f "${TEMP_SINGLE_REPORT}" ]; then sed 1d "${TEMP_SINGLE_REPORT}" >> "${REPORT}" rm "${TEMP_SINGLE_REPORT}" From 92d2d9344a7d88f7fd20b6f67ff020a6e33d5abc Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 17:21:47 +0200 Subject: [PATCH 16/58] cleanup --- .gitignore | 2 +- aws/cmd/aws-worker-types/main.go | 7 ++++--- aws/scripts/cleanup_ec2.sh | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 6ce29864..d726e949 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ _testmain.go # For any releases made locally /target/ -worker_type_definitions +aws-provisioner-v1-worker-type-definitions # Files created by generic-worker generic-worker.config diff --git a/aws/cmd/aws-worker-types/main.go b/aws/cmd/aws-worker-types/main.go index 24d462a2..d1dbb156 100644 --- a/aws/cmd/aws-worker-types/main.go +++ b/aws/cmd/aws-worker-types/main.go @@ -29,8 +29,9 @@ type WorkerPool struct { // This program is a simple command line utility. When you run it, it will // spawn 20 go routines to download worker type defintions for // aws-provisioner-v1 in parallel. It will put them in a sub-directory called -// worker_type_definitions, creating it if necessary, with each file named as -// the workerType. I use this in combination with a cron job that runs +// aws-provisioner-v1-worker-type-definitions, creating it if necessary, with +// each file named as the workerType. I use this in combination with a cron job +// that runs // https://github.com/petemoore/myscrapbook/blob/master/sync-worker-type-definitions.sh // every 5 mins, in order to maintain a git history of worker type definitions // locally. I don't publish the git repository anywhere, since the worker type @@ -38,7 +39,7 @@ type WorkerPool struct { // secrets at some point). func main() { prov := tcawsprovisioner.NewFromEnv() - downloadDirectory := "worker_type_definitions" + downloadDirectory := "aws-provisioner-v1-worker-type-definitions" requestChannel := make(chan string) processedChannel := make(chan string) _ = NewWorkerPool(20, requestChannel, processedChannel, prov, downloadDirectory) diff --git a/aws/scripts/cleanup_ec2.sh b/aws/scripts/cleanup_ec2.sh index 2454a0f3..d656c38c 100755 --- a/aws/scripts/cleanup_ec2.sh +++ b/aws/scripts/cleanup_ec2.sh @@ -2,7 +2,7 @@ cd "$(dirname "${0}")" -go get github.com/taskcluster/generic-worker/cmd/all-worker-types -"$(go env GOPATH)/bin/all-worker-types" > /dev/null -cat worker_type_definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u -rm -r worker_type_definitions +go get github.com/taskcluster/generic-worker/cmd/aws-worker-types +"$(go env GOPATH)/bin/aws-worker-types" > /dev/null +cat aws-provisioner-v1-worker-type-definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u +rm -r aws-provisioner-v1-worker-type-definitions From 76fcb19419a97bc07bbd0f47715eddf9f0f62609 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 17:31:11 +0200 Subject: [PATCH 17/58] fix --- .taskcluster.yml | 18 +++++++++--------- cmd/generic-worker/main_test.go | 2 +- scripts/build.cmd | 2 +- scripts/build.sh | 2 +- scripts/gotestcover.sh | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index ad900476..a605fe5f 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -105,9 +105,9 @@ tasks: - set CGO_ENABLED=1 - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe @@ -176,9 +176,9 @@ tasks: - go get -ldflags "-X main.revision=%revision%" -v -u -t ./... - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v' - ineffassign . artifacts: - name: public/build/generic-worker-windows-386.exe @@ -258,9 +258,9 @@ tasks: - set CGO_ENABLED=1 - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe @@ -331,7 +331,7 @@ tasks: go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # output of wc command can contain spaces on darwin, so no quotes around expression test $(git status --porcelain | wc -l) == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -v ./... + GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -v ./... ineffassign . artifacts: - name: public/build/generic-worker-darwin-amd64 @@ -403,7 +403,7 @@ tasks: # go generate ./... # go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # test "$(git status --porcelain | wc -l)" == 0 - # GORACE=history_size=7 go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v ./... + # GORACE=history_size=7 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v ./... # ineffassign . # artifacts: # - name: public/build/generic-worker-linux-armv6l @@ -470,7 +470,7 @@ tasks: go generate ./... go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... test "$(git status --porcelain | wc -l)" == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... + GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... "$(go env GOPATH)/bin/ineffassign" . artifacts: public/build/generic-worker-linux-amd64: diff --git a/cmd/generic-worker/main_test.go b/cmd/generic-worker/main_test.go index 9d4c768e..75711a4d 100644 --- a/cmd/generic-worker/main_test.go +++ b/cmd/generic-worker/main_test.go @@ -110,7 +110,7 @@ func TestIdleWithoutCrash(t *testing.T) { func TestRevisionNumberStored(t *testing.T) { if !regexp.MustCompile("^[0-9a-f]{40}$").MatchString(revision) { t.Fatalf("Git revision could not be determined - got '%v' but expected to match regular expression '^[0-9a-f](40)$'\n"+ - "Did you specify `-ldflags \"-X github.com/taskcluster/cmd/generic-worker.revision=\"` in your go test command?\n"+ + "Did you specify `-ldflags \"-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=\"` in your go test command?\n"+ "Try using build.sh / build.cmd in root directory of generic-worker source code repository.", revision) } t.Logf("Git revision successfully retrieved: %v", revision) diff --git a/scripts/build.cmd b/scripts/build.cmd index 23031c4a..81a86785 100644 --- a/scripts/build.cmd +++ b/scripts/build.cmd @@ -23,5 +23,5 @@ git rev-parse HEAD > revision.txt set /p REVISION=< revision.txt del revision.txt set GORACE=history_size=7 -go test -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=%REVISION%" ./... || exit /b %ERRORLEVEL% +go test -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%REVISION%" ./... || exit /b %ERRORLEVEL% ineffassign . || exit /b %ERRORLEVEL% diff --git a/scripts/build.sh b/scripts/build.sh index 83e1f0f2..8e99837c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -112,7 +112,7 @@ git clean -fdX if $TEST; then go get github.com/taskcluster/taskcluster-proxy - CGO_ENABLED=1 GORACE="history_size=7" go test -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -timeout 1h ./... + CGO_ENABLED=1 GORACE="history_size=7" go test -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -timeout 1h ./... fi go vet ./... go get github.com/golang/lint/golint diff --git a/scripts/gotestcover.sh b/scripts/gotestcover.sh index 85c03d12..33dee6cd 100755 --- a/scripts/gotestcover.sh +++ b/scripts/gotestcover.sh @@ -16,7 +16,7 @@ go list ./... > "${PACKAGE_LIST}" while read package do - CGO_ENABLED=1 go test -ldflags "-X github.com/taskcluster/cmd/generic-worker.revision=${HEAD_REV}" -race -timeout 1h -covermode=atomic "-coverprofile=${TEMP_SINGLE_REPORT}" "${package}" + CGO_ENABLED=1 go test -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=${HEAD_REV}" -race -timeout 1h -covermode=atomic "-coverprofile=${TEMP_SINGLE_REPORT}" "${package}" if [ -f "${TEMP_SINGLE_REPORT}" ]; then sed 1d "${TEMP_SINGLE_REPORT}" >> "${REPORT}" rm "${TEMP_SINGLE_REPORT}" From a01a3491d1f57fa521d7ff3e41aae2f8638e353c Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 18:17:04 +0200 Subject: [PATCH 18/58] cleanup --- .../find_old_aws_objects.sh | 4 +-- aws/update-worker-types/process_region.sh | 6 ++--- aws/update-worker-types/update.sh | 26 +++++++++---------- aws/update-worker-types/worker_type.sh | 23 +++++++++------- 4 files changed, 30 insertions(+), 29 deletions(-) rename aws/{scripts => update-worker-types}/find_old_aws_objects.sh (64%) diff --git a/aws/scripts/find_old_aws_objects.sh b/aws/update-worker-types/find_old_aws_objects.sh similarity index 64% rename from aws/scripts/find_old_aws_objects.sh rename to aws/update-worker-types/find_old_aws_objects.sh index 339a4106..3e18f345 100755 --- a/aws/scripts/find_old_aws_objects.sh +++ b/aws/update-worker-types/find_old_aws_objects.sh @@ -4,8 +4,8 @@ OLD_INSTANCES="$(aws --region "${REGION}" ec2 describe-instances --filters "Name # find old amis log "Querying previous AMI..." -OLD_SNAPSHOTS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} mozillabuild version*" --query 'Images[*].BlockDeviceMappings[*].Ebs.SnapshotId' --output text)" +OLD_SNAPSHOTS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} version *" --query 'Images[*].BlockDeviceMappings[*].Ebs.SnapshotId' --output text)" # find old snapshots log "Querying snapshot used in this previous AMI..." -OLD_AMIS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} mozillabuild version*" --query 'Images[*].ImageId' --output text)" +OLD_AMIS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} version *" --query 'Images[*].ImageId' --output text)" diff --git a/aws/update-worker-types/process_region.sh b/aws/update-worker-types/process_region.sh index 24869900..40657002 100755 --- a/aws/update-worker-types/process_region.sh +++ b/aws/update-worker-types/process_region.sh @@ -29,12 +29,10 @@ if [ -z "${COLOUR}" ]; then exit 66 fi -cd "$(dirname "${0}")/${WORKER_TYPE}" - if [ "${ACTION}" == "update" ]; then - . ../update.sh + . "$(dirname "${0}")/update.sh" elif [ "${ACTION}" == "delete" ]; then - . ../delete.sh + . "$(dirname "${0}")/delete.sh" else log "$(basename "${0}"): ERROR: Unknown action '${ACTION}' ... exiting" >&2 exit 86 diff --git a/aws/update-worker-types/update.sh b/aws/update-worker-types/update.sh index 5e4a5afa..164408ae 100755 --- a/aws/update-worker-types/update.sh +++ b/aws/update-worker-types/update.sh @@ -1,14 +1,12 @@ -cd "$(dirname "${0}")/${WORKER_TYPE}" - log "Generating new ssh key..." -rm -rf "${REGION}.id_rsa" +rm -rf "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" aws --region "${REGION}" ec2 delete-key-pair --key-name "${WORKER_TYPE}_${REGION}" || true -aws --region "${REGION}" ec2 create-key-pair --key-name "${WORKER_TYPE}_${REGION}" --query 'KeyMaterial' --output text > "${REGION}.id_rsa" -chmod 400 "${REGION}.id_rsa" +aws --region "${REGION}" ec2 create-key-pair --key-name "${WORKER_TYPE}_${REGION}" --query 'KeyMaterial' --output text > "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" +chmod 400 "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" # aws cli docs lie, they say userdata must be base64 encoded, but cli encodes for you, so just cat it... -USER_DATA="$(cat userdata)" -AMI_BASE_NAME="$(cat ami-base-name)" +USER_DATA="$(cat "${WORKER_TYPE_DEFINITION_DIR}/userdata")" +AMI_BASE_NAME="$(cat "${WORKER_TYPE_DEFINITION_DIR}/ami-base-name")" # find out latest windows 2012 r2 ami to use... AMI_METADATA="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=platform,Values=windows" "Name=name,Values=${AMI_BASE_NAME}" --query 'Images[*].{A:CreationDate,B:ImageId,C:Name}' --output text | sort -u | tail -1 | cut -f2,3)" @@ -17,7 +15,7 @@ AMI="$(echo $AMI_METADATA | sed 's/ .*//')" AMI_NAME="$(echo $AMI_METADATA | sed 's/.* //')" log "Base AMI is: ${AMI} ('${AMI_NAME}')" -. ../find_old_aws_objects.sh +. "$(dirname "${0}")/find_old_aws_objects.sh" # make sure we have an ssh security group in this region # note if we *try* to create a security group that already exists (regardless of whether it is successful or not), there will be a cloudwatch alarm, so avoid this @@ -33,7 +31,7 @@ done # create new base ami, and apply user-data # filter output, to get INSTANCE_ID -INSTANCE_ID="$(aws --region "${REGION}" ec2 run-instances --image-id "${AMI}" --key-name "${WORKER_TYPE}_${REGION}" --security-groups "rdp-only" "ssh-only" --user-data "$(cat userdata)" --instance-type c4.2xlarge --block-device-mappings DeviceName=/dev/sda1,Ebs='{VolumeSize=75,DeleteOnTermination=true,VolumeType=gp2}' --instance-initiated-shutdown-behavior stop --client-token "${SLUGID}" --query 'Instances[*].InstanceId' --output text)" +INSTANCE_ID="$(aws --region "${REGION}" ec2 run-instances --image-id "${AMI}" --key-name "${WORKER_TYPE}_${REGION}" --security-groups "rdp-only" "ssh-only" --user-data "$(cat "${WORKER_TYPE_DEFINITION_DIR}/userdata")" --instance-type c4.2xlarge --block-device-mappings DeviceName=/dev/sda1,Ebs='{VolumeSize=75,DeleteOnTermination=true,VolumeType=gp2}' --instance-initiated-shutdown-behavior stop --client-token "${SLUGID}" --query 'Instances[*].InstanceId' --output text)" log "I've triggered the creation of instance ${INSTANCE_ID} - it can take a \x1B[4mVery Long Time™\x1B[24m for it to be created and bootstrapped..." aws --region "${REGION}" ec2 create-tags --resources "${INSTANCE_ID}" --tags "Key=WorkerType,Value=aws-provisioner-v1/${WORKER_TYPE}" "Key=Name,Value=${WORKER_TYPE} base instance" "Key=TC-Windows-Base,Value=true" @@ -50,11 +48,11 @@ done log "Now snapshotting the instance to create an AMI..." # now capture the AMI -IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} mozillabuild version ${SLUGID}" --description "firefox desktop builds on windows - taskcluster worker - version ${SLUGID}" --output text)" +IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} version ${SLUGID}" --description "taskcluster generic-worker worker type - ${WORKER_TYPE} - version ${SLUGID}" --output text)" log "The AMI is currently being created: ${IMAGE_ID}" -PASSWORD="$(aws --region "${REGION}" ec2 get-password-data --instance-id "${INSTANCE_ID}" --priv-launch-key ${REGION}.id_rsa --output text --query PasswordData)" +PASSWORD="$(aws --region "${REGION}" ec2 get-password-data --instance-id "${INSTANCE_ID}" --priv-launch-key "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" --output text --query PasswordData)" log "To connect to the template instance (please don't do so until AMI creation process is completed"'!'"):" log '' @@ -73,16 +71,16 @@ until aws --region "${REGION}" ec2 wait image-available --image-ids "${IMAGE_ID} sleep 30 done -touch "${REGION}.${IMAGE_ID}.latest-ami" +touch "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.${IMAGE_ID}.latest-ami" { echo "Instance: ${INSTANCE_ID}" echo "Public IP: ${PUBLIC_IP}" echo "Password: ${PASSWORD}" echo "AMI: ${IMAGE_ID}" -} > "${REGION}.secrets" +} > "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.secrets" -. ../delete.sh +. "$(dirname "${0}")/delete.sh" # log '' # log "Starting instance ${INSTANCE_ID} back up..." diff --git a/aws/update-worker-types/worker_type.sh b/aws/update-worker-types/worker_type.sh index 222fad73..0f42052f 100755 --- a/aws/update-worker-types/worker_type.sh +++ b/aws/update-worker-types/worker_type.sh @@ -17,18 +17,23 @@ fi export WORKER_TYPE="${1}" export ACTION="${2}" -WORKER_TYPES_DIR=${WORKER_TYPES_DIR:-$(dirname "${0}")} - -if [ ! -d "${WORKER_TYPES_DIR}/${WORKER_TYPE}" ]; then - echo "ERROR: No directory for worker type: '${WORKER_TYPES_DIR}/${WORKER_TYPE}'" +# Default directory to look for definitions is current directory. +# To select a different directory, simply export WORKER_TYPES_DIR +# to chosen directory before calling this script. +WORKER_TYPES_DIR=${WORKER_TYPES_DIR:-.} + +# Note we export this env var for subshells, rathing than passing explicitly as +# a command line argument, to keep xargs commands simple later on. +export WORKER_TYPE_DEFINITION_DIR="${WORKER_TYPES_DIR}/${WORKER_TYPE}" + +if [ ! -d "${WORKER_TYPE_DEFINITION_DIR}" ]; then + echo "ERROR: No directory for worker type: '${WORKER_TYPE_DEFINITION_DIR}'" >&2 + echo "Note, if your worker type definitions are stored locally in a different directory, please export WORKER_TYPES_DIR" >&2 exit 65 fi echo "$(date): Starting"'!' -# cd into worker type directory containing script... -cd "$(dirname "${0}")/${WORKER_TYPE}" - # needed to not confuse the script later rm -f *.latest-ami @@ -40,10 +45,10 @@ export SLUGID=$("$(go env GOPATH)/bin/slug") # aws ec2 describe-regions --query '{A:Regions[*].RegionName}' --output text | grep -v sa-east-1 | while read x REGION; do # (skip sa-east-1 since it doesn't support all the APIs we use in this script) -echo us-west-1 118 us-west-2 199 us-east-1 100 | xargs -P32 -n2 ../process_region.sh +echo us-west-1 118 us-west-2 199 us-east-1 100 | xargs -P32 -n2 "$(dirname "${0}")/process_region.sh" if [ "${ACTION}" == "update" ]; then - "$(go env GOPATH)/bin/update-worker-type" . + "$(go env GOPATH)/bin/update-worker-type" "${WORKER_TYPE_DEFINITION_DIR}" echo echo "The worker type has been proactively updated("'!'"):" echo From a48d4613886090e47aa1a0997846832322dc29fb Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 18:46:20 +0200 Subject: [PATCH 19/58] fix --- .taskcluster.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index a605fe5f..1caa0338 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -107,7 +107,7 @@ tasks: - C:\generic-worker\generic-worker-test-creds.cmd - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race github.com/taskcluster/generic-worker/cmd/generic-worker' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe @@ -178,7 +178,7 @@ tasks: - C:\generic-worker\generic-worker-test-creds.cmd - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v github.com/taskcluster/generic-worker/cmd/generic-worker' - ineffassign . artifacts: - name: public/build/generic-worker-windows-386.exe @@ -260,7 +260,7 @@ tasks: - C:\generic-worker\generic-worker-test-creds.cmd - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race github.com/taskcluster/generic-worker/cmd/generic-worker' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe From 52cb98663f16568590720a8198cb2c034ae710d3 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 19:03:10 +0200 Subject: [PATCH 20/58] cleanup --- docs/payload.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/payload.md b/docs/payload.md index faae6240..641b0d96 100644 --- a/docs/payload.md +++ b/docs/payload.md @@ -15,11 +15,21 @@ When submitting a task graph to the Task Cluster Queue (see payload for defining the tasks to be executed by the worker. In the case of the generic worker, the payload must conform to the following schema. -
+ +# Windows + +
+ +# macOS + +
+ +# Linux + +
The payload comprises of a command to run, environment variables to be set (optionally encrypted) and a timeout for the task (`maxRunTime`). The worker will run the task, upload log files, and report back status to the Queue. - From 3f178ec3ddfe11193b551ae443fc53458925e822 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 5 Oct 2018 19:06:12 +0200 Subject: [PATCH 21/58] fixes --- scripts/publish-payload-schema.sh | 18 +++++++++++++----- scripts/refresh_readme.sh | 4 ++-- scripts/release.sh | 8 ++++---- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/scripts/publish-payload-schema.sh b/scripts/publish-payload-schema.sh index ec81deee..3250ef01 100755 --- a/scripts/publish-payload-schema.sh +++ b/scripts/publish-payload-schema.sh @@ -2,10 +2,18 @@ cd "$(dirname "${0}")" go get github.com/taskcluster/generic-worker/cmd/yamltojson -jsonFile="$(mktemp -t jsonFile.XXXXXX)" -cat windows.yml | "$(go env GOPATH)/bin/yamltojson" > "${jsonFile}" -aws s3 cp "${jsonFile}" s3://schemas.taskcluster.net/generic-worker/v1/payload.json -rm "${jsonFile}" -curl https://schemas.taskcluster.net/generic-worker/v1/payload.json + +windows_payload_file="$(mktemp -t windows-payload.json.XXXXXX)" +all_unix_style_payload_file="$(mktemp -t all-unix-style-payload.json.XXXXXX)" +cat ../schemas/windows.yml | "$(go env GOPATH)/bin/yamltojson" > "${windows_payload_file}" +cat ../schemas/all-unix-style.yml | "$(go env GOPATH)/bin/yamltojson" > "${all_unix_style_payload_file}" +aws s3 cp "${windows_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/windows.json +aws s3 cp "${all_unix_style_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/linux.json +aws s3 cp "${all_unix_style_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/macos.json +rm "${windows_payload_file}" +rm "${all_unix_style_payload_file}" +curl https://schemas.taskcluster.net/generic-worker/v1/windows.json +curl https://schemas.taskcluster.net/generic-worker/v1/linux.json +curl https://schemas.taskcluster.net/generic-worker/v1/macos.json echo echo "Schema updated - check out https://docs.taskcluster.net/reference/workers/generic-worker/payload" diff --git a/scripts/refresh_readme.sh b/scripts/refresh_readme.sh index 5b7ae6a7..1f11929c 100755 --- a/scripts/refresh_readme.sh +++ b/scripts/refresh_readme.sh @@ -1,10 +1,10 @@ #!/bin/bash -eu -cd "$(dirname "${0}")" +cd "$(dirname "${0}")/.." VALID_FORMAT='[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*' TEMP_GW_HELP="$(mktemp -t generic-worker-help-text.XXXXXXXXXX)" TEMP_GW_README="$(mktemp -t generic-worker-readme.XXXXXXXXXX)" TEMP_GW_BINARY="$(mktemp -t generic-worker.XXXXXXXXXX)" -go build -o "${TEMP_GW_BINARY}" +go build ./cmd/generic-worker -o "${TEMP_GW_BINARY}" "${TEMP_GW_BINARY}" --help > "${TEMP_GW_HELP}" echo '```' >> "${TEMP_GW_HELP}" sed -e " diff --git a/scripts/release.sh b/scripts/release.sh index fbbe2c62..b17c144c 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -11,7 +11,7 @@ set -e OFFICIAL_GIT_REPO='git@github.com:taskcluster/generic-worker' # step into directory containing this script -cd "$(dirname "${0}")" +cd "$(dirname "${0}")/.." NEW_VERSION="${1}" @@ -102,12 +102,12 @@ if ! echo "${NEW_VERSION}" | grep -q "alpha"; then fi fi -inline_sed README.md "s/.\/release.sh ${OLD_VERSION//./\\.}/.\/release.sh ${NEW_VERSION}/" +inline_sed README.md "s/scripts\/release.sh ${OLD_VERSION//./\\.}/scripts\/release.sh ${NEW_VERSION}/" inline_sed main.go 's/\(version *= *\)"'"${OLD_VERSION//./\\.}"'"$/\1"'"${NEW_VERSION}"'"/' find . -name userdata | while read file; do inline_sed "${file}" "s:taskcluster/generic-worker/releases/download/v${OLD_VERSION//./\\.}/:taskcluster/generic-worker/releases/download/v${NEW_VERSION}/:g" done -./refresh_readme.sh +"$(dirname "${0}")/refresh_readme.sh" git add README.md git commit -m "Version bump from ${OLD_VERSION} to ${NEW_VERSION}" git tag -s "v${NEW_VERSION}" -m "Making release ${NEW_VERSION}" @@ -122,7 +122,7 @@ if ! echo "${NEW_VERSION}" | grep -q "alpha"; then echo echo 'Will you also be deploying this release to production? If so, please run:' echo - echo ' ***** ./publish-payload-schema.sh *****' + echo ' ***** scripts/publish-payload-schema.sh *****' echo echo 'This will update:' echo From 1f3c8961e036d3e05e2169e651f0ca06791f9362 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Sat, 6 Oct 2018 19:30:22 +0200 Subject: [PATCH 22/58] continue publishing to s3://schemas.taskcluster.net/generic-worker/v1/payload.json for backwards compatibility --- scripts/publish-payload-schema.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/publish-payload-schema.sh b/scripts/publish-payload-schema.sh index 3250ef01..0ff8a42b 100755 --- a/scripts/publish-payload-schema.sh +++ b/scripts/publish-payload-schema.sh @@ -8,10 +8,12 @@ all_unix_style_payload_file="$(mktemp -t all-unix-style-payload.json.XXXXXX)" cat ../schemas/windows.yml | "$(go env GOPATH)/bin/yamltojson" > "${windows_payload_file}" cat ../schemas/all-unix-style.yml | "$(go env GOPATH)/bin/yamltojson" > "${all_unix_style_payload_file}" aws s3 cp "${windows_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/windows.json +aws s3 cp "${windows_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/payload.json aws s3 cp "${all_unix_style_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/linux.json aws s3 cp "${all_unix_style_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/macos.json rm "${windows_payload_file}" rm "${all_unix_style_payload_file}" +curl https://schemas.taskcluster.net/generic-worker/v1/payload.json curl https://schemas.taskcluster.net/generic-worker/v1/windows.json curl https://schemas.taskcluster.net/generic-worker/v1/linux.json curl https://schemas.taskcluster.net/generic-worker/v1/macos.json From d98115db2ec28a06bc9e04c45d090ce9fc7b0e76 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Sat, 6 Oct 2018 19:37:49 +0200 Subject: [PATCH 23/58] Mention bug 1456357 in yaml comment so line doesn't get added again --- schemas/all-unix-style.yml | 7 +++++++ schemas/windows.yml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/schemas/all-unix-style.yml b/schemas/all-unix-style.yml index 9bbad9a9..d862b585 100644 --- a/schemas/all-unix-style.yml +++ b/schemas/all-unix-style.yml @@ -1,4 +1,11 @@ --- + +# Bug 1456357 - Do **NOT** consider adding this line back in, it causes +# github.com/xeipuuv/gojsonschema to attempt to download the schema which could +# be out-of-sync with this version which is burned into the release. +# +# ### DO NOT ADD THIS!! -> id: https://schemas.taskcluster.net/generic-worker/v1/payload.json# + "$schema": http://json-schema.org/draft-04/schema# title: Generic worker payload description: |- diff --git a/schemas/windows.yml b/schemas/windows.yml index dbea5bae..d3f806e0 100644 --- a/schemas/windows.yml +++ b/schemas/windows.yml @@ -1,4 +1,11 @@ --- + +# Bug 1456357 - Do **NOT** consider adding this line back in, it causes +# github.com/xeipuuv/gojsonschema to attempt to download the schema which could +# be out-of-sync with this version which is burned into the release. +# +# ### DO NOT ADD THIS!! -> id: https://schemas.taskcluster.net/generic-worker/v1/payload.json# + "$schema": http://json-schema.org/draft-04/schema# title: Generic worker payload description: |- From f9eada58d11b9dc0eec0969c065d9a5981776b46 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 13:01:34 +0200 Subject: [PATCH 24/58] wip --- mozilla/OpenCloudConfig/generate_occ_userdata.sh | 13 ++++++++++--- mozilla/OpenCloudConfig/transform-occ/main.go | 5 +++++ mozilla/OpenCloudConfig/transform.sh | 5 ++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/mozilla/OpenCloudConfig/generate_occ_userdata.sh b/mozilla/OpenCloudConfig/generate_occ_userdata.sh index c69d7952..99194bd3 100755 --- a/mozilla/OpenCloudConfig/generate_occ_userdata.sh +++ b/mozilla/OpenCloudConfig/generate_occ_userdata.sh @@ -1,8 +1,15 @@ #!/bin/bash -e -cd "$(dirname "${0}")" +THIS_SCRIPT_DIR="$(dirname "${0}")" + +# Default directory to look for definitions is current directory. +# To select a different directory, simply export WORKER_TYPES_DIR +# to chosen directory before calling this script. +export WORKER_TYPES_DIR=${WORKER_TYPES_DIR:-.} + +go install ./transform-occ echo "Removing..." -rm -v gecko-*/userdata +rm -v "${WORKER_TYPES_DIR}"/gecko-*/userdata echo "Generating..." -curl -L 'https://github.com/mozilla-releng/OpenCloudConfig/tree/master/userdata/Manifest' 2>/dev/null | sed -n 's/.*\(gecko[^.]*\)\.json.*/\1/p' | sort -u | xargs -n 1 -P 32 ./transform.sh +curl -L 'https://github.com/mozilla-releng/OpenCloudConfig/tree/master/userdata/Manifest' 2>/dev/null | sed -n 's/.*\(gecko[^.]*\)\.json.*/\1/p' | sort -u | xargs -n 1 -P 32 "${THIS_SCRIPT_DIR}/transform.sh" diff --git a/mozilla/OpenCloudConfig/transform-occ/main.go b/mozilla/OpenCloudConfig/transform-occ/main.go index a31debff..fcff093f 100644 --- a/mozilla/OpenCloudConfig/transform-occ/main.go +++ b/mozilla/OpenCloudConfig/transform-occ/main.go @@ -98,6 +98,11 @@ func main() { fmt.Println(``) fmt.Println(``) + fmt.Println(`# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation`) + fmt.Println(`# steps that actually occur on this worker type. It is generated by`) + fmt.Println(`# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh`) + fmt.Println(`# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/` + workerType + `.json`) + fmt.Println(``) fmt.Println(`# use TLS 1.2 (see bug 1443595)`) fmt.Println(`[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12`) fmt.Println(``) diff --git a/mozilla/OpenCloudConfig/transform.sh b/mozilla/OpenCloudConfig/transform.sh index c420d905..f5046df3 100755 --- a/mozilla/OpenCloudConfig/transform.sh +++ b/mozilla/OpenCloudConfig/transform.sh @@ -1,6 +1,5 @@ #!/bin/bash -e -cd "$(dirname "${0}")" echo "${1}" -mkdir -p "${1}" -go run transform-occ/main.go "${1}" > "${1}/userdata" || rm "${1}/userdata" +mkdir -p "${WORKER_TYPES_DIR}/${1}" +"$(go env GOPATH)/bin/transform-occ" "${1}" > "${WORKER_TYPES_DIR}/${1}/userdata" || rm "${WORKER_TYPES_DIR}/${1}/userdata" From 76c0c6b63ae06a574c0724d8ea907f566f375163 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 13:03:57 +0200 Subject: [PATCH 25/58] updated gecko worker type definitions --- .../aws-provisioner-v1/gecko-1-b-win2012-beta/userdata | 5 +++++ .../aws-provisioner-v1/gecko-1-b-win2012/userdata | 5 +++++ .../aws-provisioner-v1/gecko-2-b-win2012/userdata | 5 +++++ .../aws-provisioner-v1/gecko-3-b-win2012/userdata | 5 +++++ .../aws-provisioner-v1/gecko-t-win10-64-beta/userdata | 7 ++++++- .../aws-provisioner-v1/gecko-t-win10-64-cu/userdata | 5 +++++ .../aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata | 7 ++++++- .../aws-provisioner-v1/gecko-t-win10-64-gpu/userdata | 5 +++++ .../aws-provisioner-v1/gecko-t-win10-64/userdata | 5 +++++ .../aws-provisioner-v1/gecko-t-win7-32-beta/userdata | 7 ++++++- .../aws-provisioner-v1/gecko-t-win7-32-cu/userdata | 5 +++++ .../aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata | 7 ++++++- .../aws-provisioner-v1/gecko-t-win7-32-gpu/userdata | 5 +++++ .../aws-provisioner-v1/gecko-t-win7-32/userdata | 5 +++++ 14 files changed, 74 insertions(+), 4 deletions(-) diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata index 1a1d7d64..03d8a612 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-1-b-win2012-beta.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata index 1a1d7d64..7fce39f9 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-1-b-win2012.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata index 1a1d7d64..d26a7181 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-2-b-win2012.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata index f42c5300..d9abbdb4 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-3-b-win2012.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata index c65486a4..cd8d432e 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-beta.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -360,7 +365,7 @@ $client.DownloadFile("http://hg.mozilla.org/mozilla-central/raw-file/360ab7771e2 Start-Process "mofcomp" -ArgumentList "`"C:\Program Files (x86)\Windows Kits\10\Windows Performance Toolkit\mozprofilerprobe.mof`"" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata index e1dad0ce..e0d50019 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-cu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata index 34699249..58af0f7e 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-gpu-b.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -367,7 +372,7 @@ $client.DownloadFile("http://hg.mozilla.org/mozilla-central/raw-file/360ab7771e2 Start-Process "mofcomp" -ArgumentList "`"C:\Program Files (x86)\Windows Kits\10\Windows Performance Toolkit\mozprofilerprobe.mof`"" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata index b2468767..7a79e152 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-gpu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata index e1dad0ce..50412eb4 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata index c3557c08..d03627ec 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-beta.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -354,7 +359,7 @@ $client.DownloadFile("http://download.microsoft.com/download/4/c/b/4cbd5757-0dd4 Start-Process "C:\binaries\vc_redist.x64.exe" -ArgumentList "/install /passive /norestart /log C:\log\vcredist_vs2015_x64-install.log" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata index c3557c08..8d5680ea 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-cu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata index c3557c08..82461557 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-gpu-b.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -354,7 +359,7 @@ $client.DownloadFile("http://download.microsoft.com/download/4/c/b/4cbd5757-0dd4 Start-Process "C:\binaries\vc_redist.x64.exe" -ArgumentList "/install /passive /norestart /log C:\log\vcredist_vs2015_x64-install.log" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata index c3557c08..bdd8817d 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-gpu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata index cb26e1c0..aa51df70 100644 --- a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata +++ b/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 From a05fd7163306a2338ec763015f2912ef45d1660f Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 13:37:05 +0200 Subject: [PATCH 26/58] wip --- .../aws-provisioner-v1/ami-test-win2012r2/ami-base-name | 0 .../aws-provisioner-v1/ami-test-win2012r2/userdata | 0 .../aws-provisioner-v1/ami-test-win7sp1/ami-base-name | 0 .../aws-provisioner-v1/ami-test-win7sp1/userdata | 0 .../aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name | 0 .../aws-provisioner-v1/gecko-1-b-win2012-beta/userdata | 0 .../aws-provisioner-v1/gecko-1-b-win2012/ami-base-name | 0 .../aws-provisioner-v1/gecko-1-b-win2012/userdata | 0 .../aws-provisioner-v1/gecko-2-b-win2012/ami-base-name | 0 .../aws-provisioner-v1/gecko-2-b-win2012/userdata | 0 .../aws-provisioner-v1/gecko-3-b-win2012/ami-base-name | 0 .../aws-provisioner-v1/gecko-3-b-win2012/userdata | 0 .../aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win10-64-beta/userdata | 0 .../aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win10-64-cu/userdata | 0 .../aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata | 0 .../aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win10-64-gpu/userdata | 0 .../aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win10-64/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win10-64/userdata | 0 .../aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win7-32-beta/userdata | 0 .../aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win7-32-cu/userdata | 0 .../aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata | 0 .../aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win7-32-gpu/userdata | 0 .../aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win7-32/ami-base-name | 0 .../aws-provisioner-v1/gecko-t-win7-32/userdata | 0 .../aws-provisioner-v1/nss-win2012r2-new/ami-base-name | 0 .../aws-provisioner-v1/nss-win2012r2-new/userdata | 0 .../aws-provisioner-v1/nss-win2012r2/ami-base-name | 0 .../aws-provisioner-v1/nss-win2012r2/userdata | 0 .../aws-provisioner-v1/win2012r2-cu/ami-base-name | 0 .../aws-provisioner-v1/win2012r2-cu/userdata | 0 .../aws-provisioner-v1/win2012r2/ami-base-name | 0 .../aws-provisioner-v1/win2012r2/userdata | 0 .../aws-provisioner-v1/win2016/ami-base-name | 0 .../aws-provisioner-v1/win2016/userdata | 0 44 files changed, 0 insertions(+), 0 deletions(-) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/ami-test-win2012r2/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/ami-test-win2012r2/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/ami-test-win7sp1/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/ami-test-win7sp1/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-1-b-win2012/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-2-b-win2012/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-3-b-win2012/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-beta/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-cu/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win10-64/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-beta/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-cu/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/gecko-t-win7-32/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/nss-win2012r2-new/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/nss-win2012r2-new/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/nss-win2012r2/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/nss-win2012r2/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/win2012r2-cu/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/win2012r2-cu/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/win2012r2/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/win2012r2/userdata (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/win2016/ami-base-name (100%) rename mozilla/{worker-type-definitions => worker-type-host-definitions}/aws-provisioner-v1/win2016/userdata (100%) diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/nss-win2012r2/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2-cu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/win2012r2/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/userdata diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/win2016/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/ami-base-name similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/win2016/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/ami-base-name diff --git a/mozilla/worker-type-definitions/aws-provisioner-v1/win2016/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/userdata similarity index 100% rename from mozilla/worker-type-definitions/aws-provisioner-v1/win2016/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/userdata From becd6c07ce14f7e1f68c55b1ecded72f391ac71f Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 13:44:10 +0200 Subject: [PATCH 27/58] wip --- .../main.go | 0 aws/scripts/cleanup_ec2.sh | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename aws/cmd/{aws-worker-types => download-aws-worker-type-definitions}/main.go (100%) diff --git a/aws/cmd/aws-worker-types/main.go b/aws/cmd/download-aws-worker-type-definitions/main.go similarity index 100% rename from aws/cmd/aws-worker-types/main.go rename to aws/cmd/download-aws-worker-type-definitions/main.go diff --git a/aws/scripts/cleanup_ec2.sh b/aws/scripts/cleanup_ec2.sh index d656c38c..b4d40be3 100755 --- a/aws/scripts/cleanup_ec2.sh +++ b/aws/scripts/cleanup_ec2.sh @@ -2,7 +2,7 @@ cd "$(dirname "${0}")" -go get github.com/taskcluster/generic-worker/cmd/aws-worker-types -"$(go env GOPATH)/bin/aws-worker-types" > /dev/null +go get github.com/taskcluster/generic-worker/aws/cmd/download-aws-worker-type-definitions +"$(go env GOPATH)/bin/download-aws-worker-type-definitions" > /dev/null cat aws-provisioner-v1-worker-type-definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u rm -r aws-provisioner-v1-worker-type-definitions From ba866993c31aea509e48ffb9cc6a9db1996d3407 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 13:56:43 +0200 Subject: [PATCH 28/58] fix nss scripts to work with new directory layout --- mozilla/nss/nss-try.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mozilla/nss/nss-try.sh b/mozilla/nss/nss-try.sh index 49cd604a..6c12b7b6 100755 --- a/mozilla/nss/nss-try.sh +++ b/mozilla/nss/nss-try.sh @@ -19,7 +19,7 @@ function open_browser_page { cd "$(dirname "${0}")" THIS_SCRIPT_DIR="$(pwd)" -NEW_VERSION="$(cat ../worker_types/nss-win2012r2-new/userdata | sed -n 's_.*https://github\.com/taskcluster/generic-worker/releases/download/v\(.*\)/generic-worker-windows-amd64\.exe.*_\1_p')" +NEW_VERSION="$(cat ../worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata | sed -n 's_.*https://github\.com/taskcluster/generic-worker/releases/download/v\(.*\)/generic-worker-windows-amd64\.exe.*_\1_p')" VALID_FORMAT='^[1-9][0-9]*\.\(0\|[1-9][0-9]*\)\.\(0\|[1-9]\)\([0-9]*alpha[1-9][0-9]*\|[0-9]*\)$' if ! echo "${NEW_VERSION}" | grep -q "${VALID_FORMAT}"; then @@ -27,7 +27,8 @@ if ! echo "${NEW_VERSION}" | grep -q "${VALID_FORMAT}"; then exit 65 fi -../worker_types/worker_type.sh nss-win2012r2-new update +export WORKER_TYPES_DIR=../worker-type-host-definitions/aws-provisioner-v1 +../../aws/update-worker-types/worker_type.sh nss-win2012r2-new update NSS_CHECKOUT="$(mktemp -d -t nss-checkout.XXXXXXXXXX)" cd "${NSS_CHECKOUT}" From 5bc9bf997b1e798d36fc14ab74deddda9fb3381c Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 14:45:44 +0200 Subject: [PATCH 29/58] fix nss script --- mozilla/nss/nss-try.sh | 7 ++++++- mozilla/nss/nss.patch | 17 ----------------- 2 files changed, 6 insertions(+), 18 deletions(-) delete mode 100644 mozilla/nss/nss.patch diff --git a/mozilla/nss/nss-try.sh b/mozilla/nss/nss-try.sh index 6c12b7b6..b2a3a935 100755 --- a/mozilla/nss/nss-try.sh +++ b/mozilla/nss/nss-try.sh @@ -34,7 +34,12 @@ NSS_CHECKOUT="$(mktemp -d -t nss-checkout.XXXXXXXXXX)" cd "${NSS_CHECKOUT}" hg clone https://hg.mozilla.org/projects/nss cd nss -patch -p1 -i "${THIS_SCRIPT_DIR}/nss.patch" +grep -rl nss-win2012r2 . | while read FILE +do + cp "${FILE}" "${FILE}.x" + cat "${FILE}.x" | sed 's/nss-win2012r2/nss-win2012r2-new/g' > "${FILE}" + rm "${FILE}.x" +done hg commit -m "Testing generic-worker ${NEW_VERSION} on nss-win2012r2-new worker type; try: -p win32,win64 -t none -u all" hg push -f ssh://hg.mozilla.org/projects/nss-try -r . diff --git a/mozilla/nss/nss.patch b/mozilla/nss/nss.patch deleted file mode 100644 index 1ed2a2a3..00000000 --- a/mozilla/nss/nss.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js ---- a/automation/taskcluster/graph/src/extend.js -+++ b/automation/taskcluster/graph/src/extend.js -@@ -595,11 +595,11 @@ async function scheduleTestBuilds(base, - - /*****************************************************************************/ - - async function scheduleWindows(name, base, build_script) { - base = merge(base, { -- workerType: "nss-win2012r2", -+ workerType: "nss-win2012r2-new", - env: { - PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" + - "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" + - "c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;" + - "c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;" + - From a02d5e00f9ac9c78a92cf6ce917127b02aca00bc Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 14:54:44 +0200 Subject: [PATCH 30/58] fix --- scripts/refresh_readme.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/refresh_readme.sh b/scripts/refresh_readme.sh index 1f11929c..35236601 100755 --- a/scripts/refresh_readme.sh +++ b/scripts/refresh_readme.sh @@ -4,7 +4,7 @@ VALID_FORMAT='[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*' TEMP_GW_HELP="$(mktemp -t generic-worker-help-text.XXXXXXXXXX)" TEMP_GW_README="$(mktemp -t generic-worker-readme.XXXXXXXXXX)" TEMP_GW_BINARY="$(mktemp -t generic-worker.XXXXXXXXXX)" -go build ./cmd/generic-worker -o "${TEMP_GW_BINARY}" +go build -o "${TEMP_GW_BINARY}" ./cmd/generic-worker "${TEMP_GW_BINARY}" --help > "${TEMP_GW_HELP}" echo '```' >> "${TEMP_GW_HELP}" sed -e " From 6262423cef5fb321666c455710acf5db3d176ca0 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 15:01:23 +0200 Subject: [PATCH 31/58] fix --- aws/cmd/update-worker-type/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/cmd/update-worker-type/main.go b/aws/cmd/update-worker-type/main.go index 207ea653..2783fc0a 100644 --- a/aws/cmd/update-worker-type/main.go +++ b/aws/cmd/update-worker-type/main.go @@ -112,7 +112,7 @@ func main() { oldMetadata := config["workerTypeMetadata"].(map[string]interface{}) oldMachineSetup := oldMetadata["machine-setup"].(map[string]interface{}) oldScript := oldMachineSetup["script"].(string) - newScript := "https://raw.githubusercontent.com/taskcluster/generic-worker/" + gitRevision(workerTypeDir) + "/worker_types/" + workerType + "/userdata" + newScript := "https://raw.githubusercontent.com/taskcluster/generic-worker/" + gitRevision(workerTypeDir) + "/mozilla/worker-type-host-definitions/aws-provisioner-v1/" + workerType + "/userdata" oldMachineSetup["script"] = newScript log.Print("Old machine setup script: " + oldScript) From 4ea5bce556a0e9c1b72f6e8de9fc688f9d7500e0 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 15:26:19 +0200 Subject: [PATCH 32/58] fix --- scripts/gotestcover.sh | 3 +-- scripts/myreport.txt | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 scripts/myreport.txt diff --git a/scripts/gotestcover.sh b/scripts/gotestcover.sh index 33dee6cd..26215a31 100755 --- a/scripts/gotestcover.sh +++ b/scripts/gotestcover.sh @@ -5,14 +5,13 @@ if [ -z "${REPORT}" ]; then echo "Specify a report, e.g. '${0}' myreport.txt" >&2 exit 64 fi -cd "$(dirname "${0}")"/.. TEMP_SINGLE_REPORT="$(mktemp -t coverage.tmp.XXXXXXXXXX)" echo "mode: atomic" > "${REPORT}" HEAD_REV="$(git rev-parse HEAD)" # Dump package list to file rather than pipe, to avoid exit inside loop not # causing outer shell to exit due to running in a subshell. PACKAGE_LIST="$(mktemp -t package-list.tmp.XXXXXXXXXX)" -go list ./... > "${PACKAGE_LIST}" +go list ../... > "${PACKAGE_LIST}" while read package do diff --git a/scripts/myreport.txt b/scripts/myreport.txt new file mode 100644 index 00000000..79b28a0b --- /dev/null +++ b/scripts/myreport.txt @@ -0,0 +1 @@ +mode: atomic From 489738acb66917c1ffe2807485867a61f8229a24 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 15:46:50 +0200 Subject: [PATCH 33/58] wip --- mozilla/nss/nss-try.sh | 2 +- scripts/myreport.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 scripts/myreport.txt diff --git a/mozilla/nss/nss-try.sh b/mozilla/nss/nss-try.sh index b2a3a935..03c867a4 100755 --- a/mozilla/nss/nss-try.sh +++ b/mozilla/nss/nss-try.sh @@ -40,7 +40,7 @@ do cat "${FILE}.x" | sed 's/nss-win2012r2/nss-win2012r2-new/g' > "${FILE}" rm "${FILE}.x" done -hg commit -m "Testing generic-worker ${NEW_VERSION} on nss-win2012r2-new worker type; try: -p win32,win64 -t none -u all" +hg commit -m "Testing generic-worker ${NEW_VERSION} on nss-win2012r2-new worker type; try: -p win,win64 -t none -u all" hg push -f ssh://hg.mozilla.org/projects/nss-try -r . open_browser_page 'https://treeherder.mozilla.org/#/jobs?repo=nss-try' diff --git a/scripts/myreport.txt b/scripts/myreport.txt deleted file mode 100644 index 79b28a0b..00000000 --- a/scripts/myreport.txt +++ /dev/null @@ -1 +0,0 @@ -mode: atomic From 806e694ab7239721c4d3d9af46157d7beacd6501 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 15:52:26 +0200 Subject: [PATCH 34/58] try this --- cmd/generic-worker/plat_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/generic-worker/plat_windows.go b/cmd/generic-worker/plat_windows.go index 533cac82..766e3b15 100644 --- a/cmd/generic-worker/plat_windows.go +++ b/cmd/generic-worker/plat_windows.go @@ -59,7 +59,7 @@ func (task *TaskRun) NewPlatformData() (pd *PlatformData, err error) { if filepath.Base(os.Args[0]) == "generic-worker.exe" { exe = os.Args[0] } else { - exe = `..\..\..\..\bin\generic-worker.exe` + exe = `..\..\..\..\..\..\bin\generic-worker.exe` } cmd, err := process.NewCommand([]string{exe, "grant-winsta-access", "--sid", sid}, cwd, []string{}, pd.LoginInfo.AccessToken()) cmd.DirectOutput(os.Stdout) From a0773c5e3eb411564ecef0d6c616b4fb5319c3dc Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 17:11:52 +0200 Subject: [PATCH 35/58] fix --- scripts/gotestcover.sh | 2 +- sdsd | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 sdsd diff --git a/scripts/gotestcover.sh b/scripts/gotestcover.sh index 26215a31..d35c539e 100755 --- a/scripts/gotestcover.sh +++ b/scripts/gotestcover.sh @@ -11,7 +11,7 @@ HEAD_REV="$(git rev-parse HEAD)" # Dump package list to file rather than pipe, to avoid exit inside loop not # causing outer shell to exit due to running in a subshell. PACKAGE_LIST="$(mktemp -t package-list.tmp.XXXXXXXXXX)" -go list ../... > "${PACKAGE_LIST}" +go list github.com/taskcluster/generic-worker/... > "${PACKAGE_LIST}" while read package do diff --git a/sdsd b/sdsd new file mode 100644 index 00000000..79b28a0b --- /dev/null +++ b/sdsd @@ -0,0 +1 @@ +mode: atomic From b62ef9eac6c25772522499dc442a58c531624fee Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 17:55:22 +0200 Subject: [PATCH 36/58] wip --- sdsd | 1 - 1 file changed, 1 deletion(-) delete mode 100644 sdsd diff --git a/sdsd b/sdsd deleted file mode 100644 index 79b28a0b..00000000 --- a/sdsd +++ /dev/null @@ -1 +0,0 @@ -mode: atomic From a252755aac05403cff6801be84a0804dc23d665f Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 18:37:24 +0200 Subject: [PATCH 37/58] Started updating README.md --- README.md | 69 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 997e6b47..a8536c46 100644 --- a/README.md +++ b/README.md @@ -8,25 +8,80 @@ [![Coverage Status](https://coveralls.io/repos/taskcluster/generic-worker/badge.svg?branch=master&service=github)](https://coveralls.io/github/taskcluster/generic-worker?branch=master) [![License](https://img.shields.io/badge/license-MPL%202.0-orange.svg)](http://mozilla.org/MPL/2.0) - - -# Install binary +Generic worker is a Windows/Linux/macOS program for executing taskcluster +tasks. It communicates with the taskcluster Queue as per the [Queue-Worker +Interaction +specification](https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/docs/worker-interaction). +It is shipped as a statically linked system-native executable. + +# Payload format + +# Execution policy + +## Windows +### System calls (KnownFolders, generating processes, ...) +## Mac +No isolation +## Linux +No isolation - working on docker + +## Redeployability + +## Integrating with AWS / GCE + +## Config bootstrapping + +# "Bring your own worker": Running your own generic-worker workers +## Mac +### Installing + +## Windows +### Installing +## Linux - Docker +## Linux - Native +### Installing +# Administrative tools +## Displaying workers +# Worker Type Host Definitions +## Updating existing definitions +## Modifying definitions +## Creating your own AWS workers outside of this repo +## Puppet +# Developing Generic Worker +## Fetching source +## Credentials +## Running unit tests +## Writing unit tests +## Including bug numbers in comments + +# Releasing Generic Worker +## Release script +## Publishing schemas +## Testing in Staging +## Rolling out to Production +## Writing release notes (README.md, release page, ...) +# Repository layout + + + +# Downloading generic-worker binary release * Download the latest release for your platform from https://github.com/taskcluster/generic-worker/releases * Download the latest release of livelog for your platform from https://github.com/taskcluster/livelog/releases * For darwin/linux, make the binaries executable: `chmod a+x {generic-worker,livelog}*` -# Build from source +# Building generic-worker from source If you prefer not to use a prepackaged binary, or want to have the latest unreleased version from the development head: * Head over to https://golang.org/dl/ and follow the instructions for your platform. __Note, go 1.8 or higher is required__. -* Run `go get github.com/taskcluster/generic-worker` -* Run `go get github.com/taskcluster/livelog` +* Run `./build.sh` (macOS/Linux) or `.\build.cmd` (Windows) All being well, the binaries will be built under `$(go env GOPATH)/bin`. -# Acquire taskcluster credentials for running tests + + +# Acquire taskcluster credentials for running code tests There are two alternative mechanisms to acquire the scopes you need. From fb1ce0a37201204d1a659a6858abfbf3358a6421 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 18:53:22 +0200 Subject: [PATCH 38/58] Updated docs --- docs/payload.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/payload.md b/docs/payload.md index 641b0d96..f853b802 100644 --- a/docs/payload.md +++ b/docs/payload.md @@ -10,21 +10,22 @@ docref: true order: 1 --- -When submitting a task graph to the Task Cluster Queue (see -[createTask](/reference/platform/queue/reference/api-docs#createTask)) you must provide a -payload for defining the tasks to be executed by the worker. In the case of the -generic worker, the payload must conform to the following schema. +When submitting a task to the Taskcluster Queue (see +[createTask](/reference/platform/queue/reference/api-docs#createTask)) you must +provide a payload property for the task. This `payload` property is specific to +the worker implementation, and tells the worker what to execute, and which +artifacts to upload. This page documents the payload property for +generic-worker, on all the platforms that it is supported on. - -# Windows +# generic-worker on Windows
-# macOS +# generic-worker on macOS
-# Linux +# generic-worker on Linux
From c94d9f2fcd2c82e556926271f6ecfb4d84432664 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 19:48:28 +0200 Subject: [PATCH 39/58] wip --- README.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a8536c46..a9571fd1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -# generic-worker - A generic worker for [taskcluster](https://tools.taskcluster.net/), written in go. [![Taskcluster CI Status](https://github.taskcluster.net/v1/repository/taskcluster/generic-worker/master/badge.svg)](https://github.taskcluster.net/v1/repository/taskcluster/generic-worker/master/latest) @@ -8,12 +6,34 @@ [![Coverage Status](https://coveralls.io/repos/taskcluster/generic-worker/badge.svg?branch=master&service=github)](https://coveralls.io/github/taskcluster/generic-worker?branch=master) [![License](https://img.shields.io/badge/license-MPL%202.0-orange.svg)](http://mozilla.org/MPL/2.0) +# Introdution + Generic worker is a Windows/Linux/macOS program for executing taskcluster tasks. It communicates with the taskcluster Queue as per the [Queue-Worker Interaction specification](https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/docs/worker-interaction). It is shipped as a statically linked system-native executable. +The mechanism by which it achieves a sandbox is platform-specific, and is +explained later. + +## Imperative task payloads + +Generic worker allows tasks to run arbitrary command payloads in a sandbox. If +you are looking for a worker to run only specific commands on a privileged +environment, see +[scriptworker](https://github.com/mozilla-releng/scriptworker). + +## Docker support + +Work is underway to provide support for running generic-worker tasks inside a +docker container isolated from the host environment. However until this work is +complete, please see +[docker-worker](https://github.com/taskcluster/docker-worker) for achieving +this. + +For running + # Payload format # Execution policy From af9998134b960daef2d0cc4fa5ffb9d71cfd5a71 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 20:20:03 +0200 Subject: [PATCH 40/58] wip --- README.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a9571fd1..c02da535 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,6 @@ Interaction specification](https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/docs/worker-interaction). It is shipped as a statically linked system-native executable. -The mechanism by which it achieves a sandbox is platform-specific, and is -explained later. - ## Imperative task payloads Generic worker allows tasks to run arbitrary command payloads in a sandbox. If @@ -24,7 +21,51 @@ you are looking for a worker to run only specific commands on a privileged environment, see [scriptworker](https://github.com/mozilla-releng/scriptworker). -## Docker support +## Sandboxing + +The mechanism for providing a sandbox for a task to run in, is platform-specific. + +### Windows + +On Windows, `generic-worker.exe` runs in a Windows Service under the +[LocalSystem](https://docs.microsoft.com/en-us/windows/desktop/services/localsystem-account) +account. This service creates a fresh new (non-admin) OS user in preparation +for running a task. The worker configures the machine to automatically log in +as the newly created task user, and then triggers the machine to reboot. Once +the machine reboots, the worker running in the Windows Service waits until it +detects that the Operating System [winlogon +module](https://docs.microsoft.com/en-us/windows/desktop/secauthn/winlogon-and-credential-providers) +has completed the interactive logon of the task user. At this point it polls +the taskcluster Queue to fetch a task to execute, and when it is given one, it +executes this task in the interactive logon session of the logged-in user, +running processes using the auth token obtained from the interactive desktop +session. + +After the task completes, the home directory of the task user, and the task +directory (if different to the home directory of the task user) are erased, a +new task user is created, the machine is rebooted, and the former task user is +purged. + +In many ways, this operation can be alikened to providing a guest account on +the machine. After the task has completed, all trace of the changes made by the +task user should be gone, and the machine's state should be reset to the state +it had before the task was run. If the host environment is sufficiently locked +down, the task user should not have been able to apply any state-change to the +host environment. Please note that the worker has limited control to affect +system-wide policy, so for example if a host allows arbtirary users to write to +a system folder location, the worker is not able to prevent a task doing so. +Therefore it is up to the machine provider to ensure that the host is +sufficiently locked-down. Host environments for long-lived workers that are to +run untrusted tasks should be secured carefully, to prevent that tasks may +interfere with system state or persist changes across task runs that may affect +the reproducibility of a task, or worse, introduce a security issue. + + +### Linux + +There is no native sandbox support currently on Linux. Currently the worker +will execute tasks as the same user that the worker runs as. Use at your own +risk! Work is underway to provide support for running generic-worker tasks inside a docker container isolated from the host environment. However until this work is @@ -32,7 +73,19 @@ complete, please see [docker-worker](https://github.com/taskcluster/docker-worker) for achieving this. -For running +We may, at some point, provide OS-user sandboxing, akin to the Windows +implementation. + + +### macOS + +There is no native sandbox support currently on macOS. Currently the worker +will execute tasks as the same user that the worker runs as. Use at your own +risk! + +We intend to provide OS-user sandboxing, akin to the Windows implementation, at +some point in the future. + # Payload format From 59596b8708647184e616ecb7f6d347c1c584b60a Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 20:31:24 +0200 Subject: [PATCH 41/58] wip --- README.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/README.md b/README.md index c02da535..2c9dbd2d 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,115 @@ [![Coverage Status](https://coveralls.io/repos/taskcluster/generic-worker/badge.svg?branch=master&service=github)](https://coveralls.io/github/taskcluster/generic-worker?branch=master) [![License](https://img.shields.io/badge/license-MPL%202.0-orange.svg)](http://mozilla.org/MPL/2.0) +# Table of Contents + + * [Introdution](#introdution) + * [Imperative task payloads](#imperative-task-payloads) + * [Sandboxing](#sandboxing) + * [Windows](#windows) + * [Linux](#linux) + * [macOS](#macos) + * [Payload format](#payload-format) + * [Execution policy](#execution-policy) + * [Windows](#windows-1) + * [System calls (KnownFolders, generating processes, ...)](#system-calls-knownfolders-generating-processes-) + * [Mac](#mac) + * [Linux](#linux-1) + * [Redeployability](#redeployability) + * [Integrating with AWS / GCE](#integrating-with-aws--gce) + * [Config bootstrapping](#config-bootstrapping) + * ["Bring your own worker": Running your own generic-worker workers](#bring-your-own-worker-running-your-own-generic-worker-workers) + * [Mac](#mac-1) + * [Installing](#installing) + * [Windows](#windows-2) + * [Installing](#installing-1) + * [Linux - Docker](#linux---docker) + * [Linux - Native](#linux---native) + * [Installing](#installing-2) + * [Administrative tools](#administrative-tools) + * [Displaying workers](#displaying-workers) + * [Worker Type Host Definitions](#worker-type-host-definitions) + * [Updating existing definitions](#updating-existing-definitions) + * [Modifying definitions](#modifying-definitions) + * [Creating your own AWS workers outside of this repo](#creating-your-own-aws-workers-outside-of-this-repo) + * [Puppet](#puppet) + * [Developing Generic Worker](#developing-generic-worker) + * [Fetching source](#fetching-source) + * [Credentials](#credentials) + * [Running unit tests](#running-unit-tests) + * [Writing unit tests](#writing-unit-tests) + * [Including bug numbers in comments](#including-bug-numbers-in-comments) + * [Releasing Generic Worker](#releasing-generic-worker) + * [Release script](#release-script) + * [Publishing schemas](#publishing-schemas) + * [Testing in Staging](#testing-in-staging) + * [Rolling out to Production](#rolling-out-to-production) + * [Writing release notes (README.md, release page, ...)](#writing-release-notes-readmemd-release-page-) + * [Repository layout](#repository-layout) + * [Downloading generic-worker binary release](#downloading-generic-worker-binary-release) + * [Building generic-worker from source](#building-generic-worker-from-source) + * [Acquire taskcluster credentials for running code tests](#acquire-taskcluster-credentials-for-running-code-tests) + * [Option 1](#option-1) + * [Option 2](#option-2) + * [Set up your env](#set-up-your-env) + * [Start the generic worker](#start-the-generic-worker) + * [Create a test job](#create-a-test-job) + * [Run the generic worker test suite](#run-the-generic-worker-test-suite) + * [Making a new generic worker release](#making-a-new-generic-worker-release) + * [Creating and updating worker types](#creating-and-updating-worker-types) + * [Release notes](#release-notes) + * [In v10.11.3 since v10.11.2](#in-v10113-since-v10112) + * [In v10.11.2 since v10.11.1](#in-v10112-since-v10111) + * [In v10.11.1 since v10.11.1alpha1](#in-v10111-since-v10111alpha1) + * [In v10.10.0 since v10.9.0](#in-v10100-since-v1090) + * [In v10.8.5 since v10.8.4](#in-v1085-since-v1084) + * [In v10.8.4 since v10.8.3](#in-v1084-since-v1083) + * [In v10.8.2 since v10.8.1](#in-v1082-since-v1081) + * [In v10.8.0 since v10.7.12](#in-v1080-since-v10712) + * [In v10.7.12 since v10.7.11](#in-v10712-since-v10711) + * [In v10.7.11 since v10.7.10](#in-v10711-since-v10710) + * [In v10.7.10 since v10.7.9](#in-v10710-since-v1079) + * [In v10.7.6 since v10.7.5](#in-v1076-since-v1075) + * [In v10.7.3 since v10.7.2](#in-v1073-since-v1072) + * [In v10.6.1 since v10.6.0](#in-v1061-since-v1060) + * [In v10.6.0 since v10.5.1](#in-v1060-since-v1051) + * [In v10.5.1 since v10.5.0](#in-v1051-since-v1050) + * [In v10.5.0 since v10.5.0alpha4](#in-v1050-since-v1050alpha4) + * [In v10.4.1 since v10.4.0](#in-v1041-since-v1040) + * [In v10.4.0 since v10.3.1](#in-v1040-since-v1031) + * [In v10.3.0 since v10.2.3](#in-v1030-since-v1023) + * [In v10.2.3 since v10.2.2](#in-v1023-since-v1022) + * [In v10.2.2 since v10.2.1](#in-v1022-since-v1021) + * [In v10.2.1 since v10.2.0](#in-v1021-since-v1020) + * [In v10.2.0 since v10.1.8](#in-v1020-since-v1018) + * [In v10.1.8 since v10.1.7](#in-v1018-since-v1017) + * [In v10.1.7 since v10.1.6](#in-v1017-since-v1016) + * [In v10.1.6 since v10.1.5](#in-v1016-since-v1015) + * [In v10.1.5 since v10.1.4](#in-v1015-since-v1014) + * [In v10.1.4 since v10.1.3](#in-v1014-since-v1013) + * [In v10.0.5 since v10.0.4](#in-v1005-since-v1004) + * [In v8.5.0 since v8.4.1](#in-v850-since-v841) + * [In v8.3.0 since v8.2.0](#in-v830-since-v820) + * [In v8.2.0 since v8.1.1](#in-v820-since-v811) + * [In v8.1.0 since v8.0.1](#in-v810-since-v801) + * [In v8.0.1 since v8.0.0](#in-v801-since-v800) + * [In v7.2.13 since v7.2.12](#in-v7213-since-v7212) + * [In v7.2.6 since v7.2.5](#in-v726-since-v725) + * [In v7.2.2 since v7.2.1](#in-v722-since-v721) + * [In v7.1.1 since v7.1.0](#in-v711-since-v710) + * [In v7.1.0 since v7.0.3alpha1](#in-v710-since-v703alpha1) + * [In v7.0.3alpha1 since v7.0.2alpha1](#in-v703alpha1-since-v702alpha1) + * [In v6.1.0 since v6.1.0alpha1](#in-v610-since-v610alpha1) + * [In v6.0.0 since v5.4.0](#in-v600-since-v540) + * [In v5.4.0 since v5.3.1](#in-v540-since-v531) + * [In v5.3.0 since v5.2.0](#in-v530-since-v520) + * [In v5.1.0 since v5.0.3](#in-v510-since-v503) + * [In v4.0.0alpha1 since v3.0.0alpha1](#in-v400alpha1-since-v300alpha1) + * [In v3.0.0alpha1 since v2.1.0](#in-v300alpha1-since-v210) + * [In v2.0.0alpha44 since v2.0.0alpha43](#in-v200alpha44-since-v200alpha43) + * [Further information](#further-information) + + # Introdution Generic worker is a Windows/Linux/macOS program for executing taskcluster From b8a753d4d93bb597f56b8412cbe36b0986d8f9e4 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 20:33:50 +0200 Subject: [PATCH 42/58] fix --- README.md | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) diff --git a/README.md b/README.md index 2c9dbd2d..12233fe7 100644 --- a/README.md +++ b/README.md @@ -63,55 +63,6 @@ * [Making a new generic worker release](#making-a-new-generic-worker-release) * [Creating and updating worker types](#creating-and-updating-worker-types) * [Release notes](#release-notes) - * [In v10.11.3 since v10.11.2](#in-v10113-since-v10112) - * [In v10.11.2 since v10.11.1](#in-v10112-since-v10111) - * [In v10.11.1 since v10.11.1alpha1](#in-v10111-since-v10111alpha1) - * [In v10.10.0 since v10.9.0](#in-v10100-since-v1090) - * [In v10.8.5 since v10.8.4](#in-v1085-since-v1084) - * [In v10.8.4 since v10.8.3](#in-v1084-since-v1083) - * [In v10.8.2 since v10.8.1](#in-v1082-since-v1081) - * [In v10.8.0 since v10.7.12](#in-v1080-since-v10712) - * [In v10.7.12 since v10.7.11](#in-v10712-since-v10711) - * [In v10.7.11 since v10.7.10](#in-v10711-since-v10710) - * [In v10.7.10 since v10.7.9](#in-v10710-since-v1079) - * [In v10.7.6 since v10.7.5](#in-v1076-since-v1075) - * [In v10.7.3 since v10.7.2](#in-v1073-since-v1072) - * [In v10.6.1 since v10.6.0](#in-v1061-since-v1060) - * [In v10.6.0 since v10.5.1](#in-v1060-since-v1051) - * [In v10.5.1 since v10.5.0](#in-v1051-since-v1050) - * [In v10.5.0 since v10.5.0alpha4](#in-v1050-since-v1050alpha4) - * [In v10.4.1 since v10.4.0](#in-v1041-since-v1040) - * [In v10.4.0 since v10.3.1](#in-v1040-since-v1031) - * [In v10.3.0 since v10.2.3](#in-v1030-since-v1023) - * [In v10.2.3 since v10.2.2](#in-v1023-since-v1022) - * [In v10.2.2 since v10.2.1](#in-v1022-since-v1021) - * [In v10.2.1 since v10.2.0](#in-v1021-since-v1020) - * [In v10.2.0 since v10.1.8](#in-v1020-since-v1018) - * [In v10.1.8 since v10.1.7](#in-v1018-since-v1017) - * [In v10.1.7 since v10.1.6](#in-v1017-since-v1016) - * [In v10.1.6 since v10.1.5](#in-v1016-since-v1015) - * [In v10.1.5 since v10.1.4](#in-v1015-since-v1014) - * [In v10.1.4 since v10.1.3](#in-v1014-since-v1013) - * [In v10.0.5 since v10.0.4](#in-v1005-since-v1004) - * [In v8.5.0 since v8.4.1](#in-v850-since-v841) - * [In v8.3.0 since v8.2.0](#in-v830-since-v820) - * [In v8.2.0 since v8.1.1](#in-v820-since-v811) - * [In v8.1.0 since v8.0.1](#in-v810-since-v801) - * [In v8.0.1 since v8.0.0](#in-v801-since-v800) - * [In v7.2.13 since v7.2.12](#in-v7213-since-v7212) - * [In v7.2.6 since v7.2.5](#in-v726-since-v725) - * [In v7.2.2 since v7.2.1](#in-v722-since-v721) - * [In v7.1.1 since v7.1.0](#in-v711-since-v710) - * [In v7.1.0 since v7.0.3alpha1](#in-v710-since-v703alpha1) - * [In v7.0.3alpha1 since v7.0.2alpha1](#in-v703alpha1-since-v702alpha1) - * [In v6.1.0 since v6.1.0alpha1](#in-v610-since-v610alpha1) - * [In v6.0.0 since v5.4.0](#in-v600-since-v540) - * [In v5.4.0 since v5.3.1](#in-v540-since-v531) - * [In v5.3.0 since v5.2.0](#in-v530-since-v520) - * [In v5.1.0 since v5.0.3](#in-v510-since-v503) - * [In v4.0.0alpha1 since v3.0.0alpha1](#in-v400alpha1-since-v300alpha1) - * [In v3.0.0alpha1 since v2.1.0](#in-v300alpha1-since-v210) - * [In v2.0.0alpha44 since v2.0.0alpha43](#in-v200alpha44-since-v200alpha43) * [Further information](#further-information) From 3a80e28249105a6989650a6011881e5923b742c5 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 20:36:15 +0200 Subject: [PATCH 43/58] wip --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12233fe7..290c7b71 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Therefore it is up to the machine provider to ensure that the host is sufficiently locked-down. Host environments for long-lived workers that are to run untrusted tasks should be secured carefully, to prevent that tasks may interfere with system state or persist changes across task runs that may affect -the reproducibility of a task, or worse, introduce a security issue. +the reproducibility of a task, or worse, introduce a security vulnerability. ### Linux From e4606eb1b456e97c8b8859c97aaa0ddb9c9a57c4 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 8 Oct 2018 20:37:18 +0200 Subject: [PATCH 44/58] wip --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 290c7b71..67762ea6 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,10 @@ There is no native sandbox support currently on Linux. Currently the worker will execute tasks as the same user that the worker runs as. Use at your own risk! -Work is underway to provide support for running generic-worker tasks inside a -docker container isolated from the host environment. However until this work is -complete, please see -[docker-worker](https://github.com/taskcluster/docker-worker) for achieving +Work is [underway](https://github.com/taskcluster/generic-worker/pull/62) to +provide support for running generic-worker tasks inside a docker container +isolated from the host environment. However until this work is complete, please +see [docker-worker](https://github.com/taskcluster/docker-worker) for achieving this. We may, at some point, provide OS-user sandboxing, akin to the Windows From aadf9638c65eb7282564336695cb33938dd35b87 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Tue, 9 Oct 2018 18:21:33 +0200 Subject: [PATCH 45/58] wip --- README.md | 22 +++++++++++++++++----- docs/payload.md | 11 ++++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 67762ea6..314cd6ed 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,12 @@ # Introdution -Generic worker is a Windows/Linux/macOS program for executing taskcluster -tasks. It communicates with the taskcluster Queue as per the [Queue-Worker -Interaction +Generic worker is a native Windows/Linux/macOS program for executing +taskcluster tasks. It communicates with the taskcluster Queue as per the +[Queue-Worker Interaction specification](https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/docs/worker-interaction). -It is shipped as a statically linked system-native executable. +It is shipped as a statically linked system-native executable. It is written in +go (golang). ## Imperative task payloads @@ -83,7 +84,7 @@ environment, see ## Sandboxing -The mechanism for providing a sandbox for a task to run in, is platform-specific. +The sandbox implementation is platform-specific. ### Windows @@ -149,6 +150,17 @@ some point in the future. # Payload format +Each taskcluster task definition contains a top level property `payload` which +is a json object. The format of this object is specific to the worker +implementation. For generic-worker, this is then also further specific to the +platform (Linux/Windows/macOS). + +The per-platform payload formats are described in json schema, and can be found +in the top level `schemas` subdirectory of this repository. These schemas are +also published to the [generic-worker +page](https://docs.taskcluster.net/docs/reference/workers/generic-worker/docs/payload) +of the taskcluster docs site. + # Execution policy ## Windows diff --git a/docs/payload.md b/docs/payload.md index f853b802..898e92a9 100644 --- a/docs/payload.md +++ b/docs/payload.md @@ -17,15 +17,20 @@ the worker implementation, and tells the worker what to execute, and which artifacts to upload. This page documents the payload property for generic-worker, on all the platforms that it is supported on. -# generic-worker on Windows +* [Windows](#generic-worker-on-windows) +* [macOS](#generic-worker-on-macos) +* [Linux](#generic-worker-on-linux) + + +# Generic Worker on Windows
-# generic-worker on macOS +# Generic Worker on macOS
-# generic-worker on Linux +# Generic Worker on Linux
From d5663ae7ce037f18e283207a5bae8491c2810827 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Tue, 9 Oct 2018 19:34:41 +0200 Subject: [PATCH 46/58] wip --- README.md | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 314cd6ed..0bef9d1e 100644 --- a/README.md +++ b/README.md @@ -177,11 +177,82 @@ No isolation - working on docker ## Config bootstrapping # "Bring your own worker": Running your own generic-worker workers +## Windows +### Installing ## Mac ### Installing -## Windows -### Installing +There currently is no `install` target for macOS, like there is for Windows. + +For our own dedicated macOS workers, we install generic-worker using [this +puppet +module](https://wiki.mozilla.org/ReleaseEngineering/PuppetAgain/Modules/generic_worker). + +You can install generic-worker as a Launch Agent as follows: + +1) Create a regular unprivileged user account on your Mac to run the worker +(e.g. with name `genericworker`) and log into that user account. + +2) Download or build generic-worker, so that you have a native darwin binary, +move it to `/usr/local/bin/generic-worker`, and make sure it is executable for +your new user (`chmod u+x /usr/local/bin/generic-worker`). + +3) Create a signing key in the user home directory by running: + +``` +/usr/local/bin/generic-worker new-openpgp-keypair --file .signingkey + +3) Create `/Library/LaunchAgents/net.generic.worker.plist` with content: + +``` +<%# This Source Code Form is subject to the terms of the Mozilla Public +<%# License, v. 2.0. If a copy of the MPL was not distributed with this +<%# file, You can obtain one at http://mozilla.org/MPL/2.0/. -%> + + + + + Label + net.generic.worker + ProgramArguments + + /usr/local/bin/generic-worker + run + --config + /etc/generic-worker.config + + WorkingDirectory + <-YOUR-NEW-USER-HOME-DIRECTORY-> + RunAtLoad + + + +``` + +4) Create `/etc/generic-worker.config` with content: + +``` +{ + "accessToken": "<-YOUR-TASKCLUSTER-ACCESS-TOKEN->", + "clientId": "<-YOUR-TASKCLUSTER-CLIENT-ID->", + "idleTimeoutSecs": 0, + "livelogSecret": "<-RANDOM-SHORT-STRING-HERE->", + "provisionerBaseURL": "", + "provisionerId": "<-YOUR-PROVISIONER-ID->", + "publicIP": "<-MAKE-UP-AN-IPv4-ADDRESS-IF-YOU-DON'T-HAVE-ONE->", + "signingKeyLocation": ".signingkey", + "tasksDir": "tasks", + "workerGroup": "<-CHOOSE-A-WORKER-GROUP->", + "workerId": "<-CHOOSE-A-WORKER-ID->", + "workerType": "<-YOUR-WORKER-TYPE->", + "workerTypeMetadata": { + <--- add a json blob here with information about you, how you set up the + worker type, etc, so people know how it is configured and maintained, + and who to go to in case of problems ---> + } +} +``` + ## Linux - Docker ## Linux - Native ### Installing @@ -207,6 +278,38 @@ No isolation - working on docker ## Writing release notes (README.md, release page, ...) # Repository layout +``` +├── aws +│   ├── cmd +│   │   ├── download-aws-worker-type-definitions +│   │   ├── gw-workers +│   │   ├── update-ssl-creds +│   │   └── update-worker-type +│   ├── scripts +│   └── update-worker-types +├── cmd +│   ├── generic-worker +│   ├── gw-codegen +│   ├── inspect-worker-types +│   ├── list-worker-types +│   └── yamltojson +├── docs +├── lib +├── mozilla +│   ├── OpenCloudConfig +│   │   ├── occ-workers +│   │   ├── refresh-gw-configs +│   │   └── transform-occ +│   ├── gecko +│   ├── nss +│   └── worker-type-host-definitions +│   └── aws-provisioner-v1 +│   ├── +│   ├── +│   └── ... +├── schemas +└── scripts +``` # Downloading generic-worker binary release From df196dc44688e7a6bd403e7c88199fe2ec08f30a Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Tue, 9 Oct 2018 19:38:38 +0200 Subject: [PATCH 47/58] wip --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0bef9d1e..1b0991ce 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ your new user (`chmod u+x /usr/local/bin/generic-worker`). ``` /usr/local/bin/generic-worker new-openpgp-keypair --file .signingkey +``` 3) Create `/Library/LaunchAgents/net.generic.worker.plist` with content: From 6ab1fdfacc7fb0ef5e001ad90d44337335285854 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Tue, 9 Oct 2018 19:39:22 +0200 Subject: [PATCH 48/58] wip --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 1b0991ce..0a7ff35f 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,6 @@ your new user (`chmod u+x /usr/local/bin/generic-worker`). "clientId": "<-YOUR-TASKCLUSTER-CLIENT-ID->", "idleTimeoutSecs": 0, "livelogSecret": "<-RANDOM-SHORT-STRING-HERE->", - "provisionerBaseURL": "", "provisionerId": "<-YOUR-PROVISIONER-ID->", "publicIP": "<-MAKE-UP-AN-IPv4-ADDRESS-IF-YOU-DON'T-HAVE-ONE->", "signingKeyLocation": ".signingkey", From 5e2220a3ce84f3e6fdfafb36839e92f8a6e9910a Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Wed, 10 Oct 2018 13:36:08 +0200 Subject: [PATCH 49/58] wip --- README.md | 98 ++++++++++++++++++++++++++++++++++++-- lib/win32/win32_windows.go | 1 + 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0a7ff35f..47d98956 100644 --- a/README.md +++ b/README.md @@ -82,11 +82,15 @@ you are looking for a worker to run only specific commands on a privileged environment, see [scriptworker](https://github.com/mozilla-releng/scriptworker). -## Sandboxing +# Sandboxing -The sandbox implementation is platform-specific. +It is important that tasks run in a sandbox in order to that they are as +reproducible as possible, and are not inadvertently affected by previous tasks +that may have run on the same environment. Different operating systems provide +different sandboxing mechanisms, and therefore the approach used by +generic-worker is platform-dependent. -### Windows +## Windows On Windows, `generic-worker.exe` runs in a Windows Service under the [LocalSystem](https://docs.microsoft.com/en-us/windows/desktop/services/localsystem-account) @@ -122,7 +126,11 @@ interfere with system state or persist changes across task runs that may affect the reproducibility of a task, or worse, introduce a security vulnerability. -### Linux + + + + +## Linux There is no native sandbox support currently on Linux. Currently the worker will execute tasks as the same user that the worker runs as. Use at your own @@ -138,7 +146,7 @@ We may, at some point, provide OS-user sandboxing, akin to the Windows implementation. -### macOS +## macOS There is no native sandbox support currently on macOS. Currently the worker will execute tasks as the same user that the worker runs as. Use at your own @@ -147,6 +155,86 @@ risk! We intend to provide OS-user sandboxing, akin to the Windows implementation, at some point in the future. +# Operating System integration + +## Windows + +### Creating a task user + +The generic-worker creates non-privileged task users, with username +`task_` and a random password. Task users are created +with the command: + +``` +net user "" "" /add /expires:never /passwordchg:no /y +``` + +See [net +user](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490718%28v%253dtechnet.10%29) +for more information. + +### Setting Known Folder locations + +It may be desirable for the task directory to be in a different location to the +home directory of the generated user. The location of the home directory is +determined based on system settings defined at installation time of the +operating system, and therefore may not be ideal, especially if the host image +is provided by an external party. + +For example, perhaps the user home directory is `C:\Users\task_` but +for performance reasons, we wish the task directory to be located on a +different physical drive at `Z:\task_`. + +It is possible to configure the location for the task directories (in the above +case, `Z:\`) via the `tasksDir` property of the generic-worker configuration +file. However, this would not affect the location of the [AppData +folder](https://www.howtogeek.com/318177/what-is-the-appdata-folder-in-windows/) +used by Windows applications, which would still be located under the user's +home directory. + +Since it is usually preferable for all user data to be written +to the the task directory, and it isn't trivial to move the user home directory +to an alternative location after the operating system has already been +installed, the worker configures the user account to store the AppData folder +under the task directory. + +It does this as follows: + +1) Calling +[LogonUserW](https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-logonuserw) +to get a logon handle for the new user. +2) Calling +[LoadUserProfileW](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-loaduserprofilew) +to load the user profile. +3) Calling +[SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) +with `KNOWNFOLDERID` +[FOLDERID_RoamingAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) to set the location of `AppData\Roaming` to under the task directory. +4) Calling [SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) with [KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) in order to create `AppData\Roaming` folder. +5) Calling [CoTaskMemFree](CoTaskMemFree) to release memory from `SHGETKnownFolderPath` call in step 4. +6) Calling +[SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) +with `KNOWNFOLDERID` +[FOLDERID_LocalAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) to set the location of `AppData\Local` to under the task directory. +7) Calling [SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) with [KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) in order to create `AppData\Local` folder. +8) Calling [CoTaskMemFree](CoTaskMemFree) to release memory from `SHGETKnownFolderPath` call in step 7. +9) Calling [UnloadUserProfile](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-unloaduserprofile) to release resources from `LoadUserProfileW` call in step 2. +10) Calling [CloseHandle](https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396) to release resources from `LogonUserW` call in step 1. + + +### Configuring auto-logon of task user + +After the task user has been created, the registry is updated so that after +rebooting, the task user will be automatically logged in. + + + + + + +### + + # Payload format diff --git a/lib/win32/win32_windows.go b/lib/win32/win32_windows.go index 83cab137..58191723 100644 --- a/lib/win32/win32_windows.go +++ b/lib/win32/win32_windows.go @@ -196,6 +196,7 @@ func IsWindows8OrGreater() bool { return r } +// https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-logonuserw func LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32) (token syscall.Token, err error) { r1, _, e1 := procLogonUserW.Call( uintptr(unsafe.Pointer(username)), From 216bd99fc890dbc403eea1c0b6e290232f5d0a2e Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Wed, 10 Oct 2018 13:52:51 +0200 Subject: [PATCH 50/58] wip --- README.md | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 47d98956..e80b4218 100644 --- a/README.md +++ b/README.md @@ -92,13 +92,29 @@ generic-worker is platform-dependent. ## Windows -On Windows, `generic-worker.exe` runs in a Windows Service under the +On Windows, `generic-worker.exe` runs in a [Windows +Service](https://docs.microsoft.com/en-us/windows/desktop/services/services) +under the [LocalSystem](https://docs.microsoft.com/en-us/windows/desktop/services/localsystem-account) -account. This service creates a fresh new (non-admin) OS user in preparation -for running a task. The worker configures the machine to automatically log in -as the newly created task user, and then triggers the machine to reboot. Once -the machine reboots, the worker running in the Windows Service waits until it -detects that the Operating System [winlogon +account. + +The worker creates a unique Operating System user to sandbox the activity of +the task. + +All task commands run as the task user. After the task has completed, the user +is deleted, together with its files. + +In order for tasks to have access to a graphical logon session, the host is +configured for an automatic graphical logon as the new task user, and the +machine is rebooted. + +By default the generated users are standard (non-admin) users. + +### Task user lifecycle +The worker configures the machine to automatically log +in as the newly created task user, and then triggers the machine to reboot. +Once the machine reboots, the worker running in the Windows Service waits until +it detects that the Operating System [winlogon module](https://docs.microsoft.com/en-us/windows/desktop/secauthn/winlogon-and-credential-providers) has completed the interactive logon of the task user. At this point it polls the taskcluster Queue to fetch a task to execute, and when it is given one, it @@ -111,12 +127,14 @@ directory (if different to the home directory of the task user) are erased, a new task user is created, the machine is rebooted, and the former task user is purged. -In many ways, this operation can be alikened to providing a guest account on -the machine. After the task has completed, all trace of the changes made by the -task user should be gone, and the machine's state should be reset to the state -it had before the task was run. If the host environment is sufficiently locked -down, the task user should not have been able to apply any state-change to the -host environment. Please note that the worker has limited control to affect +In the same way that a guest account allows an untrusted user to temporarily +use a machine without impacting the rest of the machine, the generic worker +allows tasks to run on the host without having permanent affect. After the +task has completed, all trace of the changes made by the task user should be +gone, and the machine's state should be reset to the state it had before the +task was run. If the host environment is sufficiently locked down, the task +user should not have been able to apply any state-change to the host +environment. Please note that the worker has limited control to affect system-wide policy, so for example if a host allows arbtirary users to write to a system folder location, the worker is not able to prevent a task doing so. Therefore it is up to the machine provider to ensure that the host is From 2a61fef02ac39505a3df12e755876240fa19b0fd Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Wed, 10 Oct 2018 15:37:57 +0200 Subject: [PATCH 51/58] wip --- README.md | 86 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index e80b4218..4a504382 100644 --- a/README.md +++ b/README.md @@ -77,10 +77,19 @@ go (golang). ## Imperative task payloads -Generic worker allows tasks to run arbitrary command payloads in a sandbox. If -you are looking for a worker to run only specific commands on a privileged -environment, see -[scriptworker](https://github.com/mozilla-releng/scriptworker). +Generic worker allows you to execute arbitrary commands in a task. + +If you wish to only run trusted code against +input parameters passed in task payloads, see: + +* [scriptworker](https://github.com/mozilla-releng/scriptworker) + +If you are looking to isolate your tasks inside docker containers, see: + +* [docker-worker](https://github.com/taskcluster/docker-worker) + +Please note docker support is coming to generic-worker in [PR +62](https://github.com/taskcluster/generic-worker/pull/62). # Sandboxing @@ -220,38 +229,50 @@ It does this as follows: 1) Calling [LogonUserW](https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-logonuserw) -to get a logon handle for the new user. -2) Calling +to get a logon handle for the new user. 2) Calling [LoadUserProfileW](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-loaduserprofilew) -to load the user profile. -3) Calling +to load the user profile. 3) Calling [SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) with `KNOWNFOLDERID` -[FOLDERID_RoamingAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) to set the location of `AppData\Roaming` to under the task directory. -4) Calling [SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) with [KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) in order to create `AppData\Roaming` folder. -5) Calling [CoTaskMemFree](CoTaskMemFree) to release memory from `SHGETKnownFolderPath` call in step 4. -6) Calling +[FOLDERID_RoamingAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) +to set the location of `AppData\Roaming` to under the task directory. 4) +Calling +[SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) +with +[KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) +in order to create `AppData\Roaming` folder. 5) Calling +[CoTaskMemFree](CoTaskMemFree) to release resources from `SHGETKnownFolderPath` +call in step 4. 6) Calling [SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) with `KNOWNFOLDERID` -[FOLDERID_LocalAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) to set the location of `AppData\Local` to under the task directory. -7) Calling [SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) with [KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) in order to create `AppData\Local` folder. -8) Calling [CoTaskMemFree](CoTaskMemFree) to release memory from `SHGETKnownFolderPath` call in step 7. -9) Calling [UnloadUserProfile](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-unloaduserprofile) to release resources from `LoadUserProfileW` call in step 2. -10) Calling [CloseHandle](https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396) to release resources from `LogonUserW` call in step 1. +[FOLDERID_LocalAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) +to set the location of `AppData\Local` to under the task directory. 7) Calling +[SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) +with +[KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) +in order to create `AppData\Local` folder. 8) Calling +[CoTaskMemFree](CoTaskMemFree) to release resources from `SHGETKnownFolderPath` +call in step 7. 9) Calling +[UnloadUserProfile](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-unloaduserprofile) +to release resources from `LoadUserProfileW` call in step 2. 10) Calling +[CloseHandle](https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396) +to release resources from `LogonUserW` call in step 1. ### Configuring auto-logon of task user -After the task user has been created, the registry is updated so that after -rebooting, the task user will be automatically logged in. - +After the task user has been created, the Windows registry is updated so that +after rebooting, the task user will be automatically logged in. +This is achieved by configuring the registry key values: +* `\HKEY_LOCAL_MACHINE\Software\Microsoft\Windows NT\CurrentVersion\Winlogon\AutoAdminLogon = 1` +* `\HKEY_LOCAL_MACHINE\Software\Microsoft\Windows NT\CurrentVersion\Winlogon\DefaultUserName = ` +* `\HKEY_LOCAL_MACHINE\Software\Microsoft\Windows NT\CurrentVersion\Winlogon\DefaultPassword = ` - - -### - +See [Automatic +Logon](https://docs.microsoft.com/en-us/windows/desktop/secauthn/msgina-dll-features) +for more detailed information about these settings. # Payload format @@ -267,22 +288,17 @@ also published to the [generic-worker page](https://docs.taskcluster.net/docs/reference/workers/generic-worker/docs/payload) of the taskcluster docs site. -# Execution policy +# Redeployability -## Windows -### System calls (KnownFolders, generating processes, ...) -## Mac -No isolation -## Linux -No isolation - working on docker +# Integrating with AWS / GCE -## Redeployability +# Config bootstrapping -## Integrating with AWS / GCE +# Bring your own worker -## Config bootstrapping +This section explains how to configure and run your own generic-worker workers +to talk to an existing taskcluster deployment. -# "Bring your own worker": Running your own generic-worker workers ## Windows ### Installing ## Mac From de1ce0812556ff2671820cd435cbde434ca8843f Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Wed, 10 Oct 2018 15:38:43 +0200 Subject: [PATCH 52/58] wip --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 4a504382..d9b2e5af 100644 --- a/README.md +++ b/README.md @@ -81,11 +81,9 @@ Generic worker allows you to execute arbitrary commands in a task. If you wish to only run trusted code against input parameters passed in task payloads, see: - * [scriptworker](https://github.com/mozilla-releng/scriptworker) If you are looking to isolate your tasks inside docker containers, see: - * [docker-worker](https://github.com/taskcluster/docker-worker) Please note docker support is coming to generic-worker in [PR From df958081746d63dc6f4385361d4db59e2f83917c Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Wed, 10 Oct 2018 15:57:38 +0200 Subject: [PATCH 53/58] wip --- README.md | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d9b2e5af..7b92149c 100644 --- a/README.md +++ b/README.md @@ -109,13 +109,13 @@ The worker creates a unique Operating System user to sandbox the activity of the task. All task commands run as the task user. After the task has completed, the user -is deleted, together with its files. +is deleted, together with any files it has created. In order for tasks to have access to a graphical logon session, the host is -configured for an automatic graphical logon as the new task user, and the +configured to logon on boot as the new task user, and the machine is rebooted. -By default the generated users are standard (non-admin) users. +By default the generated users are standard (non-admin) OS users. ### Task user lifecycle The worker configures the machine to automatically log @@ -272,6 +272,32 @@ See [Automatic Logon](https://docs.microsoft.com/en-us/windows/desktop/secauthn/msgina-dll-features) for more detailed information about these settings. +### Rebooting + +Rebooting is achieved by executing: + +``` +C:\Windows\System32\shutdown.exe /r /t 3 /c generic-worker requested reboot +``` + +Please note, automatic reboots can be disabled (see `generic-worker --help` for +more information). + +### Executing task commands + +The Windows Command Shell does not provide a feature to enable exit-on-fail +semantics. Execution of a batch script continues if a command fails, without a +global switch available to enable such behaviour. To implement exit-on-fail +semantics in a batch script requires explicitly checking the exit code of every +command that may have failed. + +Typically tasks need to run multiple commands, and exit if any one of them +fail. Therefore, rather than relying on the native command shell for specifying +multiple commands to run, generic-worker supports running multiple commands +natively, and will handle the scheduling of the commands and evaluating the +exit codes of each command, rather than relying on the shell to do this. + +.... # Payload format From 6f3cd7648a54b5703dd4398730477d3525cff159 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Wed, 10 Oct 2018 15:59:23 +0200 Subject: [PATCH 54/58] wip --- README.md | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7b92149c..91c953df 100644 --- a/README.md +++ b/README.md @@ -10,23 +10,26 @@ * [Introdution](#introdution) * [Imperative task payloads](#imperative-task-payloads) - * [Sandboxing](#sandboxing) - * [Windows](#windows) - * [Linux](#linux) - * [macOS](#macos) - * [Payload format](#payload-format) - * [Execution policy](#execution-policy) + * [Sandboxing](#sandboxing) + * [Windows](#windows) + * [Task user lifecycle](#task-user-lifecycle) + * [Linux](#linux) + * [macOS](#macos) + * [Operating System integration](#operating-system-integration) * [Windows](#windows-1) - * [System calls (KnownFolders, generating processes, ...)](#system-calls-knownfolders-generating-processes-) - * [Mac](#mac) - * [Linux](#linux-1) - * [Redeployability](#redeployability) - * [Integrating with AWS / GCE](#integrating-with-aws--gce) - * [Config bootstrapping](#config-bootstrapping) - * ["Bring your own worker": Running your own generic-worker workers](#bring-your-own-worker-running-your-own-generic-worker-workers) - * [Mac](#mac-1) - * [Installing](#installing) + * [Creating a task user](#creating-a-task-user) + * [Setting Known Folder locations](#setting-known-folder-locations) + * [Configuring auto-logon of task user](#configuring-auto-logon-of-task-user) + * [Rebooting](#rebooting) + * [Executing task commands](#executing-task-commands) + * [Payload format](#payload-format) + * [Redeployability](#redeployability) + * [Integrating with AWS / GCE](#integrating-with-aws--gce) + * [Config bootstrapping](#config-bootstrapping) + * [Bring your own worker](#bring-your-own-worker) * [Windows](#windows-2) + * [Installing](#installing) + * [Mac](#mac) * [Installing](#installing-1) * [Linux - Docker](#linux---docker) * [Linux - Native](#linux---native) @@ -63,7 +66,7 @@ * [Making a new generic worker release](#making-a-new-generic-worker-release) * [Creating and updating worker types](#creating-and-updating-worker-types) * [Release notes](#release-notes) - * [Further information](#further-information) + # Introdution From e9d8455144429b6c16f7fb4d5c04ccfd80186912 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Wed, 10 Oct 2018 16:43:21 +0200 Subject: [PATCH 55/58] wip --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 91c953df..634b3477 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ for more detailed information about these settings. Rebooting is achieved by executing: ``` -C:\Windows\System32\shutdown.exe /r /t 3 /c generic-worker requested reboot +C:\Windows\System32\shutdown.exe /r /t 3 /c "generic-worker requested reboot" ``` Please note, automatic reboots can be disabled (see `generic-worker --help` for From 7f8e8c3672a7e3f6d772768bac1b6e627133bdeb Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Thu, 11 Oct 2018 18:34:58 +0200 Subject: [PATCH 56/58] wip --- README.md | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 634b3477..02f5ff13 100644 --- a/README.md +++ b/README.md @@ -288,19 +288,28 @@ more information). ### Executing task commands -The Windows Command Shell does not provide a feature to enable exit-on-fail -semantics. Execution of a batch script continues if a command fails, without a -global switch available to enable such behaviour. To implement exit-on-fail -semantics in a batch script requires explicitly checking the exit code of every -command that may have failed. - -Typically tasks need to run multiple commands, and exit if any one of them -fail. Therefore, rather than relying on the native command shell for specifying -multiple commands to run, generic-worker supports running multiple commands -natively, and will handle the scheduling of the commands and evaluating the -exit codes of each command, rather than relying on the shell to do this. - -.... +The Windows Command Shell does not have a setting to enable exit-on-fail +semantics. Execution of a batch script continues if a command fails. To cause +a batch script to exit after a failed command, the exit code of every command +needs to be checked, or commands need to be chained together with `&&`. + +Since this is cumbersome or error-prone, generic-worker accepts task payloads +with multiple commands. It will execute them in sequence with exit-on-fail +semantics. Each command is implicitly executed with `cmd.exe`, which means that +commands may contain any valid [command shell syntax](https://ss64.com/nt/). + +Other workers (such as docker worker) accept only a single task command. If a +task wishes to execute multiple commands, it will usually specify a single +shell command to execute them. This approach works well when the shell supports +exit-on-fail semantics, but not so well when it doesn't, which is why a +different approach was chosen for generic-worker. + +Generic worker generates a wrapper batch script for each command it runs, which +initialises environment variables, sets the working directory, executes the +task command, and then if more commands are to follow, captures the working +directory and environment variables for the next command. + + # Payload format From dca51758fa57b3e6a609c6a066b9c80d6a484a52 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Thu, 11 Oct 2018 18:48:59 +0200 Subject: [PATCH 57/58] wip --- README.md | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 02f5ff13..9a7ba3d4 100644 --- a/README.md +++ b/README.md @@ -230,32 +230,47 @@ It does this as follows: 1) Calling [LogonUserW](https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-logonuserw) -to get a logon handle for the new user. 2) Calling +to get a logon handle for the new user. + +2) Calling [LoadUserProfileW](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-loaduserprofilew) -to load the user profile. 3) Calling +to load the user profile. + +3) Calling [SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) with `KNOWNFOLDERID` [FOLDERID_RoamingAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) -to set the location of `AppData\Roaming` to under the task directory. 4) -Calling +to set the location of `AppData\Roaming` to under the task directory. + +4) Calling [SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) with [KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) -in order to create `AppData\Roaming` folder. 5) Calling -[CoTaskMemFree](CoTaskMemFree) to release resources from `SHGETKnownFolderPath` -call in step 4. 6) Calling +in order to create `AppData\Roaming` folder. + +5) Calling [CoTaskMemFree](CoTaskMemFree) to release resources from +`SHGETKnownFolderPath` call in step 4. + +6) Calling [SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) with `KNOWNFOLDERID` [FOLDERID_LocalAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) -to set the location of `AppData\Local` to under the task directory. 7) Calling +to set the location of `AppData\Local` to under the task directory. + +7) Calling [SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) with [KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) -in order to create `AppData\Local` folder. 8) Calling -[CoTaskMemFree](CoTaskMemFree) to release resources from `SHGETKnownFolderPath` -call in step 7. 9) Calling +in order to create `AppData\Local` folder. + +8) Calling [CoTaskMemFree](CoTaskMemFree) to release resources from +`SHGETKnownFolderPath` call in step 7. + +9) Calling [UnloadUserProfile](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-unloaduserprofile) -to release resources from `LoadUserProfileW` call in step 2. 10) Calling +to release resources from `LoadUserProfileW` call in step 2. + +10) Calling [CloseHandle](https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396) to release resources from `LogonUserW` call in step 1. From 05b161adcb107e72e9ab920a7788469197583bb2 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Fri, 12 Apr 2019 13:09:53 +0200 Subject: [PATCH 58/58] Added download-secrets --- cmd/download-secrets/main.go | 144 +++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 cmd/download-secrets/main.go diff --git a/cmd/download-secrets/main.go b/cmd/download-secrets/main.go new file mode 100644 index 00000000..ac00cf10 --- /dev/null +++ b/cmd/download-secrets/main.go @@ -0,0 +1,144 @@ +package main + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "sync" + "time" + + "github.com/taskcluster/httpbackoff" + "github.com/taskcluster/taskcluster-client-go/tcsecrets" +) + +type SecretFetcher struct { + Secrets *tcsecrets.Secrets + RequestChannel <-chan string + DownloadDirectory string + ProcessedChannel chan<- string +} + +type WorkerPool struct { + Workers []*SecretFetcher + WaitGroup sync.WaitGroup +} + +// This program is a simple command line utility. When you run it, it will +// spawn 20 go routines to download taskcluster secrets in parallel. It will +// put them in a sub-directory called secrets, creating it if necessary, with +// each file named as the secret. Path separators in the secret name will +// result in created subdirectories. I use this in combination with a cron job +// that runs +// https://github.com/petemoore/myscrapbook/blob/master/sync-secrets.sh every 5 +// mins, in order to maintain a git history of secrets locally, in case +// something goes horribly wrong and we need to restore them. I don't publish +// the git repository anywhere, obviously! +func main() { + ss := tcsecrets.NewFromEnv() + downloadDirectory := "secrets" + requestChannel := make(chan string) + processedChannel := make(chan string) + _ = NewWorkerPool(20, requestChannel, processedChannel, ss, downloadDirectory) + contToken := "" + allSecrets := []string{} + for { + moreSecrets, err := ss.List(contToken, "") + if err != nil { + panic(err) + } + allSecrets = append(allSecrets, moreSecrets.Secrets...) + contToken = moreSecrets.ContinuationToken + if contToken == "" { + break + } + } + err := os.MkdirAll(downloadDirectory, 0755) + if err != nil { + panic(err) + } + go func() { + defer close(requestChannel) + for _, secret := range allSecrets { + requestChannel <- secret + } + }() + for completedSecret := range processedChannel { + fmt.Println(completedSecret) + } +} + +func NewWorkerPool(capacity int, requestChannel <-chan string, processedChannel chan<- string, ss *tcsecrets.Secrets, downloadDirectory string) *WorkerPool { + wp := &WorkerPool{} + wp.WaitGroup.Add(capacity) + wp.Workers = make([]*SecretFetcher, capacity, capacity) + for i := 0; i < capacity; i++ { + wp.Workers[i] = &SecretFetcher{ + Secrets: ss, + RequestChannel: requestChannel, + DownloadDirectory: downloadDirectory, + ProcessedChannel: processedChannel, + } + go func(i int) { + wp.Workers[i].FetchUntilDone(&wp.WaitGroup) + }(i) + } + go func() { + wp.WaitGroup.Wait() + close(processedChannel) + }() + return wp +} + +func (secretFetcher *SecretFetcher) FetchUntilDone(wg *sync.WaitGroup) { + for secret := range secretFetcher.RequestChannel { + err := secretFetcher.fetch(secret) + if err != nil { + panic(err) + } + } + wg.Done() +} + +func (secretFetcher *SecretFetcher) fetch(secret string) (err error) { + var u *url.URL + u, err = secretFetcher.Secrets.Get_SignedURL(secret, time.Minute) + if err != nil { + return + } + var resp *http.Response + resp, _, err = httpbackoff.Get(u.String()) + if err != nil { + return + } + defer func() { + if err == nil { + err = resp.Body.Close() + } else { + resp.Body.Close() + } + }() + var file *os.File + path := filepath.Join(secretFetcher.DownloadDirectory, secret) + dir := filepath.Dir(path) + err = os.MkdirAll(dir, 0755) + if err != nil { + panic(err) + } + file, err = os.Create(path) + if err != nil { + return + } + defer func() { + if err == nil { + err = file.Close() + } else { + file.Close() + } + }() + io.Copy(file, resp.Body) + secretFetcher.ProcessedChannel <- secret + return +}