diff --git a/.gitignore b/.gitignore index 6ce29864..d726e949 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ _testmain.go # For any releases made locally /target/ -worker_type_definitions +aws-provisioner-v1-worker-type-definitions # Files created by generic-worker generic-worker.config diff --git a/.taskcluster.yml b/.taskcluster.yml index 1ef60ad2..1caa0338 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -86,10 +86,10 @@ tasks: - 'git clean -fdx' - 'git checkout -B tmp -t "X%TASK_ID%"' - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen + - cd cmd\gw-codegen - go get -v -u - - cd .. - - go generate + - cd ..\.. + - go generate ./... - | :: this counts the number of lines returned by git status :: dump temp file a directory higher, otherwise git status reports the tmp1.txt file! @@ -105,9 +105,9 @@ tasks: - set CGO_ENABLED=1 - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race github.com/taskcluster/generic-worker/cmd/generic-worker' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe @@ -168,17 +168,17 @@ tasks: - 'git clean -fdx' - 'git checkout -B tmp -t "X%TASK_ID%"' - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen + - cd cmd\gw-codegen - go get -v -u - - cd .. - - go generate + - cd ..\.. + - go generate ./... - set revision={{ event.head.sha }} - go get -ldflags "-X main.revision=%revision%" -v -u -t ./... - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v github.com/taskcluster/generic-worker/cmd/generic-worker' - ineffassign . artifacts: - name: public/build/generic-worker-windows-386.exe @@ -239,10 +239,10 @@ tasks: - 'git clean -fdx' - 'git checkout -B tmp -t "X%TASK_ID%"' - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen + - cd cmd\gw-codegen - go get -v -u - - cd .. - - go generate + - cd ..\.. + - go generate ./... - | :: this counts the number of lines returned by git status :: dump temp file a directory higher, otherwise git status reports the tmp1.txt file! @@ -258,9 +258,9 @@ tasks: - set CGO_ENABLED=1 - set GORACE=history_size=7 - C:\generic-worker\generic-worker-test-creds.cmd - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race ./...' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race ./...' - set GW_TESTS_RUN_AS_TASK_USER=true - - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race' + - 'go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%revision%" -v -race github.com/taskcluster/generic-worker/cmd/generic-worker' - ineffassign . artifacts: - name: public/build/generic-worker-windows-amd64.exe @@ -310,12 +310,12 @@ tasks: export CGO_ENABLED=0 export GOROOT="$(pwd)/go1.10.3/go" export GOPATH="$(pwd)/gopath1.10.3" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + export PATH="$(go env GOPATH)/bin:${GOROOT}/bin:${PATH}" go version go env source "${GW_CREDS_BOOTSTRAP}" - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" + mkdir -p "$(go env GOPATH)/src/github.com/taskcluster" + cd "$(go env GOPATH)/src/github.com/taskcluster" if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi cd 'generic-worker' git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" @@ -324,14 +324,14 @@ tasks: git clean -fdx git checkout -B tmp -t "X${TASK_ID}" go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen + cd cmd/gw-codegen go get -v -u - cd .. - go generate + cd ../.. + go generate ./... go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # output of wc command can contain spaces on darwin, so no quotes around expression test $(git status --porcelain | wc -l) == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -race -v ./... + GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -v ./... ineffassign . artifacts: - name: public/build/generic-worker-darwin-amd64 @@ -382,13 +382,13 @@ tasks: # export CGO_ENABLED=0 # export GOROOT="$(pwd)/go1.10.3/go" # export GOPATH="$(pwd)/gopath1.10.3" - # export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + # export PATH="$(go env GOPATH)/bin:${GOROOT}/bin:${PATH}" # export CGO_ENABLED=0 # go version # go env # source "${GW_CREDS_BOOTSTRAP}" - # mkdir -p "${GOPATH}/src/github.com/taskcluster" - # cd "${GOPATH}/src/github.com/taskcluster" + # mkdir -p "$(go env GOPATH)/src/github.com/taskcluster" + # cd "$(go env GOPATH)/src/github.com/taskcluster" # if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi # cd 'generic-worker' # git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" @@ -397,13 +397,13 @@ tasks: # git clean -fdx # git checkout -B tmp -t "X${TASK_ID}" # go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - # cd gw-codegen + # cd cmd/gw-codegen # go get -v -u - # cd .. - # go generate + # cd ../.. + # go generate ./... # go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... # test "$(git status --porcelain | wc -l)" == 0 - # GORACE=history_size=7 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v ./... + # GORACE=history_size=7 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v ./... # ineffassign . # artifacts: # - name: public/build/generic-worker-linux-armv6l @@ -454,8 +454,8 @@ tasks: go env curl -s http://taskcluster/secrets/v1/secret/repo:github.com/taskcluster/generic-worker | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -d > ~/env_private.sh source ~/env_private.sh - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" + mkdir -p "$(go env GOPATH)/src/github.com/taskcluster" + cd "$(go env GOPATH)/src/github.com/taskcluster" if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi cd 'generic-worker' git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" @@ -464,14 +464,14 @@ tasks: git clean -fdx git checkout -B tmp -t "X${TASK_ID}" go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen + cd cmd/gw-codegen go get -v -u - cd .. - go generate + cd ../.. + go generate ./... go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v -u -t ./... test "$(git status --porcelain | wc -l)" == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... - "${GOPATH}/bin/ineffassign" . + GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 30m -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... + "$(go env GOPATH)/bin/ineffassign" . artifacts: public/build/generic-worker-linux-amd64: path: "/go/bin/generic-worker" diff --git a/.travis.yml b/.travis.yml index 656cca7c..1307a78a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ env: - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=386" - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=amd64" - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=arm" - # currently broken + # currently broken due to upstream issue # - "CGO_ENABLED=0 GIMME_OS=linux GIMME_ARCH=arm64" - "CGO_ENABLED=0 GIMME_OS=darwin GIMME_ARCH=amd64" - "CGO_ENABLED=0 GIMME_OS=windows GIMME_ARCH=amd64" @@ -16,16 +16,16 @@ install: - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go get github.com/mattn/goveralls github.com/taskcluster/taskcluster-proxy github.com/taskcluster/livelog github.com/gordonklaus/ineffassign" script: - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go get ./gw-codegen" - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go generate" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go get ./cmd/gw-codegen" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || go generate ./..." - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || git status" - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || test $(git status --porcelain | wc -l) == 0" - "go get -ldflags \"-X main.revision=$(git rev-parse HEAD)\" -v -t ./..." - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || GORACE=history_size=7 travis_wait 30 ./gotestcover.sh coverage.report" - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || ${GOPATH}/bin/ineffassign ." + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || GORACE=history_size=7 travis_wait 30 scripts/gotestcover.sh coverage.report" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || $(go env GOPATH)/bin/ineffassign ." after_script: - - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || ${GOPATH}/bin/goveralls -coverprofile=coverage.report -service=travis-ci" + - "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || $(go env GOPATH)/bin/goveralls -coverprofile=coverage.report -service=travis-ci" before_deploy: - "source .travis_rename_releases.sh" diff --git a/README.md b/README.md index c6b36ef0..9a7ba3d4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -# generic-worker - A generic worker for [taskcluster](https://tools.taskcluster.net/), written in go. [![Taskcluster CI Status](https://github.taskcluster.net/v1/repository/taskcluster/generic-worker/master/badge.svg)](https://github.taskcluster.net/v1/repository/taskcluster/generic-worker/master/latest) @@ -8,25 +6,503 @@ [![Coverage Status](https://coveralls.io/repos/taskcluster/generic-worker/badge.svg?branch=master&service=github)](https://coveralls.io/github/taskcluster/generic-worker?branch=master) [![License](https://img.shields.io/badge/license-MPL%202.0-orange.svg)](http://mozilla.org/MPL/2.0) +# Table of Contents + + * [Introdution](#introdution) + * [Imperative task payloads](#imperative-task-payloads) + * [Sandboxing](#sandboxing) + * [Windows](#windows) + * [Task user lifecycle](#task-user-lifecycle) + * [Linux](#linux) + * [macOS](#macos) + * [Operating System integration](#operating-system-integration) + * [Windows](#windows-1) + * [Creating a task user](#creating-a-task-user) + * [Setting Known Folder locations](#setting-known-folder-locations) + * [Configuring auto-logon of task user](#configuring-auto-logon-of-task-user) + * [Rebooting](#rebooting) + * [Executing task commands](#executing-task-commands) + * [Payload format](#payload-format) + * [Redeployability](#redeployability) + * [Integrating with AWS / GCE](#integrating-with-aws--gce) + * [Config bootstrapping](#config-bootstrapping) + * [Bring your own worker](#bring-your-own-worker) + * [Windows](#windows-2) + * [Installing](#installing) + * [Mac](#mac) + * [Installing](#installing-1) + * [Linux - Docker](#linux---docker) + * [Linux - Native](#linux---native) + * [Installing](#installing-2) + * [Administrative tools](#administrative-tools) + * [Displaying workers](#displaying-workers) + * [Worker Type Host Definitions](#worker-type-host-definitions) + * [Updating existing definitions](#updating-existing-definitions) + * [Modifying definitions](#modifying-definitions) + * [Creating your own AWS workers outside of this repo](#creating-your-own-aws-workers-outside-of-this-repo) + * [Puppet](#puppet) + * [Developing Generic Worker](#developing-generic-worker) + * [Fetching source](#fetching-source) + * [Credentials](#credentials) + * [Running unit tests](#running-unit-tests) + * [Writing unit tests](#writing-unit-tests) + * [Including bug numbers in comments](#including-bug-numbers-in-comments) + * [Releasing Generic Worker](#releasing-generic-worker) + * [Release script](#release-script) + * [Publishing schemas](#publishing-schemas) + * [Testing in Staging](#testing-in-staging) + * [Rolling out to Production](#rolling-out-to-production) + * [Writing release notes (README.md, release page, ...)](#writing-release-notes-readmemd-release-page-) + * [Repository layout](#repository-layout) + * [Downloading generic-worker binary release](#downloading-generic-worker-binary-release) + * [Building generic-worker from source](#building-generic-worker-from-source) + * [Acquire taskcluster credentials for running code tests](#acquire-taskcluster-credentials-for-running-code-tests) + * [Option 1](#option-1) + * [Option 2](#option-2) + * [Set up your env](#set-up-your-env) + * [Start the generic worker](#start-the-generic-worker) + * [Create a test job](#create-a-test-job) + * [Run the generic worker test suite](#run-the-generic-worker-test-suite) + * [Making a new generic worker release](#making-a-new-generic-worker-release) + * [Creating and updating worker types](#creating-and-updating-worker-types) + * [Release notes](#release-notes) + + + +# Introdution + +Generic worker is a native Windows/Linux/macOS program for executing +taskcluster tasks. It communicates with the taskcluster Queue as per the +[Queue-Worker Interaction +specification](https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/docs/worker-interaction). +It is shipped as a statically linked system-native executable. It is written in +go (golang). + +## Imperative task payloads + +Generic worker allows you to execute arbitrary commands in a task. + +If you wish to only run trusted code against +input parameters passed in task payloads, see: +* [scriptworker](https://github.com/mozilla-releng/scriptworker) + +If you are looking to isolate your tasks inside docker containers, see: +* [docker-worker](https://github.com/taskcluster/docker-worker) + +Please note docker support is coming to generic-worker in [PR +62](https://github.com/taskcluster/generic-worker/pull/62). + +# Sandboxing + +It is important that tasks run in a sandbox in order to that they are as +reproducible as possible, and are not inadvertently affected by previous tasks +that may have run on the same environment. Different operating systems provide +different sandboxing mechanisms, and therefore the approach used by +generic-worker is platform-dependent. + +## Windows + +On Windows, `generic-worker.exe` runs in a [Windows +Service](https://docs.microsoft.com/en-us/windows/desktop/services/services) +under the +[LocalSystem](https://docs.microsoft.com/en-us/windows/desktop/services/localsystem-account) +account. + +The worker creates a unique Operating System user to sandbox the activity of +the task. + +All task commands run as the task user. After the task has completed, the user +is deleted, together with any files it has created. + +In order for tasks to have access to a graphical logon session, the host is +configured to logon on boot as the new task user, and the +machine is rebooted. + +By default the generated users are standard (non-admin) OS users. + +### Task user lifecycle +The worker configures the machine to automatically log +in as the newly created task user, and then triggers the machine to reboot. +Once the machine reboots, the worker running in the Windows Service waits until +it detects that the Operating System [winlogon +module](https://docs.microsoft.com/en-us/windows/desktop/secauthn/winlogon-and-credential-providers) +has completed the interactive logon of the task user. At this point it polls +the taskcluster Queue to fetch a task to execute, and when it is given one, it +executes this task in the interactive logon session of the logged-in user, +running processes using the auth token obtained from the interactive desktop +session. + +After the task completes, the home directory of the task user, and the task +directory (if different to the home directory of the task user) are erased, a +new task user is created, the machine is rebooted, and the former task user is +purged. + +In the same way that a guest account allows an untrusted user to temporarily +use a machine without impacting the rest of the machine, the generic worker +allows tasks to run on the host without having permanent affect. After the +task has completed, all trace of the changes made by the task user should be +gone, and the machine's state should be reset to the state it had before the +task was run. If the host environment is sufficiently locked down, the task +user should not have been able to apply any state-change to the host +environment. Please note that the worker has limited control to affect +system-wide policy, so for example if a host allows arbtirary users to write to +a system folder location, the worker is not able to prevent a task doing so. +Therefore it is up to the machine provider to ensure that the host is +sufficiently locked-down. Host environments for long-lived workers that are to +run untrusted tasks should be secured carefully, to prevent that tasks may +interfere with system state or persist changes across task runs that may affect +the reproducibility of a task, or worse, introduce a security vulnerability. + + + + + + +## Linux + +There is no native sandbox support currently on Linux. Currently the worker +will execute tasks as the same user that the worker runs as. Use at your own +risk! + +Work is [underway](https://github.com/taskcluster/generic-worker/pull/62) to +provide support for running generic-worker tasks inside a docker container +isolated from the host environment. However until this work is complete, please +see [docker-worker](https://github.com/taskcluster/docker-worker) for achieving +this. + +We may, at some point, provide OS-user sandboxing, akin to the Windows +implementation. + + +## macOS + +There is no native sandbox support currently on macOS. Currently the worker +will execute tasks as the same user that the worker runs as. Use at your own +risk! + +We intend to provide OS-user sandboxing, akin to the Windows implementation, at +some point in the future. + +# Operating System integration + +## Windows + +### Creating a task user + +The generic-worker creates non-privileged task users, with username +`task_` and a random password. Task users are created +with the command: + +``` +net user "" "" /add /expires:never /passwordchg:no /y +``` + +See [net +user](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490718%28v%253dtechnet.10%29) +for more information. + +### Setting Known Folder locations +It may be desirable for the task directory to be in a different location to the +home directory of the generated user. The location of the home directory is +determined based on system settings defined at installation time of the +operating system, and therefore may not be ideal, especially if the host image +is provided by an external party. -# Install binary +For example, perhaps the user home directory is `C:\Users\task_` but +for performance reasons, we wish the task directory to be located on a +different physical drive at `Z:\task_`. + +It is possible to configure the location for the task directories (in the above +case, `Z:\`) via the `tasksDir` property of the generic-worker configuration +file. However, this would not affect the location of the [AppData +folder](https://www.howtogeek.com/318177/what-is-the-appdata-folder-in-windows/) +used by Windows applications, which would still be located under the user's +home directory. + +Since it is usually preferable for all user data to be written +to the the task directory, and it isn't trivial to move the user home directory +to an alternative location after the operating system has already been +installed, the worker configures the user account to store the AppData folder +under the task directory. + +It does this as follows: + +1) Calling +[LogonUserW](https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-logonuserw) +to get a logon handle for the new user. + +2) Calling +[LoadUserProfileW](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-loaduserprofilew) +to load the user profile. + +3) Calling +[SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) +with `KNOWNFOLDERID` +[FOLDERID_RoamingAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) +to set the location of `AppData\Roaming` to under the task directory. + +4) Calling +[SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) +with +[KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) +in order to create `AppData\Roaming` folder. + +5) Calling [CoTaskMemFree](CoTaskMemFree) to release resources from +`SHGETKnownFolderPath` call in step 4. + +6) Calling +[SHSetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shsetknownfolderpath) +with `KNOWNFOLDERID` +[FOLDERID_LocalAppData](https://docs.microsoft.com/en-us/windows/desktop/shell/knownfolderid) +to set the location of `AppData\Local` to under the task directory. + +7) Calling +[SHGetKnownFolderPath](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetknownfolderpath) +with +[KF_FLAG_CREATE](https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/ne-shlobj_core-known_folder_flag) +in order to create `AppData\Local` folder. + +8) Calling [CoTaskMemFree](CoTaskMemFree) to release resources from +`SHGETKnownFolderPath` call in step 7. + +9) Calling +[UnloadUserProfile](https://docs.microsoft.com/en-us/windows/desktop/api/userenv/nf-userenv-unloaduserprofile) +to release resources from `LoadUserProfileW` call in step 2. + +10) Calling +[CloseHandle](https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396) +to release resources from `LogonUserW` call in step 1. + + +### Configuring auto-logon of task user + +After the task user has been created, the Windows registry is updated so that +after rebooting, the task user will be automatically logged in. + +This is achieved by configuring the registry key values: + +* `\HKEY_LOCAL_MACHINE\Software\Microsoft\Windows NT\CurrentVersion\Winlogon\AutoAdminLogon = 1` +* `\HKEY_LOCAL_MACHINE\Software\Microsoft\Windows NT\CurrentVersion\Winlogon\DefaultUserName = ` +* `\HKEY_LOCAL_MACHINE\Software\Microsoft\Windows NT\CurrentVersion\Winlogon\DefaultPassword = ` + +See [Automatic +Logon](https://docs.microsoft.com/en-us/windows/desktop/secauthn/msgina-dll-features) +for more detailed information about these settings. + +### Rebooting + +Rebooting is achieved by executing: + +``` +C:\Windows\System32\shutdown.exe /r /t 3 /c "generic-worker requested reboot" +``` + +Please note, automatic reboots can be disabled (see `generic-worker --help` for +more information). + +### Executing task commands + +The Windows Command Shell does not have a setting to enable exit-on-fail +semantics. Execution of a batch script continues if a command fails. To cause +a batch script to exit after a failed command, the exit code of every command +needs to be checked, or commands need to be chained together with `&&`. + +Since this is cumbersome or error-prone, generic-worker accepts task payloads +with multiple commands. It will execute them in sequence with exit-on-fail +semantics. Each command is implicitly executed with `cmd.exe`, which means that +commands may contain any valid [command shell syntax](https://ss64.com/nt/). + +Other workers (such as docker worker) accept only a single task command. If a +task wishes to execute multiple commands, it will usually specify a single +shell command to execute them. This approach works well when the shell supports +exit-on-fail semantics, but not so well when it doesn't, which is why a +different approach was chosen for generic-worker. + +Generic worker generates a wrapper batch script for each command it runs, which +initialises environment variables, sets the working directory, executes the +task command, and then if more commands are to follow, captures the working +directory and environment variables for the next command. + + + +# Payload format + +Each taskcluster task definition contains a top level property `payload` which +is a json object. The format of this object is specific to the worker +implementation. For generic-worker, this is then also further specific to the +platform (Linux/Windows/macOS). + +The per-platform payload formats are described in json schema, and can be found +in the top level `schemas` subdirectory of this repository. These schemas are +also published to the [generic-worker +page](https://docs.taskcluster.net/docs/reference/workers/generic-worker/docs/payload) +of the taskcluster docs site. + +# Redeployability + +# Integrating with AWS / GCE + +# Config bootstrapping + +# Bring your own worker + +This section explains how to configure and run your own generic-worker workers +to talk to an existing taskcluster deployment. + +## Windows +### Installing +## Mac +### Installing + +There currently is no `install` target for macOS, like there is for Windows. + +For our own dedicated macOS workers, we install generic-worker using [this +puppet +module](https://wiki.mozilla.org/ReleaseEngineering/PuppetAgain/Modules/generic_worker). + +You can install generic-worker as a Launch Agent as follows: + +1) Create a regular unprivileged user account on your Mac to run the worker +(e.g. with name `genericworker`) and log into that user account. + +2) Download or build generic-worker, so that you have a native darwin binary, +move it to `/usr/local/bin/generic-worker`, and make sure it is executable for +your new user (`chmod u+x /usr/local/bin/generic-worker`). + +3) Create a signing key in the user home directory by running: + +``` +/usr/local/bin/generic-worker new-openpgp-keypair --file .signingkey +``` + +3) Create `/Library/LaunchAgents/net.generic.worker.plist` with content: + +``` +<%# This Source Code Form is subject to the terms of the Mozilla Public +<%# License, v. 2.0. If a copy of the MPL was not distributed with this +<%# file, You can obtain one at http://mozilla.org/MPL/2.0/. -%> + + + + + Label + net.generic.worker + ProgramArguments + + /usr/local/bin/generic-worker + run + --config + /etc/generic-worker.config + + WorkingDirectory + <-YOUR-NEW-USER-HOME-DIRECTORY-> + RunAtLoad + + + +``` + +4) Create `/etc/generic-worker.config` with content: + +``` +{ + "accessToken": "<-YOUR-TASKCLUSTER-ACCESS-TOKEN->", + "clientId": "<-YOUR-TASKCLUSTER-CLIENT-ID->", + "idleTimeoutSecs": 0, + "livelogSecret": "<-RANDOM-SHORT-STRING-HERE->", + "provisionerId": "<-YOUR-PROVISIONER-ID->", + "publicIP": "<-MAKE-UP-AN-IPv4-ADDRESS-IF-YOU-DON'T-HAVE-ONE->", + "signingKeyLocation": ".signingkey", + "tasksDir": "tasks", + "workerGroup": "<-CHOOSE-A-WORKER-GROUP->", + "workerId": "<-CHOOSE-A-WORKER-ID->", + "workerType": "<-YOUR-WORKER-TYPE->", + "workerTypeMetadata": { + <--- add a json blob here with information about you, how you set up the + worker type, etc, so people know how it is configured and maintained, + and who to go to in case of problems ---> + } +} +``` + +## Linux - Docker +## Linux - Native +### Installing +# Administrative tools +## Displaying workers +# Worker Type Host Definitions +## Updating existing definitions +## Modifying definitions +## Creating your own AWS workers outside of this repo +## Puppet +# Developing Generic Worker +## Fetching source +## Credentials +## Running unit tests +## Writing unit tests +## Including bug numbers in comments + +# Releasing Generic Worker +## Release script +## Publishing schemas +## Testing in Staging +## Rolling out to Production +## Writing release notes (README.md, release page, ...) +# Repository layout + +``` +├── aws +│   ├── cmd +│   │   ├── download-aws-worker-type-definitions +│   │   ├── gw-workers +│   │   ├── update-ssl-creds +│   │   └── update-worker-type +│   ├── scripts +│   └── update-worker-types +├── cmd +│   ├── generic-worker +│   ├── gw-codegen +│   ├── inspect-worker-types +│   ├── list-worker-types +│   └── yamltojson +├── docs +├── lib +├── mozilla +│   ├── OpenCloudConfig +│   │   ├── occ-workers +│   │   ├── refresh-gw-configs +│   │   └── transform-occ +│   ├── gecko +│   ├── nss +│   └── worker-type-host-definitions +│   └── aws-provisioner-v1 +│   ├── +│   ├── +│   └── ... +├── schemas +└── scripts +``` + + +# Downloading generic-worker binary release * Download the latest release for your platform from https://github.com/taskcluster/generic-worker/releases * Download the latest release of livelog for your platform from https://github.com/taskcluster/livelog/releases * For darwin/linux, make the binaries executable: `chmod a+x {generic-worker,livelog}*` -# Build from source +# Building generic-worker from source If you prefer not to use a prepackaged binary, or want to have the latest unreleased version from the development head: -* Head over to https://golang.org/dl/ and follow the instructions for your platform. __Note, go 1.8 or higher is required__. Be sure to set your GOPATH to something appropriate. -* Run `go get github.com/taskcluster/generic-worker` -* Run `go get github.com/taskcluster/livelog` +* Head over to https://golang.org/dl/ and follow the instructions for your platform. __Note, go 1.8 or higher is required__. +* Run `./build.sh` (macOS/Linux) or `.\build.cmd` (Windows) + +All being well, the binaries will be built under `$(go env GOPATH)/bin`. + -All being well, the binaries will be built under `${GOPATH}/bin`. -# Acquire taskcluster credentials for running tests +# Acquire taskcluster credentials for running code tests There are two alternative mechanisms to acquire the scopes you need. diff --git a/worker_types/all-worker-types/main.go b/aws/cmd/download-aws-worker-type-definitions/main.go similarity index 92% rename from worker_types/all-worker-types/main.go rename to aws/cmd/download-aws-worker-type-definitions/main.go index 24d462a2..d1dbb156 100644 --- a/worker_types/all-worker-types/main.go +++ b/aws/cmd/download-aws-worker-type-definitions/main.go @@ -29,8 +29,9 @@ type WorkerPool struct { // This program is a simple command line utility. When you run it, it will // spawn 20 go routines to download worker type defintions for // aws-provisioner-v1 in parallel. It will put them in a sub-directory called -// worker_type_definitions, creating it if necessary, with each file named as -// the workerType. I use this in combination with a cron job that runs +// aws-provisioner-v1-worker-type-definitions, creating it if necessary, with +// each file named as the workerType. I use this in combination with a cron job +// that runs // https://github.com/petemoore/myscrapbook/blob/master/sync-worker-type-definitions.sh // every 5 mins, in order to maintain a git history of worker type definitions // locally. I don't publish the git repository anywhere, since the worker type @@ -38,7 +39,7 @@ type WorkerPool struct { // secrets at some point). func main() { prov := tcawsprovisioner.NewFromEnv() - downloadDirectory := "worker_type_definitions" + downloadDirectory := "aws-provisioner-v1-worker-type-definitions" requestChannel := make(chan string) processedChannel := make(chan string) _ = NewWorkerPool(20, requestChannel, processedChannel, prov, downloadDirectory) diff --git a/gw-workers/main.go b/aws/cmd/gw-workers/main.go similarity index 100% rename from gw-workers/main.go rename to aws/cmd/gw-workers/main.go diff --git a/update-ssl-creds/main.go b/aws/cmd/update-ssl-creds/main.go similarity index 100% rename from update-ssl-creds/main.go rename to aws/cmd/update-ssl-creds/main.go diff --git a/update-worker-type/update-worker-type.go b/aws/cmd/update-worker-type/main.go similarity index 97% rename from update-worker-type/update-worker-type.go rename to aws/cmd/update-worker-type/main.go index 207ea653..2783fc0a 100644 --- a/update-worker-type/update-worker-type.go +++ b/aws/cmd/update-worker-type/main.go @@ -112,7 +112,7 @@ func main() { oldMetadata := config["workerTypeMetadata"].(map[string]interface{}) oldMachineSetup := oldMetadata["machine-setup"].(map[string]interface{}) oldScript := oldMachineSetup["script"].(string) - newScript := "https://raw.githubusercontent.com/taskcluster/generic-worker/" + gitRevision(workerTypeDir) + "/worker_types/" + workerType + "/userdata" + newScript := "https://raw.githubusercontent.com/taskcluster/generic-worker/" + gitRevision(workerTypeDir) + "/mozilla/worker-type-host-definitions/aws-provisioner-v1/" + workerType + "/userdata" oldMachineSetup["script"] = newScript log.Print("Old machine setup script: " + oldScript) diff --git a/aws/scripts/cleanup_ec2.sh b/aws/scripts/cleanup_ec2.sh new file mode 100755 index 00000000..b4d40be3 --- /dev/null +++ b/aws/scripts/cleanup_ec2.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +cd "$(dirname "${0}")" + +go get github.com/taskcluster/generic-worker/aws/cmd/download-aws-worker-type-definitions +"$(go env GOPATH)/bin/download-aws-worker-type-definitions" > /dev/null +cat aws-provisioner-v1-worker-type-definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u +rm -r aws-provisioner-v1-worker-type-definitions diff --git a/worker_types/README.md b/aws/update-worker-types/README.md similarity index 100% rename from worker_types/README.md rename to aws/update-worker-types/README.md diff --git a/worker_types/delete.sh b/aws/update-worker-types/delete.sh similarity index 100% rename from worker_types/delete.sh rename to aws/update-worker-types/delete.sh diff --git a/worker_types/find_old_aws_objects.sh b/aws/update-worker-types/find_old_aws_objects.sh similarity index 64% rename from worker_types/find_old_aws_objects.sh rename to aws/update-worker-types/find_old_aws_objects.sh index 339a4106..3e18f345 100755 --- a/worker_types/find_old_aws_objects.sh +++ b/aws/update-worker-types/find_old_aws_objects.sh @@ -4,8 +4,8 @@ OLD_INSTANCES="$(aws --region "${REGION}" ec2 describe-instances --filters "Name # find old amis log "Querying previous AMI..." -OLD_SNAPSHOTS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} mozillabuild version*" --query 'Images[*].BlockDeviceMappings[*].Ebs.SnapshotId' --output text)" +OLD_SNAPSHOTS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} version *" --query 'Images[*].BlockDeviceMappings[*].Ebs.SnapshotId' --output text)" # find old snapshots log "Querying snapshot used in this previous AMI..." -OLD_AMIS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} mozillabuild version*" --query 'Images[*].ImageId' --output text)" +OLD_AMIS="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=name,Values=${WORKER_TYPE} version *" --query 'Images[*].ImageId' --output text)" diff --git a/worker_types/process_region.sh b/aws/update-worker-types/process_region.sh similarity index 93% rename from worker_types/process_region.sh rename to aws/update-worker-types/process_region.sh index 24869900..40657002 100755 --- a/worker_types/process_region.sh +++ b/aws/update-worker-types/process_region.sh @@ -29,12 +29,10 @@ if [ -z "${COLOUR}" ]; then exit 66 fi -cd "$(dirname "${0}")/${WORKER_TYPE}" - if [ "${ACTION}" == "update" ]; then - . ../update.sh + . "$(dirname "${0}")/update.sh" elif [ "${ACTION}" == "delete" ]; then - . ../delete.sh + . "$(dirname "${0}")/delete.sh" else log "$(basename "${0}"): ERROR: Unknown action '${ACTION}' ... exiting" >&2 exit 86 diff --git a/worker_types/update.sh b/aws/update-worker-types/update.sh similarity index 77% rename from worker_types/update.sh rename to aws/update-worker-types/update.sh index 5e4a5afa..164408ae 100755 --- a/worker_types/update.sh +++ b/aws/update-worker-types/update.sh @@ -1,14 +1,12 @@ -cd "$(dirname "${0}")/${WORKER_TYPE}" - log "Generating new ssh key..." -rm -rf "${REGION}.id_rsa" +rm -rf "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" aws --region "${REGION}" ec2 delete-key-pair --key-name "${WORKER_TYPE}_${REGION}" || true -aws --region "${REGION}" ec2 create-key-pair --key-name "${WORKER_TYPE}_${REGION}" --query 'KeyMaterial' --output text > "${REGION}.id_rsa" -chmod 400 "${REGION}.id_rsa" +aws --region "${REGION}" ec2 create-key-pair --key-name "${WORKER_TYPE}_${REGION}" --query 'KeyMaterial' --output text > "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" +chmod 400 "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" # aws cli docs lie, they say userdata must be base64 encoded, but cli encodes for you, so just cat it... -USER_DATA="$(cat userdata)" -AMI_BASE_NAME="$(cat ami-base-name)" +USER_DATA="$(cat "${WORKER_TYPE_DEFINITION_DIR}/userdata")" +AMI_BASE_NAME="$(cat "${WORKER_TYPE_DEFINITION_DIR}/ami-base-name")" # find out latest windows 2012 r2 ami to use... AMI_METADATA="$(aws --region "${REGION}" ec2 describe-images --owners self amazon --filters "Name=platform,Values=windows" "Name=name,Values=${AMI_BASE_NAME}" --query 'Images[*].{A:CreationDate,B:ImageId,C:Name}' --output text | sort -u | tail -1 | cut -f2,3)" @@ -17,7 +15,7 @@ AMI="$(echo $AMI_METADATA | sed 's/ .*//')" AMI_NAME="$(echo $AMI_METADATA | sed 's/.* //')" log "Base AMI is: ${AMI} ('${AMI_NAME}')" -. ../find_old_aws_objects.sh +. "$(dirname "${0}")/find_old_aws_objects.sh" # make sure we have an ssh security group in this region # note if we *try* to create a security group that already exists (regardless of whether it is successful or not), there will be a cloudwatch alarm, so avoid this @@ -33,7 +31,7 @@ done # create new base ami, and apply user-data # filter output, to get INSTANCE_ID -INSTANCE_ID="$(aws --region "${REGION}" ec2 run-instances --image-id "${AMI}" --key-name "${WORKER_TYPE}_${REGION}" --security-groups "rdp-only" "ssh-only" --user-data "$(cat userdata)" --instance-type c4.2xlarge --block-device-mappings DeviceName=/dev/sda1,Ebs='{VolumeSize=75,DeleteOnTermination=true,VolumeType=gp2}' --instance-initiated-shutdown-behavior stop --client-token "${SLUGID}" --query 'Instances[*].InstanceId' --output text)" +INSTANCE_ID="$(aws --region "${REGION}" ec2 run-instances --image-id "${AMI}" --key-name "${WORKER_TYPE}_${REGION}" --security-groups "rdp-only" "ssh-only" --user-data "$(cat "${WORKER_TYPE_DEFINITION_DIR}/userdata")" --instance-type c4.2xlarge --block-device-mappings DeviceName=/dev/sda1,Ebs='{VolumeSize=75,DeleteOnTermination=true,VolumeType=gp2}' --instance-initiated-shutdown-behavior stop --client-token "${SLUGID}" --query 'Instances[*].InstanceId' --output text)" log "I've triggered the creation of instance ${INSTANCE_ID} - it can take a \x1B[4mVery Long Time™\x1B[24m for it to be created and bootstrapped..." aws --region "${REGION}" ec2 create-tags --resources "${INSTANCE_ID}" --tags "Key=WorkerType,Value=aws-provisioner-v1/${WORKER_TYPE}" "Key=Name,Value=${WORKER_TYPE} base instance" "Key=TC-Windows-Base,Value=true" @@ -50,11 +48,11 @@ done log "Now snapshotting the instance to create an AMI..." # now capture the AMI -IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} mozillabuild version ${SLUGID}" --description "firefox desktop builds on windows - taskcluster worker - version ${SLUGID}" --output text)" +IMAGE_ID="$(aws --region "${REGION}" ec2 create-image --instance-id "${INSTANCE_ID}" --name "${WORKER_TYPE} version ${SLUGID}" --description "taskcluster generic-worker worker type - ${WORKER_TYPE} - version ${SLUGID}" --output text)" log "The AMI is currently being created: ${IMAGE_ID}" -PASSWORD="$(aws --region "${REGION}" ec2 get-password-data --instance-id "${INSTANCE_ID}" --priv-launch-key ${REGION}.id_rsa --output text --query PasswordData)" +PASSWORD="$(aws --region "${REGION}" ec2 get-password-data --instance-id "${INSTANCE_ID}" --priv-launch-key "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.id_rsa" --output text --query PasswordData)" log "To connect to the template instance (please don't do so until AMI creation process is completed"'!'"):" log '' @@ -73,16 +71,16 @@ until aws --region "${REGION}" ec2 wait image-available --image-ids "${IMAGE_ID} sleep 30 done -touch "${REGION}.${IMAGE_ID}.latest-ami" +touch "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.${IMAGE_ID}.latest-ami" { echo "Instance: ${INSTANCE_ID}" echo "Public IP: ${PUBLIC_IP}" echo "Password: ${PASSWORD}" echo "AMI: ${IMAGE_ID}" -} > "${REGION}.secrets" +} > "${WORKER_TYPE_DEFINITION_DIR}/${REGION}.secrets" -. ../delete.sh +. "$(dirname "${0}")/delete.sh" # log '' # log "Starting instance ${INSTANCE_ID} back up..." diff --git a/worker_types/worker_type.sh b/aws/update-worker-types/worker_type.sh similarity index 57% rename from worker_types/worker_type.sh rename to aws/update-worker-types/worker_type.sh index 4e511631..0f42052f 100755 --- a/worker_types/worker_type.sh +++ b/aws/update-worker-types/worker_type.sh @@ -6,7 +6,6 @@ ############ newly generated AMIs). # TODO: [pmoore] submit a task after updating worker type -# TODO: [pmoore] publish ssh key to secret store after generating it echo "$(date): Checking inputs..." @@ -18,31 +17,38 @@ fi export WORKER_TYPE="${1}" export ACTION="${2}" -if [ ! -d "$(dirname "${0}")/${WORKER_TYPE}" ]; then - echo "ERROR: No directory for worker type: '$(dirname "${0}")/${WORKER_TYPE}'" +# Default directory to look for definitions is current directory. +# To select a different directory, simply export WORKER_TYPES_DIR +# to chosen directory before calling this script. +WORKER_TYPES_DIR=${WORKER_TYPES_DIR:-.} + +# Note we export this env var for subshells, rathing than passing explicitly as +# a command line argument, to keep xargs commands simple later on. +export WORKER_TYPE_DEFINITION_DIR="${WORKER_TYPES_DIR}/${WORKER_TYPE}" + +if [ ! -d "${WORKER_TYPE_DEFINITION_DIR}" ]; then + echo "ERROR: No directory for worker type: '${WORKER_TYPE_DEFINITION_DIR}'" >&2 + echo "Note, if your worker type definitions are stored locally in a different directory, please export WORKER_TYPES_DIR" >&2 exit 65 fi echo "$(date): Starting"'!' -# cd into directory containing script... -cd "$(dirname "${0}")/${WORKER_TYPE}" - # needed to not confuse the script later rm -f *.latest-ami # generate a random slugid for aws client token... go get github.com/taskcluster/slugid-go/slug -go install github.com/taskcluster/generic-worker/update-worker-type -export SLUGID=$("${GOPATH}/bin/slug") +go install github.com/taskcluster/generic-worker/aws/cmd/update-worker-type +export SLUGID=$("$(go env GOPATH)/bin/slug") # aws ec2 describe-regions --query '{A:Regions[*].RegionName}' --output text | grep -v sa-east-1 | while read x REGION; do # (skip sa-east-1 since it doesn't support all the APIs we use in this script) -echo us-west-1 118 us-west-2 199 us-east-1 100 | xargs -P32 -n2 ../process_region.sh +echo us-west-1 118 us-west-2 199 us-east-1 100 | xargs -P32 -n2 "$(dirname "${0}")/process_region.sh" if [ "${ACTION}" == "update" ]; then - "${GOPATH}/bin/update-worker-type" . + "$(go env GOPATH)/bin/update-worker-type" "${WORKER_TYPE_DEFINITION_DIR}" echo echo "The worker type has been proactively updated("'!'"):" echo diff --git a/cmd/download-secrets/main.go b/cmd/download-secrets/main.go new file mode 100644 index 00000000..ac00cf10 --- /dev/null +++ b/cmd/download-secrets/main.go @@ -0,0 +1,144 @@ +package main + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "sync" + "time" + + "github.com/taskcluster/httpbackoff" + "github.com/taskcluster/taskcluster-client-go/tcsecrets" +) + +type SecretFetcher struct { + Secrets *tcsecrets.Secrets + RequestChannel <-chan string + DownloadDirectory string + ProcessedChannel chan<- string +} + +type WorkerPool struct { + Workers []*SecretFetcher + WaitGroup sync.WaitGroup +} + +// This program is a simple command line utility. When you run it, it will +// spawn 20 go routines to download taskcluster secrets in parallel. It will +// put them in a sub-directory called secrets, creating it if necessary, with +// each file named as the secret. Path separators in the secret name will +// result in created subdirectories. I use this in combination with a cron job +// that runs +// https://github.com/petemoore/myscrapbook/blob/master/sync-secrets.sh every 5 +// mins, in order to maintain a git history of secrets locally, in case +// something goes horribly wrong and we need to restore them. I don't publish +// the git repository anywhere, obviously! +func main() { + ss := tcsecrets.NewFromEnv() + downloadDirectory := "secrets" + requestChannel := make(chan string) + processedChannel := make(chan string) + _ = NewWorkerPool(20, requestChannel, processedChannel, ss, downloadDirectory) + contToken := "" + allSecrets := []string{} + for { + moreSecrets, err := ss.List(contToken, "") + if err != nil { + panic(err) + } + allSecrets = append(allSecrets, moreSecrets.Secrets...) + contToken = moreSecrets.ContinuationToken + if contToken == "" { + break + } + } + err := os.MkdirAll(downloadDirectory, 0755) + if err != nil { + panic(err) + } + go func() { + defer close(requestChannel) + for _, secret := range allSecrets { + requestChannel <- secret + } + }() + for completedSecret := range processedChannel { + fmt.Println(completedSecret) + } +} + +func NewWorkerPool(capacity int, requestChannel <-chan string, processedChannel chan<- string, ss *tcsecrets.Secrets, downloadDirectory string) *WorkerPool { + wp := &WorkerPool{} + wp.WaitGroup.Add(capacity) + wp.Workers = make([]*SecretFetcher, capacity, capacity) + for i := 0; i < capacity; i++ { + wp.Workers[i] = &SecretFetcher{ + Secrets: ss, + RequestChannel: requestChannel, + DownloadDirectory: downloadDirectory, + ProcessedChannel: processedChannel, + } + go func(i int) { + wp.Workers[i].FetchUntilDone(&wp.WaitGroup) + }(i) + } + go func() { + wp.WaitGroup.Wait() + close(processedChannel) + }() + return wp +} + +func (secretFetcher *SecretFetcher) FetchUntilDone(wg *sync.WaitGroup) { + for secret := range secretFetcher.RequestChannel { + err := secretFetcher.fetch(secret) + if err != nil { + panic(err) + } + } + wg.Done() +} + +func (secretFetcher *SecretFetcher) fetch(secret string) (err error) { + var u *url.URL + u, err = secretFetcher.Secrets.Get_SignedURL(secret, time.Minute) + if err != nil { + return + } + var resp *http.Response + resp, _, err = httpbackoff.Get(u.String()) + if err != nil { + return + } + defer func() { + if err == nil { + err = resp.Body.Close() + } else { + resp.Body.Close() + } + }() + var file *os.File + path := filepath.Join(secretFetcher.DownloadDirectory, secret) + dir := filepath.Dir(path) + err = os.MkdirAll(dir, 0755) + if err != nil { + panic(err) + } + file, err = os.Create(path) + if err != nil { + return + } + defer func() { + if err == nil { + err = file.Close() + } else { + file.Close() + } + }() + io.Copy(file, resp.Body) + secretFetcher.ProcessedChannel <- secret + return +} diff --git a/artifacts.go b/cmd/generic-worker/artifacts.go similarity index 100% rename from artifacts.go rename to cmd/generic-worker/artifacts.go diff --git a/artifacts_test.go b/cmd/generic-worker/artifacts_test.go similarity index 100% rename from artifacts_test.go rename to cmd/generic-worker/artifacts_test.go diff --git a/aws.go b/cmd/generic-worker/aws.go similarity index 99% rename from aws.go rename to cmd/generic-worker/aws.go index 83b2c296..34d16d0d 100644 --- a/aws.go +++ b/cmd/generic-worker/aws.go @@ -16,7 +16,7 @@ import ( "strings" "time" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/taskcluster-client-go/tcawsprovisioner" ) diff --git a/aws_helper_test.go b/cmd/generic-worker/aws_helper_test.go similarity index 100% rename from aws_helper_test.go rename to cmd/generic-worker/aws_helper_test.go diff --git a/aws_test.go b/cmd/generic-worker/aws_test.go similarity index 100% rename from aws_test.go rename to cmd/generic-worker/aws_test.go diff --git a/chain_of_trust.go b/cmd/generic-worker/chain_of_trust.go similarity index 99% rename from chain_of_trust.go rename to cmd/generic-worker/chain_of_trust.go index d6ec565a..bac5e46c 100644 --- a/chain_of_trust.go +++ b/cmd/generic-worker/chain_of_trust.go @@ -12,7 +12,7 @@ import ( "golang.org/x/crypto/openpgp/clearsign" "golang.org/x/crypto/openpgp/packet" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/taskcluster-base-go/scopes" "github.com/taskcluster/taskcluster-client-go/tcqueue" ) diff --git a/chain_of_trust_all-unix-style.go b/cmd/generic-worker/chain_of_trust_all-unix-style.go similarity index 95% rename from chain_of_trust_all-unix-style.go rename to cmd/generic-worker/chain_of_trust_all-unix-style.go index ee371da3..48d26af5 100644 --- a/chain_of_trust_all-unix-style.go +++ b/cmd/generic-worker/chain_of_trust_all-unix-style.go @@ -8,7 +8,7 @@ import ( "os/user" "strconv" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" ) func (cot *ChainOfTrustTaskFeature) ensureTaskUserCantReadPrivateCotKey() error { diff --git a/chain_of_trust_windows.go b/cmd/generic-worker/chain_of_trust_windows.go similarity index 95% rename from chain_of_trust_windows.go rename to cmd/generic-worker/chain_of_trust_windows.go index 155513cd..7704559a 100644 --- a/chain_of_trust_windows.go +++ b/cmd/generic-worker/chain_of_trust_windows.go @@ -7,7 +7,7 @@ import ( "golang.org/x/sys/windows" acl "github.com/hectane/go-acl" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" ) func (cot *ChainOfTrustTaskFeature) ensureTaskUserCantReadPrivateCotKey() error { diff --git a/config_test.go b/cmd/generic-worker/config_test.go similarity index 98% rename from config_test.go rename to cmd/generic-worker/config_test.go index 786c9e38..8f529407 100644 --- a/config_test.go +++ b/cmd/generic-worker/config_test.go @@ -7,7 +7,7 @@ import ( "runtime" "testing" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" ) func TestMissingIPConfig(t *testing.T) { diff --git a/diskspace_all-unix-style.go b/cmd/generic-worker/diskspace_all-unix-style.go similarity index 100% rename from diskspace_all-unix-style.go rename to cmd/generic-worker/diskspace_all-unix-style.go diff --git a/diskspace_windows.go b/cmd/generic-worker/diskspace_windows.go similarity index 100% rename from diskspace_windows.go rename to cmd/generic-worker/diskspace_windows.go diff --git a/feature.go b/cmd/generic-worker/feature.go similarity index 100% rename from feature.go rename to cmd/generic-worker/feature.go diff --git a/garbagecollector.go b/cmd/generic-worker/garbagecollector.go similarity index 100% rename from garbagecollector.go rename to cmd/generic-worker/garbagecollector.go diff --git a/generated_all-unix-style.go b/cmd/generic-worker/generated_all-unix-style.go similarity index 100% rename from generated_all-unix-style.go rename to cmd/generic-worker/generated_all-unix-style.go diff --git a/generated_windows.go b/cmd/generic-worker/generated_windows.go similarity index 100% rename from generated_windows.go rename to cmd/generic-worker/generated_windows.go diff --git a/helper_all-unix-style_test.go b/cmd/generic-worker/helper_all-unix-style_test.go similarity index 100% rename from helper_all-unix-style_test.go rename to cmd/generic-worker/helper_all-unix-style_test.go diff --git a/helper_test.go b/cmd/generic-worker/helper_test.go similarity index 99% rename from helper_test.go rename to cmd/generic-worker/helper_test.go index 52313664..4350271a 100644 --- a/helper_test.go +++ b/cmd/generic-worker/helper_test.go @@ -19,7 +19,7 @@ import ( "testing" "time" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/slugid-go/slugid" tcclient "github.com/taskcluster/taskcluster-client-go" diff --git a/helper_windows_test.go b/cmd/generic-worker/helper_windows_test.go similarity index 100% rename from helper_windows_test.go rename to cmd/generic-worker/helper_windows_test.go diff --git a/intermittent_task_test.go b/cmd/generic-worker/intermittent_task_test.go similarity index 100% rename from intermittent_task_test.go rename to cmd/generic-worker/intermittent_task_test.go diff --git a/livelog.go b/cmd/generic-worker/livelog.go similarity index 98% rename from livelog.go rename to cmd/generic-worker/livelog.go index 0447f826..aa6115bc 100644 --- a/livelog.go +++ b/cmd/generic-worker/livelog.go @@ -9,8 +9,8 @@ import ( "strconv" "time" - "github.com/taskcluster/generic-worker/livelog" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/livelog" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/stateless-dns-go/hostname" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" diff --git a/main.go b/cmd/generic-worker/main.go similarity index 99% rename from main.go rename to cmd/generic-worker/main.go index a459f975..78808ffb 100644 --- a/main.go +++ b/cmd/generic-worker/main.go @@ -1,5 +1,5 @@ -//go:generate gw-codegen all-unix-style.yml generated_all-unix-style.go !windows -//go:generate gw-codegen windows.yml generated_windows.go +//go:generate gw-codegen ../../schemas/all-unix-style.yml generated_all-unix-style.go !windows +//go:generate gw-codegen ../../schemas/windows.yml generated_windows.go package main @@ -19,8 +19,8 @@ import ( "time" docopt "github.com/docopt/docopt-go" - "github.com/taskcluster/generic-worker/gwconfig" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/gwconfig" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" "github.com/taskcluster/taskcluster-client-go/tcauth" diff --git a/main_test.go b/cmd/generic-worker/main_test.go similarity index 98% rename from main_test.go rename to cmd/generic-worker/main_test.go index 762e242b..75711a4d 100644 --- a/main_test.go +++ b/cmd/generic-worker/main_test.go @@ -110,7 +110,7 @@ func TestIdleWithoutCrash(t *testing.T) { func TestRevisionNumberStored(t *testing.T) { if !regexp.MustCompile("^[0-9a-f]{40}$").MatchString(revision) { t.Fatalf("Git revision could not be determined - got '%v' but expected to match regular expression '^[0-9a-f](40)$'\n"+ - "Did you specify `-ldflags \"-X github.com/taskcluster/generic-worker.revision=\"` in your go test command?\n"+ + "Did you specify `-ldflags \"-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=\"` in your go test command?\n"+ "Try using build.sh / build.cmd in root directory of generic-worker source code repository.", revision) } t.Logf("Git revision successfully retrieved: %v", revision) diff --git a/model.go b/cmd/generic-worker/model.go similarity index 99% rename from model.go rename to cmd/generic-worker/model.go index 558fd98d..e52672d1 100644 --- a/model.go +++ b/cmd/generic-worker/model.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/taskcluster-client-go/tcqueue" ) diff --git a/mounts.go b/cmd/generic-worker/mounts.go similarity index 99% rename from mounts.go rename to cmd/generic-worker/mounts.go index 56d21027..c423c7b7 100644 --- a/mounts.go +++ b/cmd/generic-worker/mounts.go @@ -15,7 +15,7 @@ import ( "time" "github.com/mholt/archiver" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/slugid-go/slugid" "github.com/taskcluster/taskcluster-base-go/scopes" diff --git a/mounts_test.go b/cmd/generic-worker/mounts_test.go similarity index 99% rename from mounts_test.go rename to cmd/generic-worker/mounts_test.go index d56c968d..04d7e153 100644 --- a/mounts_test.go +++ b/cmd/generic-worker/mounts_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" ) func TestMounts(t *testing.T) { diff --git a/mounts_windows_test.go b/cmd/generic-worker/mounts_windows_test.go similarity index 100% rename from mounts_windows_test.go rename to cmd/generic-worker/mounts_windows_test.go diff --git a/openpgp.go b/cmd/generic-worker/openpgp.go similarity index 100% rename from openpgp.go rename to cmd/generic-worker/openpgp.go diff --git a/os_groups.go b/cmd/generic-worker/os_groups.go similarity index 100% rename from os_groups.go rename to cmd/generic-worker/os_groups.go diff --git a/os_groups_all-unix-style.go b/cmd/generic-worker/os_groups_all-unix-style.go similarity index 100% rename from os_groups_all-unix-style.go rename to cmd/generic-worker/os_groups_all-unix-style.go diff --git a/os_groups_all-unix-style_test.go b/cmd/generic-worker/os_groups_all-unix-style_test.go similarity index 100% rename from os_groups_all-unix-style_test.go rename to cmd/generic-worker/os_groups_all-unix-style_test.go diff --git a/os_groups_windows.go b/cmd/generic-worker/os_groups_windows.go similarity index 100% rename from os_groups_windows.go rename to cmd/generic-worker/os_groups_windows.go diff --git a/os_groups_windows_test.go b/cmd/generic-worker/os_groups_windows_test.go similarity index 100% rename from os_groups_windows_test.go rename to cmd/generic-worker/os_groups_windows_test.go diff --git a/payload_test.go b/cmd/generic-worker/payload_test.go similarity index 100% rename from payload_test.go rename to cmd/generic-worker/payload_test.go diff --git a/plat_all-unix-style.go b/cmd/generic-worker/plat_all-unix-style.go similarity index 98% rename from plat_all-unix-style.go rename to cmd/generic-worker/plat_all-unix-style.go index be0abc97..4bde3373 100644 --- a/plat_all-unix-style.go +++ b/cmd/generic-worker/plat_all-unix-style.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/taskcluster/generic-worker/process" + "github.com/taskcluster/generic-worker/lib/process" "github.com/taskcluster/shell" ) diff --git a/plat_all-unix-style_test.go b/cmd/generic-worker/plat_all-unix-style_test.go similarity index 100% rename from plat_all-unix-style_test.go rename to cmd/generic-worker/plat_all-unix-style_test.go diff --git a/plat_darwin.go b/cmd/generic-worker/plat_darwin.go similarity index 100% rename from plat_darwin.go rename to cmd/generic-worker/plat_darwin.go diff --git a/plat_windows.go b/cmd/generic-worker/plat_windows.go similarity index 99% rename from plat_windows.go rename to cmd/generic-worker/plat_windows.go index 25839b70..766e3b15 100644 --- a/plat_windows.go +++ b/cmd/generic-worker/plat_windows.go @@ -16,9 +16,9 @@ import ( "unsafe" "github.com/dchest/uniuri" - "github.com/taskcluster/generic-worker/process" - "github.com/taskcluster/generic-worker/runtime" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/process" + "github.com/taskcluster/generic-worker/lib/runtime" + "github.com/taskcluster/generic-worker/lib/win32" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" ) @@ -59,7 +59,7 @@ func (task *TaskRun) NewPlatformData() (pd *PlatformData, err error) { if filepath.Base(os.Args[0]) == "generic-worker.exe" { exe = os.Args[0] } else { - exe = `..\..\..\..\bin\generic-worker.exe` + exe = `..\..\..\..\..\..\bin\generic-worker.exe` } cmd, err := process.NewCommand([]string{exe, "grant-winsta-access", "--sid", sid}, cwd, []string{}, pd.LoginInfo.AccessToken()) cmd.DirectOutput(os.Stdout) diff --git a/plat_windows_test.go b/cmd/generic-worker/plat_windows_test.go similarity index 100% rename from plat_windows_test.go rename to cmd/generic-worker/plat_windows_test.go diff --git a/rdp_windows.go b/cmd/generic-worker/rdp_windows.go similarity index 97% rename from rdp_windows.go rename to cmd/generic-worker/rdp_windows.go index c6c385bf..d0b06178 100644 --- a/rdp_windows.go +++ b/cmd/generic-worker/rdp_windows.go @@ -5,7 +5,7 @@ import ( "path/filepath" "time" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" ) diff --git a/runasadministrator_windows.go b/cmd/generic-worker/runasadministrator_windows.go similarity index 97% rename from runasadministrator_windows.go rename to cmd/generic-worker/runasadministrator_windows.go index 423dec88..fefd5ac0 100644 --- a/runasadministrator_windows.go +++ b/cmd/generic-worker/runasadministrator_windows.go @@ -3,7 +3,7 @@ package main import ( "fmt" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/win32" "github.com/taskcluster/taskcluster-base-go/scopes" ) diff --git a/runasadministrator_windows_test.go b/cmd/generic-worker/runasadministrator_windows_test.go similarity index 100% rename from runasadministrator_windows_test.go rename to cmd/generic-worker/runasadministrator_windows_test.go diff --git a/runtime.go b/cmd/generic-worker/runtime.go similarity index 100% rename from runtime.go rename to cmd/generic-worker/runtime.go diff --git a/sentry.go b/cmd/generic-worker/sentry.go similarity index 100% rename from sentry.go rename to cmd/generic-worker/sentry.go diff --git a/supersede.go b/cmd/generic-worker/supersede.go similarity index 98% rename from supersede.go rename to cmd/generic-worker/supersede.go index af676244..6d718330 100644 --- a/supersede.go +++ b/cmd/generic-worker/supersede.go @@ -5,7 +5,7 @@ import ( "fmt" "path/filepath" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" "github.com/taskcluster/httpbackoff" "github.com/taskcluster/taskcluster-base-go/scopes" ) diff --git a/supersede_test.go b/cmd/generic-worker/supersede_test.go similarity index 100% rename from supersede_test.go rename to cmd/generic-worker/supersede_test.go diff --git a/taskcluster_proxy.go b/cmd/generic-worker/taskcluster_proxy.go similarity index 98% rename from taskcluster_proxy.go rename to cmd/generic-worker/taskcluster_proxy.go index b4d6ac09..e3cb8bad 100644 --- a/taskcluster_proxy.go +++ b/cmd/generic-worker/taskcluster_proxy.go @@ -7,7 +7,7 @@ import ( "log" "net/http" - "github.com/taskcluster/generic-worker/tcproxy" + "github.com/taskcluster/generic-worker/lib/tcproxy" "github.com/taskcluster/taskcluster-base-go/scopes" tcclient "github.com/taskcluster/taskcluster-client-go" ) diff --git a/taskcluster_proxy_test.go b/cmd/generic-worker/taskcluster_proxy_test.go similarity index 100% rename from taskcluster_proxy_test.go rename to cmd/generic-worker/taskcluster_proxy_test.go diff --git a/taskstatus.go b/cmd/generic-worker/taskstatus.go similarity index 100% rename from taskstatus.go rename to cmd/generic-worker/taskstatus.go diff --git a/taskstatus_test.go b/cmd/generic-worker/taskstatus_test.go similarity index 100% rename from taskstatus_test.go rename to cmd/generic-worker/taskstatus_test.go diff --git a/testdata/SampleArtifacts/%%%/v/X b/cmd/generic-worker/testdata/SampleArtifacts/%%%/v/X similarity index 100% rename from testdata/SampleArtifacts/%%%/v/X rename to cmd/generic-worker/testdata/SampleArtifacts/%%%/v/X diff --git a/testdata/SampleArtifacts/_/X.txt b/cmd/generic-worker/testdata/SampleArtifacts/_/X.txt similarity index 100% rename from testdata/SampleArtifacts/_/X.txt rename to cmd/generic-worker/testdata/SampleArtifacts/_/X.txt diff --git a/testdata/SampleArtifacts/b/c/d.jpg b/cmd/generic-worker/testdata/SampleArtifacts/b/c/d.jpg similarity index 100% rename from testdata/SampleArtifacts/b/c/d.jpg rename to cmd/generic-worker/testdata/SampleArtifacts/b/c/d.jpg diff --git a/testdata/config/bool-as-string.json b/cmd/generic-worker/testdata/config/bool-as-string.json similarity index 100% rename from testdata/config/bool-as-string.json rename to cmd/generic-worker/testdata/config/bool-as-string.json diff --git a/testdata/config/invalid-ip.json b/cmd/generic-worker/testdata/config/invalid-ip.json similarity index 100% rename from testdata/config/invalid-ip.json rename to cmd/generic-worker/testdata/config/invalid-ip.json diff --git a/testdata/config/invalid-json.json b/cmd/generic-worker/testdata/config/invalid-json.json similarity index 100% rename from testdata/config/invalid-json.json rename to cmd/generic-worker/testdata/config/invalid-json.json diff --git a/testdata/config/noip.json b/cmd/generic-worker/testdata/config/noip.json similarity index 100% rename from testdata/config/noip.json rename to cmd/generic-worker/testdata/config/noip.json diff --git a/testdata/config/valid.json b/cmd/generic-worker/testdata/config/valid.json similarity index 100% rename from testdata/config/valid.json rename to cmd/generic-worker/testdata/config/valid.json diff --git a/testdata/config/worker-type-metadata.json b/cmd/generic-worker/testdata/config/worker-type-metadata.json similarity index 100% rename from testdata/config/worker-type-metadata.json rename to cmd/generic-worker/testdata/config/worker-type-metadata.json diff --git a/testdata/curlget.go b/cmd/generic-worker/testdata/curlget.go similarity index 100% rename from testdata/curlget.go rename to cmd/generic-worker/testdata/curlget.go diff --git a/testdata/machine-configuration.json b/cmd/generic-worker/testdata/machine-configuration.json similarity index 100% rename from testdata/machine-configuration.json rename to cmd/generic-worker/testdata/machine-configuration.json diff --git a/testdata/mouse_and_screen_resolution.py b/cmd/generic-worker/testdata/mouse_and_screen_resolution.py similarity index 100% rename from testdata/mouse_and_screen_resolution.py rename to cmd/generic-worker/testdata/mouse_and_screen_resolution.py diff --git a/testdata/move-file.go b/cmd/generic-worker/testdata/move-file.go similarity index 100% rename from testdata/move-file.go rename to cmd/generic-worker/testdata/move-file.go diff --git a/testdata/private-opengpg-key b/cmd/generic-worker/testdata/private-opengpg-key similarity index 100% rename from testdata/private-opengpg-key rename to cmd/generic-worker/testdata/private-opengpg-key diff --git a/testdata/public-openpgp-key b/cmd/generic-worker/testdata/public-openpgp-key similarity index 100% rename from testdata/public-openpgp-key rename to cmd/generic-worker/testdata/public-openpgp-key diff --git a/testdata/resolvetask.go b/cmd/generic-worker/testdata/resolvetask.go similarity index 100% rename from testdata/resolvetask.go rename to cmd/generic-worker/testdata/resolvetask.go diff --git a/testdata/run-after-user.bat b/cmd/generic-worker/testdata/run-after-user.bat similarity index 100% rename from testdata/run-after-user.bat rename to cmd/generic-worker/testdata/run-after-user.bat diff --git a/testdata/spawn-orphan-process.go b/cmd/generic-worker/testdata/spawn-orphan-process.go similarity index 100% rename from testdata/spawn-orphan-process.go rename to cmd/generic-worker/testdata/spawn-orphan-process.go diff --git a/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json b/cmd/generic-worker/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json similarity index 100% rename from testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json rename to cmd/generic-worker/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json diff --git a/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json b/cmd/generic-worker/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json similarity index 100% rename from testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json rename to cmd/generic-worker/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json diff --git a/testdata/tasks/README.md b/cmd/generic-worker/testdata/tasks/README.md similarity index 100% rename from testdata/tasks/README.md rename to cmd/generic-worker/testdata/tasks/README.md diff --git a/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json b/cmd/generic-worker/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json similarity index 100% rename from testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json rename to cmd/generic-worker/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json diff --git a/user_creation_windows_test.go b/cmd/generic-worker/user_creation_windows_test.go similarity index 100% rename from user_creation_windows_test.go rename to cmd/generic-worker/user_creation_windows_test.go diff --git a/gw-codegen/main.go b/cmd/gw-codegen/main.go similarity index 100% rename from gw-codegen/main.go rename to cmd/gw-codegen/main.go diff --git a/cmd/inspect-worker-types/main.go b/cmd/inspect-worker-types/main.go new file mode 100644 index 00000000..7a8eea6d --- /dev/null +++ b/cmd/inspect-worker-types/main.go @@ -0,0 +1,101 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "regexp" + "strings" + "time" + + "github.com/taskcluster/httpbackoff" + "github.com/taskcluster/taskcluster-client-go/tcqueue" +) + +func main() { + taskGroupID := os.Args[1] + fmt.Printf("Task group: %v\n\n", taskGroupID) + queue := tcqueue.NewFromEnv() + query := func(ct string) *tcqueue.ListTaskGroupResponse { + lgtr, err := queue.ListTaskGroup(taskGroupID, ct, "") + if err != nil { + log.Fatal(1, err) + } + for _, t := range lgtr.Tasks { + show(queue, t) + } + return lgtr + } + ct := query("").ContinuationToken + for ct != "" { + ct = query(ct).ContinuationToken + } +} + +func show(queue *tcqueue.Queue, t tcqueue.TaskDefinitionAndStatus) { + name := t.Task.ProvisionerID + "/" + t.Task.WorkerType + ": " + fmt.Print(name[:75]) + if t.Status.State == "pending" { + fmt.Printf("not yet determined - task %v still pending...\n", t.Status.TaskID) + return + } + var resp *http.Response + artifactFound := "" + for _, artifact := range []string{ + "public/logs/live_backing.log", + "public/logs/chain_of_trust.log", + } { + logURL, err := queue.GetLatestArtifact_SignedURL(t.Status.TaskID, artifact, time.Hour) + if err != nil { + log.Fatal(2, err) + } + // log.Printf("URL: %v", logURL) + resp, _, err = httpbackoff.Get(logURL.String()) + if err == nil { + artifactFound = artifact + break + } + switch e := err.(type) { + case httpbackoff.BadHttpResponseCode: + if e.HttpResponseCode == 404 { + continue + } + } + log.Fatal(3, err) + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Print("*** ") + } + logContent := string(data) + switch true { + case strings.Contains(logContent, "Task not successful due to following exception"): + fmt.Println("generic-worker - unknown version") + // case strings.Contains(logContent, "Rejecting Schema: http://schemas.taskcluster.net/docker-worker/v1/payload.json"): + //fmt.Println("worker: docker-worker - unknown version") + case strings.Contains(logContent, "Worker Node Type:"): + fmt.Println("docker-worker - unknown version") + case strings.Contains(logContent, `"release": "https://github.com/taskcluster/generic-worker/releases/tag/v`): + re := regexp.MustCompile(`"https://github.com/taskcluster/generic-worker/releases/tag/v([^"]*)"`) + match := re.FindStringSubmatch(logContent) + fmt.Printf("generic-worker %v\n", match[1]) + case strings.Contains(logContent, `not allowed at task.payload.features`): + fmt.Println("taskcluster-worker - unknown version") + case strings.Contains(logContent, `raise TaskVerificationError`): + fmt.Println("scriptworker - unknown version") + case strings.Contains(logContent, `KeyError: 'artifacts_deps'`): + fmt.Println("some kind of scriptworker - unknown version") + case artifactFound == "": + fmt.Println("No artifacts found!") + case artifactFound == "public/logs/chain_of_trust.log": + fmt.Println("scriptworker chain of trust - unknown version") + case strings.Contains(logContent, `os.environ.get('GITHUB_HEAD_REPO_URL', decision_json['payload']['env']['GITHUB_HEAD_REPO_URL'])`): + fmt.Println("scriptworker - deepspeech - unknown version") + default: + fmt.Println("UNKNOWN") + log.Fatal(5, logContent) + } +} diff --git a/cmd/list-worker-types/main.go b/cmd/list-worker-types/main.go new file mode 100644 index 00000000..a4a10d9c --- /dev/null +++ b/cmd/list-worker-types/main.go @@ -0,0 +1,195 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "sort" + "strings" + "sync" + "time" + + "github.com/taskcluster/slugid-go/slugid" + tcclient "github.com/taskcluster/taskcluster-client-go" + "github.com/taskcluster/taskcluster-client-go/tcawsprovisioner" + "github.com/taskcluster/taskcluster-client-go/tcqueue" +) + +type Queue tcqueue.Queue +type Provisioner tcawsprovisioner.AwsProvisioner + +func main() { + taskIDs := map[string]string{} + myQueue := tcqueue.NewFromEnv() + taskGroupID := slugid.Nice() + created := time.Now() + for _, wt := range AllWorkerTypes() { + fmt.Println(wt) + x := strings.Split(wt, "/") + provisionerID := x[0] + workerType := x[1] + taskID := slugid.Nice() + taskIDs[wt] = taskID + payload := GenericWorkerPayload{ + MaxRunTime: 3600, + Command: []string{ + `echo`, + }, + } + payloadJSON := mustCompileToRawMessage(payload) + taskDef := &tcqueue.TaskDefinitionRequest{ + Created: tcclient.Time(created), + Deadline: tcclient.Time(created.Add(time.Hour * 3)), + Dependencies: []string{}, + Expires: tcclient.Time(created.Add(time.Hour * 24 * 30)), + Extra: json.RawMessage("{}"), + Metadata: struct { + Description string `json:"description"` + Name string `json:"name"` + Owner string `json:"owner"` + Source string `json:"source"` + }{ + Description: "Checking worker version on " + provisionerID + "/" + workerType, + Name: "Checking worker version on " + provisionerID + "/" + workerType, + Owner: "pmoore@mozilla.com", + Source: "https://github.com/petemoore", + }, + Payload: *payloadJSON, + Priority: "highest", + ProvisionerID: provisionerID, + Requires: "all-completed", + Retries: 5, + Routes: []string{}, + SchedulerID: "-", + Scopes: []string{}, + Tags: map[string]string{}, + TaskGroupID: taskGroupID, + WorkerType: workerType, + } + tsr, err := myQueue.CreateTask(taskID, taskDef) + fatalOnError(err) + + respJSON, err := json.MarshalIndent(tsr, "", " ") + fatalOnError(err) + + fmt.Println(string(respJSON)) + } + + fmt.Println("Task Group ID: " + taskGroupID) +} + +func mustCompileToRawMessage(data interface{}) *json.RawMessage { + bytes, err := json.Marshal(data) + fatalOnError(err) + var JSON json.RawMessage + err = json.Unmarshal(bytes, &JSON) + fatalOnError(err) + return &JSON +} + +func fatalOnError(err error) { + if err != nil { + log.Fatalf("Error:\n%v", err) + } +} + +func NewProvisioner() *Provisioner { + p := tcawsprovisioner.NewFromEnv() + P := Provisioner(*p) + return &P +} + +func NewQueue() *Queue { + q := tcqueue.NewFromEnv() + Q := Queue(*q) + return &Q +} + +func (p *Provisioner) AllWorkerTypes() []string { + prov := tcawsprovisioner.AwsProvisioner(*p) + wt, err := prov.ListWorkerTypes() + if err != nil { + panic(err) + } + return []string(*wt) +} + +func AllWorkerTypes() []string { + uniqueWorkerTypes := map[string]bool{} + q := NewQueue() + provisioners := q.AllProvisionerIDs() + workerTypes := make([][]string, len(provisioners), len(provisioners)) + var wg sync.WaitGroup + for i, p := range provisioners { + if p == "test-provisioner" || p == "no-provisioning-nope" || p == "dummy-test-provisioner" || p == "test-dummy-provisioner" { + continue + } + wg.Add(1) + go func(p string, i int) { + defer wg.Done() + provWorkerTypes := q.ProvisionerWorkerTypes(p) + workerTypes[i] = make([]string, len(provWorkerTypes), len(provWorkerTypes)) + for j, wt := range provWorkerTypes { + workerTypes[i][j] = p + "/" + wt + } + }(p, i) + } + wg.Wait() + for _, p := range workerTypes { + for _, wt := range p { + uniqueWorkerTypes[wt] = true + } + } + + // Now merge in known worker types according to AWS provisioner + p := NewProvisioner() + provisionerWorkerTypes := p.AllWorkerTypes() + for _, wt := range provisionerWorkerTypes { + uniqueWorkerTypes["aws-provisioner-v1/"+wt] = true + } + + keys := make([]string, 0, len(uniqueWorkerTypes)) + for key := range uniqueWorkerTypes { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func (q *Queue) AllProvisionerIDs() []string { + Q := tcqueue.Queue(*q) + provisioners := []string{} + var r *tcqueue.ListProvisionersResponse + ct := "" + for r == nil || r.ContinuationToken != "" { + var err error + r, err = (&Q).ListProvisioners(ct, "") + if err != nil { + panic(err) + } + ct = r.ContinuationToken + for _, p := range r.Provisioners { + provisioners = append(provisioners, p.ProvisionerID) + } + } + return provisioners +} + +func (q *Queue) ProvisionerWorkerTypes(provisionerID string) []string { + Q := tcqueue.Queue(*q) + workerTypes := []string{} + var r *tcqueue.ListWorkerTypesResponse + ct := "" + for r == nil || r.ContinuationToken != "" { + var err error + r, err = (&Q).ListWorkerTypes(provisionerID, ct, "") + if err != nil { + panic(err) + } + ct = r.ContinuationToken + for _, p := range r.WorkerTypes { + workerTypes = append(workerTypes, p.WorkerType) + } + } + return workerTypes +} diff --git a/cmd/list-worker-types/types.go b/cmd/list-worker-types/types.go new file mode 100644 index 00000000..2d1a5e2a --- /dev/null +++ b/cmd/list-worker-types/types.go @@ -0,0 +1,177 @@ +package main + +import ( + "encoding/json" + + tcclient "github.com/taskcluster/taskcluster-client-go" +) + +// ********************************************************* +// These type definitions are copied from: +// https://github.com/taskcluster/generic-worker/blob/ec86473df8dba68631a50af98e5af7d44d7e1717/generated_windows.go#L40-L201 +// ********************************************************* + +type ( + // This schema defines the structure of the `payload` property referred to in a + // Taskcluster Task definition. + GenericWorkerPayload struct { + + // Artifacts to be published. + // + // Since: generic-worker 1.0.0 + Artifacts []struct { + + // Explicitly set the value of the HTTP `Content-Type` response header when the artifact(s) + // is/are served over HTTP(S). If not provided (this property is optional) the worker will + // guess the content type of artifacts based on the filename extension of the file storing + // the artifact content. It does this by looking at the system filename-to-mimetype mappings + // defined in the Windows registry. Note, setting `contentType` on a directory artifact will + // apply the same contentType to all files contained in the directory. + // + // See [mime.TypeByExtension](https://godoc.org/mime#TypeByExtension). + // + // Since: generic-worker 10.4.0 + ContentType string `json:"contentType,omitempty"` + + // Date when artifact should expire must be in the future, no earlier than task deadline, but + // no later than task expiry. If not set, defaults to task expiry. + // + // Since: generic-worker 1.0.0 + Expires tcclient.Time `json:"expires,omitempty"` + + // Name of the artifact, as it will be published. If not set, `path` will be used. + // Conventionally (although not enforced) path elements are forward slash separated. Example: + // `public/build/a/house`. Note, no scopes are required to read artifacts beginning `public/`. + // Artifact names not beginning `public/` are scope-protected (caller requires scopes to + // download the artifact). See the Queue documentation for more information. + // + // Since: generic-worker 8.1.0 + Name string `json:"name,omitempty"` + + // Relative path of the file/directory from the task directory. Note this is not an absolute + // path as is typically used in docker-worker, since the absolute task directory name is not + // known when the task is submitted. Example: `dist\regedit.exe`. It doesn't matter if + // forward slashes or backslashes are used. + // + // Since: generic-worker 1.0.0 + Path string `json:"path"` + + // Artifacts can be either an individual `file` or a `directory` containing + // potentially multiple files with recursively included subdirectories. + // + // Since: generic-worker 1.0.0 + // + // Possible values: + // * "file" + // * "directory" + Type string `json:"type"` + } `json:"artifacts,omitempty"` + + // One entry per command (consider each entry to be interpreted as a full line of + // a Windows™ .bat file). For example: + // ``` + // [ + // "set", + // "echo hello world > hello_world.txt", + // "set GOPATH=C:\\Go" + // ] + // ``` + // + // Since: generic-worker 0.0.1 + Command []string `json:"command"` + + // Env vars must be string to __string__ mappings (not number or boolean). For example: + // ``` + // { + // "PATH": "C:\\Windows\\system32;C:\\Windows", + // "GOOS": "windows", + // "FOO_ENABLE": "true", + // "BAR_TOTAL": "3" + // } + // ``` + // + // Since: generic-worker 0.0.1 + Env map[string]string `json:"env,omitempty"` + + // Feature flags enable additional functionality. + // + // Since: generic-worker 5.3.0 + Features struct { + + // An artifact named `public/chainOfTrust.json.asc` should be generated + // which will include information for downstream tasks to build a level + // of trust for the artifacts produced by the task and the environment + // it ran in. + // + // Since: generic-worker 5.3.0 + ChainOfTrust bool `json:"chainOfTrust,omitempty"` + + // The taskcluster proxy provides an easy and safe way to make authenticated + // taskcluster requests within the scope(s) of a particular task. See + // [the github project](https://github.com/taskcluster/taskcluster-proxy) for more information. + // + // Since: generic-worker 10.6.0 + TaskclusterProxy bool `json:"taskclusterProxy,omitempty"` + } `json:"features,omitempty"` + + // Maximum time the task container can run in seconds. + // + // Since: generic-worker 0.0.1 + // + // Mininum: 1 + // Maximum: 86400 + MaxRunTime int64 `json:"maxRunTime"` + + // Directories and/or files to be mounted. + // + // Since: generic-worker 5.4.0 + Mounts []Mount `json:"mounts,omitempty"` + + // A list of OS Groups that the task user should be a member of. Requires + // scope `generic-worker:os-group:` for each group listed. + // + // Since: generic-worker 6.0.0 + OSGroups []string `json:"osGroups,omitempty"` + + // Specifies an artifact name for publishing RDP connection information. + // + // Since this is potentially sensitive data, care should be taken to publish + // to a suitably locked down path, such as + // `login-identity//rdpinfo.json` which is only readable for + // the given login identity (for example + // `login-identity/mozilla-ldap/pmoore@mozilla.com/rdpInfo.txt`). See the + // [artifact namespace guide](https://docs.taskcluster.net/manual/design/namespaces#artifacts) for more information. + // + // Use of this feature requires scope + // `generic-worker:allow-rdp:/` which must be + // declared as a task scope. + // + // The RDP connection data is published during task startup so that a user + // may interact with the running task. + // + // The task environment will be retained for 12 hours after the task + // completes, to enable an interactive user to perform investigative tasks. + // After these 12 hours, the worker will delete the task's Windows user + // account, and then continue with other tasks. + // + // No guarantees are given about the resolution status of the interactive + // task, since the task is inherently non-reproducible and no automation + // should rely on this value. + // + // Since: generic-worker 10.5.0 + RdpInfo string `json:"rdpInfo,omitempty"` + + // URL of a service that can indicate tasks superseding this one; the current `taskId` + // will be appended as a query argument `taskId`. The service should return an object with + // a `supersedes` key containing a list of `taskId`s, including the supplied `taskId`. The + // tasks should be ordered such that each task supersedes all tasks appearing later in the + // list. + // + // See [superseding](https://docs.taskcluster.net/reference/platform/taskcluster-queue/docs/superseding) for more detail. + // + // Since: generic-worker 10.2.2 + SupersederURL string `json:"supersederUrl,omitempty"` + } + + Mount json.RawMessage +) diff --git a/yamltojson/main.go b/cmd/yamltojson/main.go similarity index 100% rename from yamltojson/main.go rename to cmd/yamltojson/main.go diff --git a/docs/payload.md b/docs/payload.md index faae6240..898e92a9 100644 --- a/docs/payload.md +++ b/docs/payload.md @@ -10,16 +10,32 @@ docref: true order: 1 --- -When submitting a task graph to the Task Cluster Queue (see -[createTask](/reference/platform/queue/reference/api-docs#createTask)) you must provide a -payload for defining the tasks to be executed by the worker. In the case of the -generic worker, the payload must conform to the following schema. +When submitting a task to the Taskcluster Queue (see +[createTask](/reference/platform/queue/reference/api-docs#createTask)) you must +provide a payload property for the task. This `payload` property is specific to +the worker implementation, and tells the worker what to execute, and which +artifacts to upload. This page documents the payload property for +generic-worker, on all the platforms that it is supported on. -
+* [Windows](#generic-worker-on-windows) +* [macOS](#generic-worker-on-macos) +* [Linux](#generic-worker-on-linux) + + +# Generic Worker on Windows + +
+ +# Generic Worker on macOS + +
+ +# Generic Worker on Linux + +
The payload comprises of a command to run, environment variables to be set (optionally encrypted) and a timeout for the task (`maxRunTime`). The worker will run the task, upload log files, and report back status to the Queue. - diff --git a/fileutil/fileutil.go b/lib/fileutil/fileutil.go similarity index 100% rename from fileutil/fileutil.go rename to lib/fileutil/fileutil.go diff --git a/gwconfig/config.go b/lib/gwconfig/config.go similarity index 99% rename from gwconfig/config.go rename to lib/gwconfig/config.go index c246a333..6157d91b 100644 --- a/gwconfig/config.go +++ b/lib/gwconfig/config.go @@ -8,7 +8,7 @@ import ( "reflect" "runtime" - "github.com/taskcluster/generic-worker/fileutil" + "github.com/taskcluster/generic-worker/lib/fileutil" ) type ( diff --git a/gwconfig/mergeconfig.go b/lib/gwconfig/mergeconfig.go similarity index 100% rename from gwconfig/mergeconfig.go rename to lib/gwconfig/mergeconfig.go diff --git a/livelog/livelog.go b/lib/livelog/livelog.go similarity index 100% rename from livelog/livelog.go rename to lib/livelog/livelog.go diff --git a/livelog/livelog_test.go b/lib/livelog/livelog_test.go similarity index 100% rename from livelog/livelog_test.go rename to lib/livelog/livelog_test.go diff --git a/process/logininfo_windows.go b/lib/process/logininfo_windows.go similarity index 98% rename from process/logininfo_windows.go rename to lib/process/logininfo_windows.go index f70464b0..344c1b54 100644 --- a/process/logininfo_windows.go +++ b/lib/process/logininfo_windows.go @@ -7,7 +7,7 @@ import ( "time" "unsafe" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/win32" ) // LoginInfo represents a logged in user session diff --git a/process/misc_windows.go b/lib/process/misc_windows.go similarity index 100% rename from process/misc_windows.go rename to lib/process/misc_windows.go diff --git a/process/process.go b/lib/process/process.go similarity index 100% rename from process/process.go rename to lib/process/process.go diff --git a/process/process_all-unix-style.go b/lib/process/process_all-unix-style.go similarity index 100% rename from process/process_all-unix-style.go rename to lib/process/process_all-unix-style.go diff --git a/process/process_windows.go b/lib/process/process_windows.go similarity index 97% rename from process/process_windows.go rename to lib/process/process_windows.go index ea9a52dc..5b6f729b 100644 --- a/process/process_windows.go +++ b/lib/process/process_windows.go @@ -10,8 +10,8 @@ import ( "syscall" "time" - "github.com/taskcluster/generic-worker/runtime" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/runtime" + "github.com/taskcluster/generic-worker/lib/win32" ) type Command struct { diff --git a/runtime/runtime_windows.go b/lib/runtime/runtime_windows.go similarity index 100% rename from runtime/runtime_windows.go rename to lib/runtime/runtime_windows.go diff --git a/tcproxy/tcproxy.go b/lib/tcproxy/tcproxy.go similarity index 100% rename from tcproxy/tcproxy.go rename to lib/tcproxy/tcproxy.go diff --git a/tcproxy/tcproxy_test.go b/lib/tcproxy/tcproxy_test.go similarity index 100% rename from tcproxy/tcproxy_test.go rename to lib/tcproxy/tcproxy_test.go diff --git a/win32/win32_windows.go b/lib/win32/win32_windows.go similarity index 99% rename from win32/win32_windows.go rename to lib/win32/win32_windows.go index 83cab137..58191723 100644 --- a/win32/win32_windows.go +++ b/lib/win32/win32_windows.go @@ -196,6 +196,7 @@ func IsWindows8OrGreater() bool { return r } +// https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-logonuserw func LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32) (token syscall.Token, err error) { r1, _, e1 := procLogonUserW.Call( uintptr(unsafe.Pointer(username)), diff --git a/win32/win32_windows_386.go b/lib/win32/win32_windows_386.go similarity index 100% rename from win32/win32_windows_386.go rename to lib/win32/win32_windows_386.go diff --git a/win32/win32_windows_amd64.go b/lib/win32/win32_windows_amd64.go similarity index 100% rename from win32/win32_windows_amd64.go rename to lib/win32/win32_windows_amd64.go diff --git a/win32/win32_windows_test.go b/lib/win32/win32_windows_test.go similarity index 87% rename from win32/win32_windows_test.go rename to lib/win32/win32_windows_test.go index c7ef4ed7..9c816ed6 100644 --- a/win32/win32_windows_test.go +++ b/lib/win32/win32_windows_test.go @@ -4,7 +4,7 @@ import ( "fmt" "log" - "github.com/taskcluster/generic-worker/win32" + "github.com/taskcluster/generic-worker/lib/win32" ) func ExampleMergeEnvLists() { diff --git a/win32/win32acl_windows.go b/lib/win32/win32acl_windows.go similarity index 100% rename from win32/win32acl_windows.go rename to lib/win32/win32acl_windows.go diff --git a/win32/wrap_dll_windows.go b/lib/win32/wrap_dll_windows.go similarity index 100% rename from win32/wrap_dll_windows.go rename to lib/win32/wrap_dll_windows.go diff --git a/mozilla-try-scripts/nss.patch b/mozilla-try-scripts/nss.patch deleted file mode 100644 index 1ed2a2a3..00000000 --- a/mozilla-try-scripts/nss.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js ---- a/automation/taskcluster/graph/src/extend.js -+++ b/automation/taskcluster/graph/src/extend.js -@@ -595,11 +595,11 @@ async function scheduleTestBuilds(base, - - /*****************************************************************************/ - - async function scheduleWindows(name, base, build_script) { - base = merge(base, { -- workerType: "nss-win2012r2", -+ workerType: "nss-win2012r2-new", - env: { - PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" + - "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" + - "c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;" + - "c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;" + - diff --git a/mozilla/OpenCloudConfig/generate_occ_userdata.sh b/mozilla/OpenCloudConfig/generate_occ_userdata.sh new file mode 100755 index 00000000..99194bd3 --- /dev/null +++ b/mozilla/OpenCloudConfig/generate_occ_userdata.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e +THIS_SCRIPT_DIR="$(dirname "${0}")" + +# Default directory to look for definitions is current directory. +# To select a different directory, simply export WORKER_TYPES_DIR +# to chosen directory before calling this script. +export WORKER_TYPES_DIR=${WORKER_TYPES_DIR:-.} + +go install ./transform-occ + +echo "Removing..." +rm -v "${WORKER_TYPES_DIR}"/gecko-*/userdata + +echo "Generating..." +curl -L 'https://github.com/mozilla-releng/OpenCloudConfig/tree/master/userdata/Manifest' 2>/dev/null | sed -n 's/.*\(gecko[^.]*\)\.json.*/\1/p' | sort -u | xargs -n 1 -P 32 "${THIS_SCRIPT_DIR}/transform.sh" diff --git a/occ-workers/main.go b/mozilla/OpenCloudConfig/occ-workers/main.go similarity index 100% rename from occ-workers/main.go rename to mozilla/OpenCloudConfig/occ-workers/main.go diff --git a/worker_types/refresh-gw-configs/main.go b/mozilla/OpenCloudConfig/refresh-gw-configs/main.go similarity index 98% rename from worker_types/refresh-gw-configs/main.go rename to mozilla/OpenCloudConfig/refresh-gw-configs/main.go index c608f158..50538860 100644 --- a/worker_types/refresh-gw-configs/main.go +++ b/mozilla/OpenCloudConfig/refresh-gw-configs/main.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/generic-worker/lib/gwconfig" "github.com/taskcluster/taskcluster-client-go/tcawsprovisioner" ) diff --git a/worker_types/transform-occ/main.go b/mozilla/OpenCloudConfig/transform-occ/main.go similarity index 96% rename from worker_types/transform-occ/main.go rename to mozilla/OpenCloudConfig/transform-occ/main.go index a31debff..fcff093f 100644 --- a/worker_types/transform-occ/main.go +++ b/mozilla/OpenCloudConfig/transform-occ/main.go @@ -98,6 +98,11 @@ func main() { fmt.Println(``) fmt.Println(``) + fmt.Println(`# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation`) + fmt.Println(`# steps that actually occur on this worker type. It is generated by`) + fmt.Println(`# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh`) + fmt.Println(`# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/` + workerType + `.json`) + fmt.Println(``) fmt.Println(`# use TLS 1.2 (see bug 1443595)`) fmt.Println(`[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12`) fmt.Println(``) diff --git a/worker_types/transform-occ/main_test.go b/mozilla/OpenCloudConfig/transform-occ/main_test.go similarity index 100% rename from worker_types/transform-occ/main_test.go rename to mozilla/OpenCloudConfig/transform-occ/main_test.go diff --git a/mozilla/OpenCloudConfig/transform.sh b/mozilla/OpenCloudConfig/transform.sh new file mode 100755 index 00000000..f5046df3 --- /dev/null +++ b/mozilla/OpenCloudConfig/transform.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +echo "${1}" +mkdir -p "${WORKER_TYPES_DIR}/${1}" +"$(go env GOPATH)/bin/transform-occ" "${1}" > "${WORKER_TYPES_DIR}/${1}/userdata" || rm "${WORKER_TYPES_DIR}/${1}/userdata" diff --git a/mozilla-try-scripts/waitforOCC.go b/mozilla/OpenCloudConfig/waitforOCC.go similarity index 100% rename from mozilla-try-scripts/waitforOCC.go rename to mozilla/OpenCloudConfig/waitforOCC.go diff --git a/mozilla-try-scripts/gecko-try.sh b/mozilla/gecko/gecko-try.sh similarity index 100% rename from mozilla-try-scripts/gecko-try.sh rename to mozilla/gecko/gecko-try.sh diff --git a/mozilla-try-scripts/gecko.patch b/mozilla/gecko/gecko.patch similarity index 100% rename from mozilla-try-scripts/gecko.patch rename to mozilla/gecko/gecko.patch diff --git a/mozilla-try-scripts/lib/tooltool.py b/mozilla/gecko/lib/tooltool.py similarity index 100% rename from mozilla-try-scripts/lib/tooltool.py rename to mozilla/gecko/lib/tooltool.py diff --git a/mozilla-try-scripts/nss-try.sh b/mozilla/nss/nss-try.sh similarity index 61% rename from mozilla-try-scripts/nss-try.sh rename to mozilla/nss/nss-try.sh index 49cd604a..03c867a4 100755 --- a/mozilla-try-scripts/nss-try.sh +++ b/mozilla/nss/nss-try.sh @@ -19,7 +19,7 @@ function open_browser_page { cd "$(dirname "${0}")" THIS_SCRIPT_DIR="$(pwd)" -NEW_VERSION="$(cat ../worker_types/nss-win2012r2-new/userdata | sed -n 's_.*https://github\.com/taskcluster/generic-worker/releases/download/v\(.*\)/generic-worker-windows-amd64\.exe.*_\1_p')" +NEW_VERSION="$(cat ../worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata | sed -n 's_.*https://github\.com/taskcluster/generic-worker/releases/download/v\(.*\)/generic-worker-windows-amd64\.exe.*_\1_p')" VALID_FORMAT='^[1-9][0-9]*\.\(0\|[1-9][0-9]*\)\.\(0\|[1-9]\)\([0-9]*alpha[1-9][0-9]*\|[0-9]*\)$' if ! echo "${NEW_VERSION}" | grep -q "${VALID_FORMAT}"; then @@ -27,14 +27,20 @@ if ! echo "${NEW_VERSION}" | grep -q "${VALID_FORMAT}"; then exit 65 fi -../worker_types/worker_type.sh nss-win2012r2-new update +export WORKER_TYPES_DIR=../worker-type-host-definitions/aws-provisioner-v1 +../../aws/update-worker-types/worker_type.sh nss-win2012r2-new update NSS_CHECKOUT="$(mktemp -d -t nss-checkout.XXXXXXXXXX)" cd "${NSS_CHECKOUT}" hg clone https://hg.mozilla.org/projects/nss cd nss -patch -p1 -i "${THIS_SCRIPT_DIR}/nss.patch" -hg commit -m "Testing generic-worker ${NEW_VERSION} on nss-win2012r2-new worker type; try: -p win32,win64 -t none -u all" +grep -rl nss-win2012r2 . | while read FILE +do + cp "${FILE}" "${FILE}.x" + cat "${FILE}.x" | sed 's/nss-win2012r2/nss-win2012r2-new/g' > "${FILE}" + rm "${FILE}.x" +done +hg commit -m "Testing generic-worker ${NEW_VERSION} on nss-win2012r2-new worker type; try: -p win,win64 -t none -u all" hg push -f ssh://hg.mozilla.org/projects/nss-try -r . open_browser_page 'https://treeherder.mozilla.org/#/jobs?repo=nss-try' diff --git a/worker_types/ami-test-win2012r2/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name similarity index 100% rename from worker_types/ami-test-win2012r2/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/ami-base-name diff --git a/worker_types/ami-test-win2012r2/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata similarity index 100% rename from worker_types/ami-test-win2012r2/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win2012r2/userdata diff --git a/worker_types/ami-test-win7sp1/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name similarity index 100% rename from worker_types/ami-test-win7sp1/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/ami-base-name diff --git a/worker_types/ami-test-win7sp1/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata similarity index 100% rename from worker_types/ami-test-win7sp1/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/ami-test-win7sp1/userdata diff --git a/worker_types/gecko-1-b-win2012-beta/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name similarity index 100% rename from worker_types/gecko-1-b-win2012-beta/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/ami-base-name diff --git a/worker_types/gecko-2-b-win2012/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata similarity index 98% rename from worker_types/gecko-2-b-win2012/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata index 1a1d7d64..03d8a612 100644 --- a/worker_types/gecko-2-b-win2012/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012-beta/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-1-b-win2012-beta.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-1-b-win2012/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name similarity index 100% rename from worker_types/gecko-1-b-win2012/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/ami-base-name diff --git a/worker_types/gecko-1-b-win2012-beta/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata similarity index 98% rename from worker_types/gecko-1-b-win2012-beta/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata index 1a1d7d64..7fce39f9 100644 --- a/worker_types/gecko-1-b-win2012-beta/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-1-b-win2012/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-1-b-win2012.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-2-b-win2012/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name similarity index 100% rename from worker_types/gecko-2-b-win2012/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/ami-base-name diff --git a/worker_types/gecko-1-b-win2012/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata similarity index 98% rename from worker_types/gecko-1-b-win2012/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata index 1a1d7d64..d26a7181 100644 --- a/worker_types/gecko-1-b-win2012/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-2-b-win2012/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-2-b-win2012.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-3-b-win2012/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name similarity index 100% rename from worker_types/gecko-3-b-win2012/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/ami-base-name diff --git a/worker_types/gecko-3-b-win2012/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata similarity index 97% rename from worker_types/gecko-3-b-win2012/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata index f42c5300..d9abbdb4 100644 --- a/worker_types/gecko-3-b-win2012/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-3-b-win2012/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-3-b-win2012.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-t-win10-64-beta/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-beta/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/ami-base-name diff --git a/worker_types/gecko-t-win10-64-beta/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata similarity index 98% rename from worker_types/gecko-t-win10-64-beta/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata index c65486a4..cd8d432e 100644 --- a/worker_types/gecko-t-win10-64-beta/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-beta/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-beta.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -360,7 +365,7 @@ $client.DownloadFile("http://hg.mozilla.org/mozilla-central/raw-file/360ab7771e2 Start-Process "mofcomp" -ArgumentList "`"C:\Program Files (x86)\Windows Kits\10\Windows Performance Toolkit\mozprofilerprobe.mof`"" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/worker_types/gecko-t-win10-64-cu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-cu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/ami-base-name diff --git a/worker_types/gecko-t-win10-64/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata similarity index 98% rename from worker_types/gecko-t-win10-64/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata index e1dad0ce..e0d50019 100644 --- a/worker_types/gecko-t-win10-64/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-cu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-cu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-t-win10-64-gpu-b/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-gpu-b/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/ami-base-name diff --git a/worker_types/gecko-t-win10-64-gpu-b/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata similarity index 98% rename from worker_types/gecko-t-win10-64-gpu-b/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata index 34699249..58af0f7e 100644 --- a/worker_types/gecko-t-win10-64-gpu-b/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu-b/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-gpu-b.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -367,7 +372,7 @@ $client.DownloadFile("http://hg.mozilla.org/mozilla-central/raw-file/360ab7771e2 Start-Process "mofcomp" -ArgumentList "`"C:\Program Files (x86)\Windows Kits\10\Windows Performance Toolkit\mozprofilerprobe.mof`"" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/worker_types/gecko-t-win10-64-gpu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-gpu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/ami-base-name diff --git a/worker_types/gecko-t-win10-64-gpu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata similarity index 98% rename from worker_types/gecko-t-win10-64-gpu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata index b2468767..7a79e152 100644 --- a/worker_types/gecko-t-win10-64-gpu/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-gpu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64-gpu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-t-win10-64-hw/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64-hw/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64-hw/ami-base-name diff --git a/worker_types/gecko-t-win10-64/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name similarity index 100% rename from worker_types/gecko-t-win10-64/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/ami-base-name diff --git a/worker_types/gecko-t-win10-64-cu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata similarity index 98% rename from worker_types/gecko-t-win10-64-cu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata index e1dad0ce..50412eb4 100644 --- a/worker_types/gecko-t-win10-64-cu/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win10-64/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win10-64.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-t-win7-32-beta/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-beta/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/ami-base-name diff --git a/worker_types/gecko-t-win7-32-gpu-b/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata similarity index 98% rename from worker_types/gecko-t-win7-32-gpu-b/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata index c3557c08..d03627ec 100644 --- a/worker_types/gecko-t-win7-32-gpu-b/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-beta/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-beta.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -354,7 +359,7 @@ $client.DownloadFile("http://download.microsoft.com/download/4/c/b/4cbd5757-0dd4 Start-Process "C:\binaries\vc_redist.x64.exe" -ArgumentList "/install /passive /norestart /log C:\log\vcredist_vs2015_x64-install.log" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/worker_types/gecko-t-win7-32-cu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-cu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/ami-base-name diff --git a/worker_types/gecko-t-win7-32-cu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata similarity index 98% rename from worker_types/gecko-t-win7-32-cu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata index c3557c08..8d5680ea 100644 --- a/worker_types/gecko-t-win7-32-cu/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-cu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-cu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-t-win7-32-gpu-b/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-gpu-b/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/ami-base-name diff --git a/worker_types/gecko-t-win7-32-gpu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata similarity index 98% rename from worker_types/gecko-t-win7-32-gpu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata index c3557c08..82461557 100644 --- a/worker_types/gecko-t-win7-32-gpu/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu-b/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-gpu-b.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -354,7 +359,7 @@ $client.DownloadFile("http://download.microsoft.com/download/4/c/b/4cbd5757-0dd4 Start-Process "C:\binaries\vc_redist.x64.exe" -ArgumentList "/install /passive /norestart /log C:\log\vcredist_vs2015_x64-install.log" -Wait -NoNewWindow # ProgramData_Mozilla_AccessRights: https://bugzilla.mozilla.org/show_bug.cgi?id=1494048 -Start-Process "icacls.exe" -ArgumentList "c:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow +Start-Process "icacls.exe" -ArgumentList "C:\ProgramData\Mozilla /grant Everyone:(OI)(CI)F" -Wait -NoNewWindow # now shutdown, in preparation for creating an image # Stop-Computer isn't working, also not when specifying -AsJob, so reverting to using `shutdown` command instead diff --git a/worker_types/gecko-t-win7-32-gpu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-gpu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/ami-base-name diff --git a/worker_types/gecko-t-win7-32-beta/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata similarity index 98% rename from worker_types/gecko-t-win7-32-beta/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata index c3557c08..bdd8817d 100644 --- a/worker_types/gecko-t-win7-32-beta/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-gpu/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32-gpu.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/gecko-t-win7-32-hw/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32-hw/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32-hw/ami-base-name diff --git a/worker_types/gecko-t-win7-32/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name similarity index 100% rename from worker_types/gecko-t-win7-32/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/ami-base-name diff --git a/worker_types/gecko-t-win7-32/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata similarity index 98% rename from worker_types/gecko-t-win7-32/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata index cb26e1c0..aa51df70 100644 --- a/worker_types/gecko-t-win7-32/userdata +++ b/mozilla/worker-type-host-definitions/aws-provisioner-v1/gecko-t-win7-32/userdata @@ -1,5 +1,10 @@ +# This powershell is AUTO-GENERATED and is an APPROXIMATION only(!) to the installation +# steps that actually occur on this worker type. It is generated by +# https://raw.githubusercontent.com/taskcluster/generic-worker/master/mozilla/OpenCloudConfig/generate_occ_userdata.sh +# from https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Manifest/gecko-t-win7-32.json + # use TLS 1.2 (see bug 1443595) [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 diff --git a/worker_types/nss-win2012r2-new/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name similarity index 100% rename from worker_types/nss-win2012r2-new/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/ami-base-name diff --git a/worker_types/nss-win2012r2-new/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata similarity index 100% rename from worker_types/nss-win2012r2-new/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2-new/userdata diff --git a/worker_types/nss-win2012r2/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name similarity index 100% rename from worker_types/nss-win2012r2/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/ami-base-name diff --git a/worker_types/nss-win2012r2/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/userdata similarity index 100% rename from worker_types/nss-win2012r2/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/nss-win2012r2/userdata diff --git a/worker_types/win2012r2-cu/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name similarity index 100% rename from worker_types/win2012r2-cu/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/ami-base-name diff --git a/worker_types/win2012r2-cu/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/userdata similarity index 100% rename from worker_types/win2012r2-cu/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2-cu/userdata diff --git a/worker_types/win2012r2/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/ami-base-name similarity index 100% rename from worker_types/win2012r2/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/ami-base-name diff --git a/worker_types/win2012r2/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/userdata similarity index 100% rename from worker_types/win2012r2/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2012r2/userdata diff --git a/worker_types/win2016/ami-base-name b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/ami-base-name similarity index 100% rename from worker_types/win2016/ami-base-name rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/ami-base-name diff --git a/worker_types/win2016/userdata b/mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/userdata similarity index 100% rename from worker_types/win2016/userdata rename to mozilla/worker-type-host-definitions/aws-provisioner-v1/win2016/userdata diff --git a/publish-payload-schema.sh b/publish-payload-schema.sh deleted file mode 100755 index e05f7f53..00000000 --- a/publish-payload-schema.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -e - -cd "$(dirname "${0}")" -go get github.com/taskcluster/generic-worker/yamltojson -jsonFile="$(mktemp -t jsonFile.XXXXXX)" -cat windows.yml | yamltojson > "${jsonFile}" -aws s3 cp "${jsonFile}" s3://schemas.taskcluster.net/generic-worker/v1/payload.json -rm "${jsonFile}" -curl https://schemas.taskcluster.net/generic-worker/v1/payload.json -echo -echo "Schema updated - check out https://docs.taskcluster.net/reference/workers/generic-worker/payload" diff --git a/all-unix-style.yml b/schemas/all-unix-style.yml similarity index 97% rename from all-unix-style.yml rename to schemas/all-unix-style.yml index 9bbad9a9..d862b585 100644 --- a/all-unix-style.yml +++ b/schemas/all-unix-style.yml @@ -1,4 +1,11 @@ --- + +# Bug 1456357 - Do **NOT** consider adding this line back in, it causes +# github.com/xeipuuv/gojsonschema to attempt to download the schema which could +# be out-of-sync with this version which is burned into the release. +# +# ### DO NOT ADD THIS!! -> id: https://schemas.taskcluster.net/generic-worker/v1/payload.json# + "$schema": http://json-schema.org/draft-04/schema# title: Generic worker payload description: |- diff --git a/windows.yml b/schemas/windows.yml similarity index 97% rename from windows.yml rename to schemas/windows.yml index dbea5bae..d3f806e0 100644 --- a/windows.yml +++ b/schemas/windows.yml @@ -1,4 +1,11 @@ --- + +# Bug 1456357 - Do **NOT** consider adding this line back in, it causes +# github.com/xeipuuv/gojsonschema to attempt to download the schema which could +# be out-of-sync with this version which is burned into the release. +# +# ### DO NOT ADD THIS!! -> id: https://schemas.taskcluster.net/generic-worker/v1/payload.json# + "$schema": http://json-schema.org/draft-04/schema# title: Generic worker payload description: |- diff --git a/build.cmd b/scripts/build.cmd similarity index 73% rename from build.cmd rename to scripts/build.cmd index 5e449666..81a86785 100644 --- a/build.cmd +++ b/scripts/build.cmd @@ -2,14 +2,14 @@ @echo on -:: cd to dir containing this script -pushd %~dp0 +:: cd to parent dir of the dir of this script +pushd %~dp0\.. go get github.com/taskcluster/livelog github.com/gordonklaus/ineffassign || exit /b %ERRORLEVEL% -cd gw-codegen +cd cmd\gw-codegen go get -v || exit /b %ERRORLEVEL% -cd .. -go generate || exit /b %ERRORLEVEL% +cd ..\.. +go generate ./... || exit /b %ERRORLEVEL% go get -v -t ./... || exit /b %ERRORLEVEL% :: this counts the number of lines returned by git status @@ -23,5 +23,5 @@ git rev-parse HEAD > revision.txt set /p REVISION=< revision.txt del revision.txt set GORACE=history_size=7 -go test -ldflags "-X github.com/taskcluster/generic-worker.revision=%REVISION%" ./... || exit /b %ERRORLEVEL% +go test -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=%REVISION%" ./... || exit /b %ERRORLEVEL% ineffassign . || exit /b %ERRORLEVEL% diff --git a/build.sh b/scripts/build.sh similarity index 86% rename from build.sh rename to scripts/build.sh index 36ef5ac1..8e99837c 100755 --- a/build.sh +++ b/scripts/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -cd "$(dirname "${0}")" +cd "$(dirname "${0}")"/.. # Support go 1 release 1.9 or higher. Let's not move this to 1.10 until # https://bugzil.la/1441889 is resolved, and travis-ci.org works correctly with @@ -42,7 +42,7 @@ done echo "${OUTPUT_ALL_PLATFORMS}" echo "${OUTPUT_TEST}" -go get github.com/taskcluster/generic-worker/gw-codegen +go get github.com/taskcluster/generic-worker/cmd/gw-codegen export PATH="$(go env GOPATH)/bin:${PATH}" go generate ./... @@ -51,14 +51,16 @@ function install { GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v ./... # GOOS="${1}" GOARCH="${2}" go vet ./... # note, this just builds tests, it doesn't run them! - GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker - GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/livelog + go list ./... | while read PACKAGE; do + GOOS="${1}" GOARCH="${2}" CGO_ENABLED=0 go test -c "${PACKAGE}" + done else CGO_ENABLED=0 go get -ldflags "-X main.revision=$(git rev-parse HEAD)" -v ./... go vet ./... # note, this just builds tests, it doesn't run them! - CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker - CGO_ENABLED=0 go test -c github.com/taskcluster/generic-worker/livelog + go list ./... | while read PACKAGE; do + CGO_ENABLED=0 go test -c "${PACKAGE}" + done fi } @@ -102,7 +104,7 @@ fi ##install plan9 amd64 #install solaris amd64 -find "${GOPATH}/bin" -name 'generic-worker*' +find "$(go env GOPATH)/bin" -name 'generic-worker*' CGO_ENABLED=0 go get github.com/taskcluster/livelog # capital X here ... we only want to delete things that are ignored! @@ -110,7 +112,7 @@ git clean -fdX if $TEST; then go get github.com/taskcluster/taskcluster-proxy - CGO_ENABLED=1 GORACE="history_size=7" go test -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -race -timeout 1h ./... + CGO_ENABLED=1 GORACE="history_size=7" go test -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=$(git rev-parse HEAD)" -race -timeout 1h ./... fi go vet ./... go get github.com/golang/lint/golint diff --git a/clean-builds.sh b/scripts/clean-builds.sh similarity index 60% rename from clean-builds.sh rename to scripts/clean-builds.sh index 83fb824f..d1d78660 100755 --- a/clean-builds.sh +++ b/scripts/clean-builds.sh @@ -33,19 +33,19 @@ export GOARCH="${MY_GOARCH}" go version go env export GOPATH="$(mktemp -d -t generic-worker.XXXXXXXXXX)" -export PATH="${GOPATH}/bin:${PATH}" -rm -rf "${GOPATH}" +export PATH="$(go env GOPATH)/bin:${PATH}" +rm -rf "$(go env GOPATH)" go get -d -v 'github.com/taskcluster/generic-worker' -"${GOPATH}/src/github.com/taskcluster/generic-worker/build.sh" -a -t +"$(go env GOPATH)/src/github.com/taskcluster/generic-worker/build.sh" -a -t -mv "${GOPATH}/bin/darwin_386/generic-worker" target/generic-worker-darwin-386 -mv "${GOPATH}/bin/generic-worker" target/generic-worker-darwin-amd64 -mv "${GOPATH}/bin/linux_386/generic-worker" target/generic-worker-linux-386 -mv "${GOPATH}/bin/linux_amd64/generic-worker" target/generic-worker-linux-amd64 -mv "${GOPATH}/bin/linux_arm/generic-worker" target/generic-worker-linux-arm -mv "${GOPATH}/bin/linux_arm64/generic-worker" target/generic-worker-linux-arm64 -mv "${GOPATH}/bin/windows_386/generic-worker.exe" target/generic-worker-windows-386 -mv "${GOPATH}/bin/windows_amd64/generic-worker.exe" target/generic-worker-windows-amd64 +mv "$(go env GOPATH)/bin/darwin_386/generic-worker" target/generic-worker-darwin-386 +mv "$(go env GOPATH)/bin/generic-worker" target/generic-worker-darwin-amd64 +mv "$(go env GOPATH)/bin/linux_386/generic-worker" target/generic-worker-linux-386 +mv "$(go env GOPATH)/bin/linux_amd64/generic-worker" target/generic-worker-linux-amd64 +mv "$(go env GOPATH)/bin/linux_arm/generic-worker" target/generic-worker-linux-arm +mv "$(go env GOPATH)/bin/linux_arm64/generic-worker" target/generic-worker-linux-arm64 +mv "$(go env GOPATH)/bin/windows_386/generic-worker.exe" target/generic-worker-windows-386 +mv "$(go env GOPATH)/bin/windows_amd64/generic-worker.exe" target/generic-worker-windows-amd64 echo echo "Release binaries available in directory '$(pwd)/target'" @@ -56,5 +56,5 @@ echo echo rm -rf "${GO_DOWNLOAD_DIR}" -rm -rf "${GOPATH}" +rm -rf "$(go env GOPATH)" rm target/go.tar.gz diff --git a/gotestcover.sh b/scripts/gotestcover.sh similarity index 77% rename from gotestcover.sh rename to scripts/gotestcover.sh index 995172bd..d35c539e 100755 --- a/gotestcover.sh +++ b/scripts/gotestcover.sh @@ -5,18 +5,17 @@ if [ -z "${REPORT}" ]; then echo "Specify a report, e.g. '${0}' myreport.txt" >&2 exit 64 fi -cd "$(dirname "${0}")" TEMP_SINGLE_REPORT="$(mktemp -t coverage.tmp.XXXXXXXXXX)" echo "mode: atomic" > "${REPORT}" HEAD_REV="$(git rev-parse HEAD)" # Dump package list to file rather than pipe, to avoid exit inside loop not # causing outer shell to exit due to running in a subshell. PACKAGE_LIST="$(mktemp -t package-list.tmp.XXXXXXXXXX)" -go list ./... > "${PACKAGE_LIST}" +go list github.com/taskcluster/generic-worker/... > "${PACKAGE_LIST}" while read package do - CGO_ENABLED=1 go test -ldflags "-X github.com/taskcluster/generic-worker.revision=${HEAD_REV}" -race -timeout 1h -covermode=atomic "-coverprofile=${TEMP_SINGLE_REPORT}" "${package}" + CGO_ENABLED=1 go test -ldflags "-X github.com/taskcluster/generic-worker/cmd/generic-worker.revision=${HEAD_REV}" -race -timeout 1h -covermode=atomic "-coverprofile=${TEMP_SINGLE_REPORT}" "${package}" if [ -f "${TEMP_SINGLE_REPORT}" ]; then sed 1d "${TEMP_SINGLE_REPORT}" >> "${REPORT}" rm "${TEMP_SINGLE_REPORT}" diff --git a/scripts/publish-payload-schema.sh b/scripts/publish-payload-schema.sh new file mode 100755 index 00000000..0ff8a42b --- /dev/null +++ b/scripts/publish-payload-schema.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e + +cd "$(dirname "${0}")" +go get github.com/taskcluster/generic-worker/cmd/yamltojson + +windows_payload_file="$(mktemp -t windows-payload.json.XXXXXX)" +all_unix_style_payload_file="$(mktemp -t all-unix-style-payload.json.XXXXXX)" +cat ../schemas/windows.yml | "$(go env GOPATH)/bin/yamltojson" > "${windows_payload_file}" +cat ../schemas/all-unix-style.yml | "$(go env GOPATH)/bin/yamltojson" > "${all_unix_style_payload_file}" +aws s3 cp "${windows_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/windows.json +aws s3 cp "${windows_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/payload.json +aws s3 cp "${all_unix_style_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/linux.json +aws s3 cp "${all_unix_style_payload_file}" s3://schemas.taskcluster.net/generic-worker/v1/macos.json +rm "${windows_payload_file}" +rm "${all_unix_style_payload_file}" +curl https://schemas.taskcluster.net/generic-worker/v1/payload.json +curl https://schemas.taskcluster.net/generic-worker/v1/windows.json +curl https://schemas.taskcluster.net/generic-worker/v1/linux.json +curl https://schemas.taskcluster.net/generic-worker/v1/macos.json +echo +echo "Schema updated - check out https://docs.taskcluster.net/reference/workers/generic-worker/payload" diff --git a/refresh_readme.sh b/scripts/refresh_readme.sh similarity index 88% rename from refresh_readme.sh rename to scripts/refresh_readme.sh index 5b7ae6a7..35236601 100755 --- a/refresh_readme.sh +++ b/scripts/refresh_readme.sh @@ -1,10 +1,10 @@ #!/bin/bash -eu -cd "$(dirname "${0}")" +cd "$(dirname "${0}")/.." VALID_FORMAT='[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*' TEMP_GW_HELP="$(mktemp -t generic-worker-help-text.XXXXXXXXXX)" TEMP_GW_README="$(mktemp -t generic-worker-readme.XXXXXXXXXX)" TEMP_GW_BINARY="$(mktemp -t generic-worker.XXXXXXXXXX)" -go build -o "${TEMP_GW_BINARY}" +go build -o "${TEMP_GW_BINARY}" ./cmd/generic-worker "${TEMP_GW_BINARY}" --help > "${TEMP_GW_HELP}" echo '```' >> "${TEMP_GW_HELP}" sed -e " diff --git a/release.sh b/scripts/release.sh similarity index 95% rename from release.sh rename to scripts/release.sh index fbbe2c62..b17c144c 100755 --- a/release.sh +++ b/scripts/release.sh @@ -11,7 +11,7 @@ set -e OFFICIAL_GIT_REPO='git@github.com:taskcluster/generic-worker' # step into directory containing this script -cd "$(dirname "${0}")" +cd "$(dirname "${0}")/.." NEW_VERSION="${1}" @@ -102,12 +102,12 @@ if ! echo "${NEW_VERSION}" | grep -q "alpha"; then fi fi -inline_sed README.md "s/.\/release.sh ${OLD_VERSION//./\\.}/.\/release.sh ${NEW_VERSION}/" +inline_sed README.md "s/scripts\/release.sh ${OLD_VERSION//./\\.}/scripts\/release.sh ${NEW_VERSION}/" inline_sed main.go 's/\(version *= *\)"'"${OLD_VERSION//./\\.}"'"$/\1"'"${NEW_VERSION}"'"/' find . -name userdata | while read file; do inline_sed "${file}" "s:taskcluster/generic-worker/releases/download/v${OLD_VERSION//./\\.}/:taskcluster/generic-worker/releases/download/v${NEW_VERSION}/:g" done -./refresh_readme.sh +"$(dirname "${0}")/refresh_readme.sh" git add README.md git commit -m "Version bump from ${OLD_VERSION} to ${NEW_VERSION}" git tag -s "v${NEW_VERSION}" -m "Making release ${NEW_VERSION}" @@ -122,7 +122,7 @@ if ! echo "${NEW_VERSION}" | grep -q "alpha"; then echo echo 'Will you also be deploying this release to production? If so, please run:' echo - echo ' ***** ./publish-payload-schema.sh *****' + echo ' ***** scripts/publish-payload-schema.sh *****' echo echo 'This will update:' echo diff --git a/tutorial-audio.sh b/scripts/tutorial-audio.sh similarity index 100% rename from tutorial-audio.sh rename to scripts/tutorial-audio.sh diff --git a/worker_types/cleanup_ec2.sh b/worker_types/cleanup_ec2.sh deleted file mode 100755 index 68f7363c..00000000 --- a/worker_types/cleanup_ec2.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -cd "$(dirname "${0}")" - -go run all-worker-types/main.go -ALL_REFERENCED_AMIS="$(cat worker_type_definitions/* | sed -n 's/^[[:space:]]*"ImageId": "//p' | sed -n 's/".*//p' | sort -u)" -rm -r worker_type_definitions -echo "All referenced amis: ${ALL_REFERENCED_AMIS}" diff --git a/worker_types/generate_occ_userdata.sh b/worker_types/generate_occ_userdata.sh deleted file mode 100755 index c69d7952..00000000 --- a/worker_types/generate_occ_userdata.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -e -cd "$(dirname "${0}")" - -echo "Removing..." -rm -v gecko-*/userdata - -echo "Generating..." -curl -L 'https://github.com/mozilla-releng/OpenCloudConfig/tree/master/userdata/Manifest' 2>/dev/null | sed -n 's/.*\(gecko[^.]*\)\.json.*/\1/p' | sort -u | xargs -n 1 -P 32 ./transform.sh diff --git a/worker_types/set_occ_ami_base_names.sh b/worker_types/set_occ_ami_base_names.sh deleted file mode 100755 index 38b5a183..00000000 --- a/worker_types/set_occ_ami_base_names.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -cd "$(dirname "${0}")" -aws_region=us-west-2 - -curl -L 'https://github.com/mozilla-releng/OpenCloudConfig/tree/master/userdata/Manifest' 2>/dev/null | sed -n 's/.*\(gecko[^.]*\)\.json.*/\1/p' | sort -u | while read tc_worker_type -do - case "${tc_worker_type}" in - gecko-t-win7-32-gpu*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win7-32-base-20170905'} - aws_instance_type=${aws_instance_type:='g2.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko test worker for Windows 7 32 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=root - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":30,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdc","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-t-win7-32*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win7-32-base-20170905'} - aws_instance_type=${aws_instance_type:='c4.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko test worker for Windows 7 32 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=root - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":30,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdc","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-t-win10-64-gpu*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win10-64-gpu-base-20170921'} - aws_instance_type=${aws_instance_type:='g2.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko tester for Windows 10 64 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=Administrator - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-t-win10-64*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-t-win10-64-base-20170905'} - aws_instance_type=${aws_instance_type:='c4.2xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko tester for Windows 10 64 bit; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=Administrator - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-east-2' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - gecko-[123]-b-win2012*) - aws_base_ami_search_term=${aws_base_ami_search_term:='gecko-b-win2012-base-*'} - aws_instance_type=${aws_instance_type:='c4.4xlarge'} - aws_base_ami_id="$(aws ec2 describe-images --region ${aws_region} --owners self --filters "Name=state,Values=available" "Name=name,Values=${aws_base_ami_search_term}" --query 'Images[*].{A:CreationDate,B:ImageId}' --output text | sort -u | tail -1 | cut -f2)" - ami_description="Gecko builder for Windows; TaskCluster worker type: ${tc_worker_type}, OCC version ${aws_client_token}, https://github.com/mozilla-releng/OpenCloudConfig/tree/${GITHUB_HEAD_SHA}"} - gw_tasks_dir='Z:\' - root_username=Administrator - worker_username=GenericWorker - aws_copy_regions=('us-east-1' 'us-west-1' 'eu-central-1') - block_device_mappings='[{"DeviceName":"/dev/sda1","Ebs":{"VolumeType":"gp2","VolumeSize":40,"DeleteOnTermination":true}},{"DeviceName":"/dev/sdb","Ebs":{"VolumeType":"gp2","VolumeSize":120,"DeleteOnTermination":true}}]' - ;; - *) - echo "ERROR: unknown worker type: '${tc_worker_type}'" - exit 67 - ;; - esac - echo "${aws_base_ami_search_term}" > "${tc_worker_type}/ami-base-name" -done diff --git a/worker_types/transform.sh b/worker_types/transform.sh deleted file mode 100755 index c420d905..00000000 --- a/worker_types/transform.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -e -cd "$(dirname "${0}")" - -echo "${1}" -mkdir -p "${1}" -go run transform-occ/main.go "${1}" > "${1}/userdata" || rm "${1}/userdata"