diff --git a/src/mozilla_taskgraph/transforms/beetmover_apt.py b/src/mozilla_taskgraph/transforms/beetmover_apt.py new file mode 100644 index 0000000..ee2b17d --- /dev/null +++ b/src/mozilla_taskgraph/transforms/beetmover_apt.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from itertools import islice + +from taskgraph import MAX_DEPENDENCIES +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.dependencies import get_primary_dependency + +from gecko_taskgraph.util.platforms import architecture +from gecko_taskgraph.util.scriptworker import ( + generate_artifact_registry_gcs_sources, + get_beetmover_apt_repo_scope, + get_beetmover_repo_action_scope, +) + +transforms = TransformSequence() + + +@transforms.add +def beetmover_apt(config, tasks): + if config.params["project"].startswith("comm"): + nightly_product = "thunderbird" + else: + nightly_product = "firefox" + + product = ( + nightly_product + if config.params["release_type"] == "nightly" + else config.params["release_product"] + ) + filtered_tasks = filter_beetmover_apt_tasks(config, tasks, product) + # There are too many beetmover-repackage dependencies for a single task + # and we hit the taskgraph dependencies limit. + # To work around this limitation, we chunk the would be task + # into tasks dependendent on, at most, half of MAX_DEPENDENCIES. + batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2) + for index, batch in enumerate(batches): + dependencies = {} + gcs_sources = [] + for task in batch: + dep = get_primary_dependency(config, task) + assert dep + + dependencies[dep.label] = dep.label + gcs_sources.extend(generate_artifact_registry_gcs_sources(dep)) + description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages" + platform = f"{product}-release/opt" + treeherder = { + "platform": platform, + "tier": 1, + "kind": "other", + "symbol": f"BM-apt(batch-{index + 1})", + } + apt_repo_scope = get_beetmover_apt_repo_scope(config) + repo_action_scope = get_beetmover_repo_action_scope(config) + attributes = { + "required_signoffs": ["mar-signing"], + "shippable": True, + "shipping_product": product, + } + task = { + "label": f"{config.kind}-{index + 1}-{platform}", + "description": description, + "worker-type": "beetmover", + "treeherder": treeherder, + "scopes": [apt_repo_scope, repo_action_scope], + "attributes": attributes, + "shipping-phase": "ship", + "shipping-product": product, + "dependencies": dependencies, + } + worker = { + "implementation": "beetmover-import-from-gcs-to-artifact-registry", + "product": product, + "gcs-sources": gcs_sources, + } + task["worker"] = worker + yield task + + +def batched(iterable, n): + "Batch data into tuples of length n. The last batch may be shorter." + # batched('ABCDEFG', 3) --> ABC DEF G + if n < 1: + raise ValueError("n must be at least one") + it = iter(iterable) + batch = tuple(islice(it, n)) + while batch: + yield batch + batch = tuple(islice(it, n)) + + +def filter_beetmover_apt_tasks(config, tasks, product): + for task in tasks: + task["primary-dependency"] = get_primary_dependency(config, task) + if filter_beetmover_apt_task(task, product): + yield task + + +def filter_beetmover_apt_task(task, product): + # We only create beetmover-apt tasks for l10n beetmover-repackage tasks that + # beetmove langpack .deb packages. The langpack .deb packages support all + # architectures, so we generate them only on x86_64 tasks. + return ( + is_x86_64_l10n_task(task) or is_not_l10n_task(task) + ) and is_task_for_product(task, product) + + +def is_x86_64_l10n_task(task): + dep = task["primary-dependency"] + locale = dep.attributes.get("locale") + return locale and architecture(dep.attributes["build_platform"]) == "x86_64" + + +def is_not_l10n_task(task): + dep = task["primary-dependency"] + locale = dep.attributes.get("locale") + return not locale + + +def is_task_for_product(task, product): + dep = task["primary-dependency"] + return dep.attributes.get("shipping_product") == product diff --git a/src/mozilla_taskgraph/transforms/update_verify.py b/src/mozilla_taskgraph/transforms/update_verify.py new file mode 100644 index 0000000..e94ea87 --- /dev/null +++ b/src/mozilla_taskgraph/transforms/update_verify.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the beetmover task into an actual task description. +""" + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.copy import deepcopy +from taskgraph.util.treeherder import add_suffix, inherit_treeherder_from_dep + +from gecko_taskgraph.util.attributes import task_name + +transforms = TransformSequence() + + +@transforms.add +def add_command(config, tasks): + config_tasks = {} + for dep in config.kind_dependencies_tasks.values(): + if ( + "update-verify-config" in dep.kind + or "update-verify-next-config" in dep.kind + ): + config_tasks[task_name(dep)] = dep + + for task in tasks: + config_task = config_tasks[task["name"]] + total_chunks = task["extra"]["chunks"] + task["worker"].setdefault("env", {})["CHANNEL"] = config_task.task["extra"][ + "channel" + ] + task.setdefault("fetches", {})[config_task.label] = [ + "update-verify.cfg", + ] + task["treeherder"] = inherit_treeherder_from_dep(task, config_task) + + for this_chunk in range(1, total_chunks + 1): + chunked = deepcopy(task) + chunked["treeherder"]["symbol"] = add_suffix( + chunked["treeherder"]["symbol"], this_chunk + ) + chunked["label"] = "release-update-verify-{}-{}/{}".format( + chunked["name"], this_chunk, total_chunks + ) + if not chunked["worker"].get("env"): + chunked["worker"]["env"] = {} + + command = [ + "tools/update-verify/scripts/chunked-verify.sh", + f"--total-chunks={total_chunks} --this-chunk={this_chunk}", + ] + + # Add upstream tools to the path + if "linux64-libdmg" in chunked.get("fetches", {}).get("toolchain", []): + path_override = "export PATH=$PATH:$MOZ_FETCHES_DIR/dmg &&" + command.insert(0, path_override) + + chunked["run"] = { + "using": "run-task", + "cwd": "{checkout}", + "command": " ".join(command), + "sparse-profile": "update-verify", + } + + yield chunked