Bug 2019411 - Move update_verify and beetmover_transform from firefox-main to mozilla-taskgraph

src/mozilla_taskgraph/transforms/beetmover_apt.py

@@ -0,0 +1,124 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from itertools import islice
+
+from taskgraph import MAX_DEPENDENCIES
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.dependencies import get_primary_dependency
+
+from gecko_taskgraph.util.platforms import architecture
+from gecko_taskgraph.util.scriptworker import (
+    generate_artifact_registry_gcs_sources,
+    get_beetmover_apt_repo_scope,
+    get_beetmover_repo_action_scope,
+)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def beetmover_apt(config, tasks):
+    if config.params["project"].startswith("comm"):
+        nightly_product = "thunderbird"
+    else:
+        nightly_product = "firefox"
+
+    product = (
+        nightly_product
+        if config.params["release_type"] == "nightly"
+        else config.params["release_product"]
+    )
+    filtered_tasks = filter_beetmover_apt_tasks(config, tasks, product)
+    # There are too many beetmover-repackage dependencies for a single task
+    # and we hit the taskgraph dependencies limit.
+    # To work around this limitation, we chunk the would be task
+    # into tasks dependendent on, at most, half of MAX_DEPENDENCIES.
+    batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2)
+    for index, batch in enumerate(batches):
+        dependencies = {}
+        gcs_sources = []
+        for task in batch:
+            dep = get_primary_dependency(config, task)
+            assert dep
+
+            dependencies[dep.label] = dep.label
+            gcs_sources.extend(generate_artifact_registry_gcs_sources(dep))
+        description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages"
+        platform = f"{product}-release/opt"
+        treeherder = {
+            "platform": platform,
+            "tier": 1,
+            "kind": "other",
+            "symbol": f"BM-apt(batch-{index + 1})",
+        }
+        apt_repo_scope = get_beetmover_apt_repo_scope(config)
+        repo_action_scope = get_beetmover_repo_action_scope(config)
+        attributes = {
+            "required_signoffs": ["mar-signing"],
+            "shippable": True,
+            "shipping_product": product,
+        }
+        task = {
+            "label": f"{config.kind}-{index + 1}-{platform}",
+            "description": description,
+            "worker-type": "beetmover",
+            "treeherder": treeherder,
+            "scopes": [apt_repo_scope, repo_action_scope],
+            "attributes": attributes,
+            "shipping-phase": "ship",
+            "shipping-product": product,
+            "dependencies": dependencies,
+        }
+        worker = {
+            "implementation": "beetmover-import-from-gcs-to-artifact-registry",
+            "product": product,
+            "gcs-sources": gcs_sources,
+        }
+        task["worker"] = worker
+        yield task
+
+
+def batched(iterable, n):
+    "Batch data into tuples of length n. The last batch may be shorter."
+    # batched('ABCDEFG', 3) --> ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    it = iter(iterable)
+    batch = tuple(islice(it, n))
+    while batch:
+        yield batch
+        batch = tuple(islice(it, n))
+
+
+def filter_beetmover_apt_tasks(config, tasks, product):
+    for task in tasks:
+        task["primary-dependency"] = get_primary_dependency(config, task)
+        if filter_beetmover_apt_task(task, product):
+            yield task
+
+
+def filter_beetmover_apt_task(task, product):
+    # We only create beetmover-apt tasks for l10n beetmover-repackage tasks that
+    # beetmove langpack .deb packages. The langpack .deb packages support all
+    # architectures, so we generate them only on x86_64 tasks.
+    return (
+        is_x86_64_l10n_task(task) or is_not_l10n_task(task)
+    ) and is_task_for_product(task, product)
+
+
+def is_x86_64_l10n_task(task):
+    dep = task["primary-dependency"]
+    locale = dep.attributes.get("locale")
+    return locale and architecture(dep.attributes["build_platform"]) == "x86_64"
+
+
+def is_not_l10n_task(task):
+    dep = task["primary-dependency"]
+    locale = dep.attributes.get("locale")
+    return not locale
+
+
+def is_task_for_product(task, product):
+    dep = task["primary-dependency"]
+    return dep.attributes.get("shipping_product") == product

src/mozilla_taskgraph/transforms/update_verify.py

@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.copy import deepcopy
+from taskgraph.util.treeherder import add_suffix, inherit_treeherder_from_dep
+
+from gecko_taskgraph.util.attributes import task_name
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_command(config, tasks):
+    config_tasks = {}
+    for dep in config.kind_dependencies_tasks.values():
+        if (
+            "update-verify-config" in dep.kind
+            or "update-verify-next-config" in dep.kind
+        ):
+            config_tasks[task_name(dep)] = dep
+
+    for task in tasks:
+        config_task = config_tasks[task["name"]]
+        total_chunks = task["extra"]["chunks"]
+        task["worker"].setdefault("env", {})["CHANNEL"] = config_task.task["extra"][
+            "channel"
+        ]
+        task.setdefault("fetches", {})[config_task.label] = [
+            "update-verify.cfg",
+        ]
+        task["treeherder"] = inherit_treeherder_from_dep(task, config_task)
+
+        for this_chunk in range(1, total_chunks + 1):
+            chunked = deepcopy(task)
+            chunked["treeherder"]["symbol"] = add_suffix(
+                chunked["treeherder"]["symbol"], this_chunk
+            )
+            chunked["label"] = "release-update-verify-{}-{}/{}".format(
+                chunked["name"], this_chunk, total_chunks
+            )
+            if not chunked["worker"].get("env"):
+                chunked["worker"]["env"] = {}
+
+            command = [
+                "tools/update-verify/scripts/chunked-verify.sh",
+                f"--total-chunks={total_chunks} --this-chunk={this_chunk}",
+            ]
+
+            # Add upstream tools to the path
+            if "linux64-libdmg" in chunked.get("fetches", {}).get("toolchain", []):
+                path_override = "export PATH=$PATH:$MOZ_FETCHES_DIR/dmg &&"
+                command.insert(0, path_override)
+
+            chunked["run"] = {
+                "using": "run-task",
+                "cwd": "{checkout}",
+                "command": " ".join(command),
+                "sparse-profile": "update-verify",
+            }
+
+            yield chunked