Wip/mgiacomo/integ3142 flex instance 0302 2

cli/src/pcluster/api/models/describe_cluster_instances_response_content.py

@@ -32,7 +32,7 @@ def __init__(self, instances=None, next_token=None):
         """
         self.openapi_types = {"instances": List[ClusterInstance], "next_token": str}
 
-        self.attribute_map = {"instances": "instances", "next_token": "nextToken"}
+        self.attribute_map = {"instances": "instances", "next_token": "nextToken"}  # nosec B105
 
         self._instances = instances
         self._next_token = next_token

cli/src/pcluster/api/models/get_cluster_log_events_response_content.py

@@ -28,7 +28,7 @@ def __init__(self, next_token=None, prev_token=None, events=None):  # noqa: E501
         """
         self.openapi_types = {"next_token": str, "prev_token": str, "events": List[LogEvent]}
 
-        self.attribute_map = {"next_token": "nextToken", "prev_token": "prevToken", "events": "events"}
+        self.attribute_map = {"next_token": "nextToken", "prev_token": "prevToken", "events": "events"}  # nosec B105
 
         self._next_token = next_token
         self._prev_token = prev_token

cli/src/pcluster/api/models/get_cluster_stack_events_response_content.py

@@ -26,7 +26,7 @@ def __init__(self, next_token=None, events=None):  # noqa: E501
         """
         self.openapi_types = {"next_token": str, "events": List[StackEvent]}
 
-        self.attribute_map = {"next_token": "nextToken", "events": "events"}
+        self.attribute_map = {"next_token": "nextToken", "events": "events"}  # nosec B105
 
         self._next_token = next_token
         self._events = events

cli/src/pcluster/api/models/get_image_log_events_response_content.py

@@ -28,7 +28,7 @@ def __init__(self, next_token=None, prev_token=None, events=None):  # noqa: E501
         """
         self.openapi_types = {"next_token": str, "prev_token": str, "events": List[LogEvent]}
 
-        self.attribute_map = {"next_token": "nextToken", "prev_token": "prevToken", "events": "events"}
+        self.attribute_map = {"next_token": "nextToken", "prev_token": "prevToken", "events": "events"}  # nosec B105
 
         self._next_token = next_token
         self._prev_token = prev_token

cli/src/pcluster/api/models/get_image_stack_events_response_content.py

@@ -26,7 +26,7 @@ def __init__(self, next_token=None, events=None):  # noqa: E501
         """
         self.openapi_types = {"next_token": str, "events": List[StackEvent]}
 
-        self.attribute_map = {"next_token": "nextToken", "events": "events"}
+        self.attribute_map = {"next_token": "nextToken", "events": "events"}  # nosec B105
 
         self._next_token = next_token
         self._events = events

cli/src/pcluster/api/models/list_cluster_log_streams_response_content.py

@@ -26,7 +26,7 @@ def __init__(self, log_streams=None, next_token=None):  # noqa: E501
         """
         self.openapi_types = {"log_streams": List[LogStream], "next_token": str}
 
-        self.attribute_map = {"log_streams": "logStreams", "next_token": "nextToken"}
+        self.attribute_map = {"log_streams": "logStreams", "next_token": "nextToken"}  # nosec B105
 
         self._log_streams = log_streams
         self._next_token = next_token

cli/src/pcluster/api/models/list_clusters_response_content.py

@@ -32,7 +32,7 @@ def __init__(self, next_token=None, clusters=None):
         """
         self.openapi_types = {"next_token": str, "clusters": List[ClusterInfoSummary]}
 
-        self.attribute_map = {"next_token": "nextToken", "clusters": "clusters"}
+        self.attribute_map = {"next_token": "nextToken", "clusters": "clusters"}  # nosec B105
 
         self._next_token = next_token
         self._clusters = clusters

cli/src/pcluster/api/models/list_image_log_streams_response_content.py

@@ -26,7 +26,7 @@ def __init__(self, log_streams=None, next_token=None):  # noqa: E501
         """
         self.openapi_types = {"log_streams": List[LogStream], "next_token": str}
 
-        self.attribute_map = {"log_streams": "logStreams", "next_token": "nextToken"}
+        self.attribute_map = {"log_streams": "logStreams", "next_token": "nextToken"}  # nosec B105
 
         self._log_streams = log_streams
         self._next_token = next_token

cli/src/pcluster/api/models/list_images_response_content.py

@@ -32,7 +32,7 @@ def __init__(self, images=None, next_token=None):
         """
         self.openapi_types = {"images": List[ImageInfoSummary], "next_token": str}
 
-        self.attribute_map = {"images": "images", "next_token": "nextToken"}
+        self.attribute_map = {"images": "images", "next_token": "nextToken"}  # nosec B105
 
         self._images = images
         self._next_token = next_token

cli/src/pcluster/api/models/log_stream.py

@@ -52,7 +52,7 @@ def __init__(
             "upload_sequence_token": str,
         }
 
-        self.attribute_map = {
+        self.attribute_map = {  # nosec B105
             "log_stream_arn": "logStreamArn",
             "creation_time": "creationTime",
             "log_stream_name": "logStreamName",

cli/src/pcluster/api/models/stack_event.py

@@ -69,7 +69,7 @@ def __init__(
             "timestamp": datetime,
         }
 
-        self.attribute_map = {
+        self.attribute_map = {  # nosec B105
             "event_id": "eventId",
             "physical_resource_id": "physicalResourceId",
             "resource_status": "resourceStatus",

cli/src/pcluster/cli/commands/configure/easyconfig.py

@@ -177,9 +177,9 @@ def configure(args):  # noqa: C901
     for queue_index in range(number_of_queues):
         while True:
             queue_name = prompt(
-                f"Name of queue {queue_index+1}",
+                f"Name of queue {queue_index + 1}",
                 validator=lambda x: len(NameValidator().execute(x)) == 0,
-                default_value=f"queue{queue_index+1}",
+                default_value=f"queue{queue_index + 1}",
             )
             if queue_name not in queue_names:
                 break
@@ -208,7 +208,7 @@ def configure(args):  # noqa: C901
             if scheduler != "awsbatch":
                 while True:
                     compute_instance_type = prompt(
-                        f"Compute instance type for compute resource {compute_resource_index+1} in {queue_name}",
+                        f"Compute instance type for compute resource {compute_resource_index + 1} in {queue_name}",
                         validator=lambda x: x in AWSApi.instance().ec2.list_instance_types(),
                         default_value=default_instance_type,
                     )

cli/src/pcluster/cli/commands/ssh.py

@@ -79,15 +79,13 @@ class SshCommand(CliCommand):
         "Run ssh command with the cluster username and IP address pre-populated. "
         "Arbitrary arguments are appended to the end of the ssh command."
     )
-    epilog = textwrap.dedent(
-        """Example:
+    epilog = textwrap.dedent("""Example:
 
   pcluster ssh --cluster-name mycluster -i ~/.ssh/id_rsa
 
 Returns an ssh command with the cluster username and IP address pre-populated:
 
-  ssh ec2-user@1.1.1.1 -i ~/.ssh/id_rsa"""
-    )
+  ssh ec2-user@1.1.1.1 -i ~/.ssh/id_rsa""")
 
     def __init__(self, subparsers):
         super().__init__(

cli/src/pcluster/cli/model.py

@@ -20,12 +20,6 @@
 from pcluster.cli.exceptions import APIOperationException
 from pcluster.utils import to_kebab_case, to_snake_case, yaml_load
 
-# For importing package resources
-try:
-    import importlib.resources as pkg_resources  # pylint: disable=ungrouped-imports # nosem
-except ImportError:
-    import importlib_resources as pkg_resources
-
 
 def _param_overrides(operation, param):
     """Provide updates to the model that are specific to the CLI."""
@@ -88,7 +82,7 @@ def _resolve_body(spec, operation):
 
 def package_spec():
     """Load the OpenAPI specification from the package."""
-    with pkg_resources.open_text(openapi, "openapi.yaml") as spec_file:  # pylint: disable=deprecated-method
+    with importlib.resources.open_text(openapi, "openapi.yaml") as spec_file:  # pylint: disable=deprecated-method
         return yaml_load(spec_file.read())
 
 

cli/src/pcluster/constants.py

@@ -354,3 +354,28 @@ class Operation(Enum):
 }
 # Capacity Block states that are considered inactive (cannot check health status)
 CAPACITY_BLOCK_INACTIVE_STATES = ["scheduled", "payment-pending", "assessing", "delayed"]
+
+# Older generation instance types
+EXCLUDED_INSTANCE_TYPE_PREFIXES = (
+    "m1",
+    "m2",
+    "m3",
+    "m4",
+    "t1",
+    "t2",
+    "c1",
+    "c3",
+    "c4",
+    "r3",
+    "r4",
+    "x1",
+    "x1e",
+    "d2",
+    "h1",
+    "i2",
+    "i3",
+    "f1",
+    "g3",
+    "p2",
+    "p3",
+)

cli/tests/pcluster/models/test_s3_bucket.py

@@ -245,15 +245,13 @@ def test_get_resource_url(region, bucket_name, cluster_name, resource_name, expe
                 "B": {"B1": "M"},
             },
             S3FileFormat.YAML,
-            textwrap.dedent(
-                """\
+            textwrap.dedent("""\
                 A:
                   A1: X
                   A2: Y
                 B:
                   B1: M
-                """
-            ),
+                """),
         ),
         (
             {

tests/integration-tests/conftest_resource_bucket.py

@@ -22,7 +22,6 @@
 from pathlib import Path
 
 import boto3
-import pkg_resources
 import pytest
 import urllib3
 from framework.fixture_utils import xdist_session_fixture
@@ -36,7 +35,7 @@
 def install_pc(basepath, pc_version):
     """Install ParallelCluster to a temporary directory"""
     tempdir = Path(basepath) / "python"
-    root = Path(pkg_resources.resource_filename(__name__, "/../.."))
+    root = Path(__file__).parent.parent
     cli_dir = root / "cli"
     try:
         logger.info("installing ParallelCluster packages...")
@@ -105,7 +104,7 @@ def get_resource_map():
 
 @xdist_session_fixture()
 def resource_bucket_shared(request, s3_bucket_factory_shared, lambda_layer_source):
-    root = Path(pkg_resources.resource_filename(__name__, "/../.."))
+    root = Path(__file__).parent.parent
     if request.config.getoption("resource_bucket"):
         return None  # short-circuit this fixture if a resource-bucket is provided
 

tests/integration-tests/framework/tests_configuration/config_renderer.py

@@ -22,6 +22,7 @@
 from utils import InstanceTypesData
 
 from pcluster.constants import (
+    EXCLUDED_INSTANCE_TYPE_PREFIXES,
     SUPPORTED_OSES,
     SUPPORTED_OSES_FOR_SCHEDULER,
     UNSUPPORTED_ARM_OSES_FOR_DCV,
@@ -96,29 +97,6 @@ def _get_instance_type_parameters():  # noqa: C901
         return _get_instance_type_parameters._cache
 
     result = {}
-    excluded_instance_type_prefixes = (
-        "m1",
-        "m2",
-        "m3",
-        "m4",
-        "t1",
-        "t2",
-        "c1",
-        "c3",
-        "c4",
-        "r3",
-        "r4",
-        "x1",
-        "x1e",
-        "d2",
-        "h1",
-        "i2",
-        "i3",
-        "f1",
-        "g3",
-        "p2",
-        "p3",
-    )
 
     for region in ["us-east-1", "us-west-2"]:  # Only populate instance type for big regions
         ec2_client = boto3.client("ec2", region_name=region)
@@ -137,13 +115,13 @@ def _get_instance_type_parameters():  # noqa: C901
                     instance_type_availability_zones[instance_type_name].append(offering["Location"])
                     # Check if instance type ends with '.xlarge'
                     if instance_type_name.endswith(".xlarge") and _is_current_instance_type_generation(
-                        excluded_instance_type_prefixes, offering
+                        EXCLUDED_INSTANCE_TYPE_PREFIXES, offering
                     ):
                         xlarge_instances.add(instance_type_name)
                     # Get a list of only GPU instances of any size available in the region
                     if (
                         instance_type_name.startswith("p") or instance_type_name.startswith("g")
-                    ) and _is_current_instance_type_generation(excluded_instance_type_prefixes, offering):
+                    ) and _is_current_instance_type_generation(EXCLUDED_INSTANCE_TYPE_PREFIXES, offering):
                         all_gpu_instances.add(instance_type_name)
 
             # Get GPU instance details in batches of 100
@@ -161,7 +139,7 @@ def _get_instance_type_parameters():  # noqa: C901
                             if instance_type.get("GpuInfo").get("Gpus")[0].get(
                                 "Count"
                             ) >= 4 and _is_current_instance_type_generation(
-                                excluded_instance_type_prefixes, instance_type
+                                EXCLUDED_INSTANCE_TYPE_PREFIXES, instance_type
                             ):
                                 # Find instance types with 4 or more GPUs. Number of GPUs can change test behavior.
                                 # For example, it takes longer for DCGM health check to diagnose multiple GPUs.

tests/integration-tests/tests/common/utils.py

@@ -15,11 +15,12 @@
 import random
 import string
 import time
+from importlib.metadata import version as get_package_version
 
 import boto3
-import pkg_resources
 from assertpy import assert_that
 from botocore.exceptions import ClientError
+from packaging import version as packaging_version
 from remote_command_executor import RemoteCommandExecutionError, RemoteCommandExecutor
 from retrying import retry
 from time_utils import seconds
@@ -263,14 +264,14 @@ def _assert_ami_is_available(region, ami_id):
 def get_installed_parallelcluster_version():
     """Get the version of the installed aws-parallelcluster package."""
     try:
-        return pkg_resources.get_distribution("aws-parallelcluster").version
+        return get_package_version("aws-parallelcluster")
     except Exception:
         logging.info("aws-parallelcluster is not installed through Python. Getting version from `pcluster version`.")
         return json.loads(run_command(["pcluster", "version"]).stdout.strip())["version"]
 
 
 def get_installed_parallelcluster_base_version():
-    return pkg_resources.packaging.version.parse(get_installed_parallelcluster_version()).base_version
+    return packaging_version.parse(get_installed_parallelcluster_version()).base_version
 
 
 CLASSIC_AWS_DOMAIN = "amazonaws.com"

tests/integration-tests/tests/create/test_create.py

@@ -159,7 +159,7 @@ def test_cluster_creation_with_problematic_preinstall_script(
     assert_lines_in_logs(
         remote_command_executor,
         ["/var/log/cfn-init.log"],
-        [f"Failed to execute OnNodeStart script 1 s3://{ bucket_name }/scripts/{script_name}"],
+        [f"Failed to execute OnNodeStart script 1 s3://{bucket_name}/scripts/{script_name}"],
     )
     logging.info("Verifying error in cloudformation failure reason")
     stack_events = cluster.get_stack_events().get("events")
@@ -173,7 +173,7 @@ def test_cluster_creation_with_problematic_preinstall_script(
     )
 
     assert_that(cfn_failure_reason).contains(expected_cfn_failure_reason)
-    assert_that(cfn_failure_reason).does_not_contain(f"s3://{ bucket_name }/scripts/{script_name}")
+    assert_that(cfn_failure_reason).does_not_contain(f"s3://{bucket_name}/scripts/{script_name}")
 
     logging.info("Verifying failures in describe-clusters output")
     expected_failures = [

tests/integration-tests/tests/custom_resource/conftest.py

@@ -15,7 +15,6 @@
 
 import boto3
 import cfn_tools
-import pkg_resources
 import pytest
 import yaml
 from cfn_stacks_factory import CfnStack
@@ -34,7 +33,7 @@ def cfn_fixture(region):
 
 @pytest.fixture(scope="session", name="resources_dir")
 def resources_dir_fixture():
-    return Path(pkg_resources.resource_filename(__name__, "/../../resources"))
+    return Path(__file__).parent.parent.parent / "resources"
 
 
 @pytest.fixture(scope="session", name="cluster_custom_resource_template")

tests/integration-tests/tests/pcluster_api/test_api.py

@@ -17,7 +17,6 @@
 
 import boto3
 import cfn_tools
-import pkg_resources
 import pytest
 from assertpy import assert_that
 from benchmarks.common.util import get_instance_vcpus
@@ -128,7 +127,7 @@ def _ec2_wait(region, instances, waiter_type):
 
 @pytest.fixture(scope="session", name="resources_dir")
 def resources_dir_fixture():
-    return Path(pkg_resources.resource_filename(__name__, "/../../resources"))
+    return Path(__file__).parent.parent.parent / "resources"
 
 
 @pytest.fixture(scope="session", name="policies_template_path")

tests/integration-tests/tests/schedulers/test_slurm.py

@@ -169,7 +169,7 @@ def test_slurm_ticket_17399(
             "partition": "gpu",
             "test_only": True,
             "other_options": f"--gpus {gpus_per_instance} --nodes 1 --ntasks-per-node 1 "
-            f"--cpus-per-task={cpus_per_instance//gpus_per_instance}",
+            f"--cpus-per-task={cpus_per_instance // gpus_per_instance}",
         }
     )
 
@@ -180,7 +180,7 @@ def test_slurm_ticket_17399(
             "partition": "gpu",
             "test_only": True,
             "other_options": f"--gpus {gpus_per_instance} --nodes 1 --ntasks-per-node 1 "
-            f"--cpus-per-task={cpus_per_instance//gpus_per_instance + 1}",
+            f"--cpus-per-task={cpus_per_instance // gpus_per_instance + 1}",
         }
     )