[CLOUDP-352109] Run MCK E2E tests against OCI published helm chart

.evergreen-release.yml

@@ -214,6 +214,8 @@ buildvariants:
     depends_on:
       - name: release_kubectl_mongodb_plugin
         variant: release_kubectl_mongodb_plugin
+      - name: release_chart_to_oci_registry
+        variant: release_chart_to_oci_registry
     tasks:
       - name: build_test_image
 
@@ -228,6 +230,8 @@ buildvariants:
     depends_on:
       - name: release_kubectl_mongodb_plugin
         variant: release_kubectl_mongodb_plugin
+      - name: release_chart_to_oci_registry
+        variant: release_chart_to_oci_registry
     tasks:
       - name: build_test_image_ibm_power
 
@@ -245,6 +249,8 @@ buildvariants:
     depends_on:
       - name: release_kubectl_mongodb_plugin
         variant: release_kubectl_mongodb_plugin
+      - name: release_chart_to_oci_registry
+        variant: release_chart_to_oci_registry
     tasks:
       - name: build_test_image_ibm_z
 
@@ -258,6 +264,8 @@ buildvariants:
     depends_on:
       - name: release_kubectl_mongodb_plugin
         variant: release_kubectl_mongodb_plugin
+      - name: release_chart_to_oci_registry
+        variant: release_chart_to_oci_registry
     tasks:
       - name: build_test_image_arm
 

.evergreen.yml

@@ -37,6 +37,8 @@ variables:
         variant: init_test_run
       - name: build_init_om_images_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
   - &base_no_om_image_dependency
     depends_on:
@@ -54,6 +56,8 @@ variables:
         variant: init_test_run
       - name: build_init_appdb_images_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
   - &community_dependency
     depends_on:
@@ -67,6 +71,8 @@ variables:
         variant: init_test_run
       - name: build_mco_test_image
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
   - &setup_group
     setup_group_can_fail_task: true
@@ -138,6 +144,8 @@ variables:
         variant: init_test_run
       - name: build_init_om_images_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
   - &base_om7_dependency_with_race
     depends_on:
@@ -155,6 +163,8 @@ variables:
         variant: init_test_run
       - name: build_init_om_images_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
   # Any change to base_om8_dependency should be reflected to its copy in .evergreen-snippets.yml
   - &base_om8_dependency
@@ -173,6 +183,8 @@ variables:
         variant: init_test_run
       - name: build_init_om_images_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
 parameters:
   - key: evergreen_retry
@@ -1305,6 +1317,8 @@ buildvariants:
         variant: init_test_run
       - name: build_init_database_image_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
     tasks:
       - name: e2e_custom_domain_task_group
@@ -1442,6 +1456,8 @@ buildvariants:
         variant: init_test_run
       - name: build_test_image_ibm_power
         variant: init_test_run_ibm_power
+      - name: publish_helm_chart
+        variant: init_test_run
     tasks:
       - name: e2e_smoke_ibm_task_group
 
@@ -1461,6 +1477,8 @@ buildvariants:
         variant: init_test_run
       - name: build_test_image_ibm_power
         variant: init_test_run_ibm_power
+      - name: publish_helm_chart
+        variant: init_test_run
     tasks:
       - name: e2e_smoke_ibm_task_group
 
@@ -1483,6 +1501,8 @@ buildvariants:
         variant: init_test_run
       - name: build_test_image_ibm_z
         variant: init_test_run_ibm_z
+      - name: publish_helm_chart
+        variant: init_test_run
     tasks:
       - name: e2e_smoke_ibm_task_group
 
@@ -1503,6 +1523,8 @@ buildvariants:
         variant: init_test_run
       - name: build_test_image_ibm_z
         variant: init_test_run_ibm_z
+      - name: publish_helm_chart
+        variant: init_test_run
     tasks:
       - name: e2e_smoke_ibm_task_group
 
@@ -1521,6 +1543,8 @@ buildvariants:
         variant: init_test_run
       - name: build_test_image_arm
         variant: init_test_run_arm
+      - name: publish_helm_chart
+        variant: init_test_run
     tasks:
       - name: e2e_smoke_arm_task_group
 
@@ -1537,6 +1561,8 @@ buildvariants:
         variant: init_test_run
       - name: build_test_image_arm
         variant: init_test_run_arm
+      - name: publish_helm_chart
+        variant: init_test_run
     tasks:
       - name: e2e_smoke_arm_task_group
 

build_info.json

@@ -453,12 +453,14 @@
   "helm-charts": {
     "mongodb-kubernetes": {
       "patch": {
+        "version-prefix": "0.0.0+",
         "registry": "268558157000.dkr.ecr.us-east-1.amazonaws.com",
         "region": "us-east-1",
         "repository": "dev/mongodb/helm-charts"
       },
       "staging": {
         "sign": true,
+        "version-prefix": "0.0.0+",
         "registry": "268558157000.dkr.ecr.us-east-1.amazonaws.com",
         "region": "us-east-1",
         "repository": "staging/mongodb/helm-charts"

docker/mongodb-kubernetes-tests/Dockerfile

@@ -16,7 +16,12 @@ RUN apt-get -qq update \
 
 COPY requirements.txt requirements.txt
 
-RUN python3 -m venv /venv && . /venv/bin/activate && pip install --upgrade pip && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 pip install -r requirements.txt
+# install aws, required to run helm registry login while running the tests
+RUN python3 -m venv /venv  \
+    && . /venv/bin/activate  \
+    && pip install --upgrade pip  \
+    && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 pip install -r requirements.txt \
+    && pip install awscli
 
 FROM scratch AS tools_downloader
 
@@ -75,8 +80,9 @@ WORKDIR /tests
 
 # copying the test files after python build, otherwise pip install will be called each time the tests change
 COPY . /tests
-# copying the helm_chart directory as well to support installation of the Operator from the test application
-COPY helm_chart /helm_chart
+# copying the helm_chart/crds directory so that we can install CRDs before installing the operator helm chart to run a test.
+# operator is installed via published OCI helm repo and not the local helm repo.
+COPY helm_chart/crds /helm_chart/crds
 COPY release.json /release.json
 # we use the public directory to automatically test resources samples
 COPY public /mongodb-kubernetes/public

docker/mongodb-kubernetes-tests/kubetester/consts.py

@@ -0,0 +1,7 @@
+OCI_HELM_VERSION = "OCI_HELM_VERSION"
+OCI_HELM_REGISTRY_ENV_VAR_NAME = "OCI_HELM_REGISTRY"
+OCI_HELM_REPOSITORY_ENV_VAR_NAME = "OCI_HELM_REPOSITORY"
+OCI_HELM_REGION_ENV_VAR_NAME = "OCI_HELM_REGION"
+
+LEGACY_OPERATOR_CHART = "mongodb/enterprise-operator"
+MCK_HELM_CHART = "mongodb/mongodb-kubernetes"

docker/mongodb-kubernetes-tests/kubetester/helm.py

@@ -1,15 +1,19 @@
 import glob
-import logging
 import os
 import re
 import subprocess
 import uuid
 from typing import Dict, List, Optional, Tuple
 
+from kubetester.consts import *
 from tests import test_logger
 
 logger = test_logger.get_test_logger(__name__)
 
+# LOCAL_CRDs_DIR is the dir where local helm chart's CRDs are copied in tests image
+LOCAL_CRDs_DIR = "helm_chart/crds"
+OCI_HELM_REGISTRY_ECR = "268558157000.dkr.ecr.us-east-1.amazonaws.com"
+
 
 def helm_template(
     helm_args: Dict,
@@ -25,7 +29,7 @@ def helm_template(
         command_args.append("--show-only")
         command_args.append(templates)
 
-    args = ("helm", "template", *(command_args), helm_chart_path)
+    args = ("helm", "template", *command_args, helm_chart_path)
     logger.info(" ".join(args))
 
     yaml_file_name = "{}.yaml".format(str(uuid.uuid4()))
@@ -145,6 +149,55 @@ def process_run_and_check(args, **kwargs):
         raise
 
 
+def helm_registry_login_to_ecr(helm_registry: str, region: str):
+    logger.info(f"Attempting to log into ECR registry: {helm_registry}, using helm registry login.")
+
+    aws_command = ["aws", "ecr", "get-login-password", "--region", region]
+
+    # as we can see the password is being provided by stdin, that would mean we will have to
+    # pipe the aws_command (it figures out the password) into helm_command.
+    helm_command = ["helm", "registry", "login", "--username", "AWS", "--password-stdin", helm_registry]
+
+    try:
+        logger.info("Starting AWS ECR credential retrieval.")
+        aws_proc = subprocess.Popen(
+            aws_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True  # Treat input/output as text strings
+        )
+
+        logger.info("Starting Helm registry login.")
+        helm_proc = subprocess.Popen(
+            helm_command, stdin=aws_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+        )
+
+        # Close the stdout stream of aws_proc in the parent process
+        # to prevent resource leakage (only needed if you plan to do more processing)
+        aws_proc.stdout.close()
+
+        # Wait for the Helm command (helm_proc) to finish and capture its output
+        helm_stdout, helm_stderr = helm_proc.communicate()
+
+        # Wait for the AWS process to finish as well
+        aws_proc.wait()
+
+        if aws_proc.returncode != 0:
+            _, aws_stderr = aws_proc.communicate()
+            raise Exception(f"aws command to get password failed. Error: {aws_stderr}")
+
+        if helm_proc.returncode == 0:
+            logger.info("Login to helm registry was successful.")
+            logger.info(helm_stdout.strip())
+        else:
+            raise Exception(
+                f"Login to helm registry failed, Exit code: {helm_proc.returncode}, Error: {helm_stderr.strip()}"
+            )
+
+    except FileNotFoundError as e:
+        # This catches errors if 'aws' or 'helm' are not in the PATH
+        raise Exception(f"Command not found. Please ensure '{e.filename}' is installed and in your system's PATH.")
+    except Exception as e:
+        raise Exception(f"An unexpected error occurred: {e}.")
+
+
 def helm_upgrade(
     name: str,
     namespace: str,
@@ -162,7 +215,7 @@ def helm_upgrade(
     chart_dir = helm_chart_path
 
     if apply_crds_first:
-        apply_crds_from_chart(chart_dir)
+        apply_crds_from_chart(LOCAL_CRDs_DIR)
 
     command_args = _create_helm_args(helm_args, helm_options)
     args = [
@@ -183,11 +236,24 @@ def helm_upgrade(
     process_run_and_check(command, check=True, capture_output=True, shell=True)
 
 
-def apply_crds_from_chart(chart_dir: str):
-    crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml"))
+# oci_chart_info returns the respective registry/repo and region information
+# based on the build scenario (dev/staging) tests are being run in. These are
+# read from build_info.json and then set to the tests image as env vars.
+def oci_chart_info():
+    registry = os.environ.get(OCI_HELM_REGISTRY_ENV_VAR_NAME)
+    repository = os.environ.get(OCI_HELM_REPOSITORY_ENV_VAR_NAME)
+    region = os.environ.get(OCI_HELM_REGION_ENV_VAR_NAME)
+
+    logger.info(f"oci chart details in test image is registry {registry}, repo {repository}, region {region}")
+
+    return registry, f"{repository}/mongodb-kubernetes", region
+
+
+def apply_crds_from_chart(crds_dir: str):
+    crd_files = glob.glob(os.path.join(crds_dir, "*.yaml"))
 
     if not crd_files:
-        raise Exception(f"No CRD files found in chart directory: {chart_dir}")
+        raise Exception(f"No CRD files found in chart directory: {crds_dir}")
 
     logger.info(f"Found {len(crd_files)} CRD files to apply:")
 
@@ -230,3 +296,35 @@ def _create_helm_args(helm_args: Dict[str, str], helm_options: Optional[List[str
         command_args.extend(helm_options)
 
     return command_args
+
+
+# helm_chart_path_and_version returns the chart path and version that we would like to install to run the E2E tests.
+# for local tests it returns early with local helm chart dir and for other scenarios it figures out the chart and version
+# based on the caller. In most of the cases we will install chart from OCI registry but for the tests where we would like
+# to install MEKO's specific version or MCK's specific version, we would expect `helm_chart_path` to set already.
+def helm_chart_path_and_version(helm_chart_path: str, operator_version: str) -> tuple[str, str]:
+    # these are imported here to resolve import cycle issue
+    from tests.conftest import LOCAL_HELM_CHART_DIR, local_operator
+
+    if local_operator():
+        return LOCAL_HELM_CHART_DIR, ""
+
+    # helm_chart_path not being passed would mean we would like to install helm chart from OCI registry.
+    if not helm_chart_path:
+        # If operator_version is not passed, we want to install the current version.
+        if not operator_version:
+            operator_version = os.environ.get(OCI_HELM_VERSION)
+
+        registry, repository, region = oci_chart_info()
+        # If ECR we need to login first to the OCI container registry
+        if registry == OCI_HELM_REGISTRY_ECR:
+            try:
+                helm_registry_login_to_ecr(registry, region)
+            except Exception as e:
+                raise Exception(f"Failed to login to ECR helm registry {registry}. Error: {e}")
+
+        # figure out the registry URI, based on dev/staging scenario
+        chart_uri = f"oci://{registry}/{repository}"
+        helm_chart_path = chart_uri
+
+    return helm_chart_path, operator_version