Update z-stream to trigger jobs for rosa versions (#185)

* support for trigger rosa-jobs

* update and rebase

* fix pre-commit

* update docs and rosa versions code

* update zstream tests

* fix typo and add comments for rosa-version

* skip if version not found

* add typing and default value

ci_jobs_trigger/libs/addons_webhook_trigger/addons_webhook_trigger.py

@@ -1,9 +1,7 @@
 import re
 
-import gitlab
-
 from ci_jobs_trigger.libs.utils.general import trigger_ci_job
-from ci_jobs_trigger.utils.general import get_config, AddonsWebhookTriggerError
+from ci_jobs_trigger.utils.general import get_config, get_gitlab_api, AddonsWebhookTriggerError
 
 ADDONS_WEBHOOK_JOBS_TRIGGER_CONFIG_STR = "ADDONS_WEBHOOK_JOBS_TRIGGER_CONFIG"
 
@@ -12,12 +10,6 @@ class RepositoryNotFoundError(Exception):
     pass
 
 
-def get_gitlab_api(url, token):
-    gitlab_api = gitlab.Gitlab(url=url, private_token=token, ssl_verify=False)
-    gitlab_api.auth()
-    return gitlab_api
-
-
 def repo_data_from_config(repository_name, config_data):
     data = config_data["repositories"].get(repository_name)
     if not data:

ci_jobs_trigger/libs/openshift_ci/zstream_trigger/README.md

@@ -12,6 +12,8 @@ Processed versions will be stored in `processed_versions_file_path`
 ### Configuration
 - Create a yaml file [example](../../../../config-examples/zstream-trigger-config.example.yaml) and update the relevant fields.
 - Export `OPENSHIFT_CI_ZSTREAM_TRIGGER_CONFIG` environment variable which points to the configuration yaml file
+- Z-stream supports triggering jobs for both OCP and ROSA versions. See [config file](../../../../config-examples/zstream-trigger-config.example.yaml) for examples on how to provide these versions.
+
 
 ```bash
 export OPENSHIFT_CI_ZSTREAM_TRIGGER_CONFIG="<path to yaml file>"

ci_jobs_trigger/libs/openshift_ci/zstream_trigger/zstream_trigger.py

@@ -3,17 +3,20 @@
 import logging
 import time
 import datetime
+import yaml
 
 from croniter import CroniterBadCronError, croniter
 from pyhelper_utils.general import stt, tts
 from typing import Dict, List
 
 from ocp_utilities.cluster_versions import get_accepted_cluster_versions
+from ocm_python_wrapper.ocm_client import OCMPythonClient
+from rosa.rosa_versions import get_rosa_versions
 from semver import Version
 import packaging.version
 
 from ci_jobs_trigger.utils.constant import DAYS_TO_SECONDS
-from ci_jobs_trigger.utils.general import get_config, send_slack_message
+from ci_jobs_trigger.utils.general import get_config, get_gitlab_api, send_slack_message
 from ci_jobs_trigger.libs.openshift_ci.utils.general import openshift_ci_trigger_job
 
 
@@ -55,6 +58,64 @@ def already_processed_version(
     return False
 
 
+def get_gitlab_project_file(config: Dict, ocm_env: str) -> Dict:
+    api = get_gitlab_api(url=config["gitlab_url"], token=config["gitlab_token"])
+    project = api.projects.get(config["gitlab_project"])
+    project_file_content = project.files.get(
+        file_path=f"config/{'prod' if ocm_env == 'production' else ocm_env}.yaml", ref="master"
+    )
+    return yaml.safe_load(project_file_content.decode().decode("utf-8"))
+
+
+def is_rosa_version_enabled(config: Dict, version: str, channel: str, ocm_env: str, logger: logging.Logger) -> bool:
+    processed_versions_file_path = config["processed_versions_file_path"]
+    processed_versions_file_content = processed_versions_file(
+        processed_versions_file_path=processed_versions_file_path, logger=logger
+    )
+    channel_version = f"{channel}-{version}"
+    enable_channel_version_key = f"{channel_version}-{ocm_env}-enable"
+    if processed_versions_file_content.get(enable_channel_version_key):
+        return True
+
+    project_file_content = get_gitlab_project_file(config=config, ocm_env=ocm_env)
+    for channel_groups in project_file_content.get("channel_groups", []):
+        if channel_version in channel_groups.get("channels", []):
+            processed_versions_file_content[enable_channel_version_key] = True
+            with open(processed_versions_file_path, "w") as fd:
+                json.dump(processed_versions_file_content, fd)
+            return True
+
+    return False
+
+
+def filter_rosa_versions_by_channel(all_rosa_versions: Dict, rosa_channel: str, version_channel: str) -> Dict:
+    filtered_rosa_dict: Dict[str, Dict[str, List[str]]] = {version_channel: {}}
+    for version_key, versions in all_rosa_versions[rosa_channel].items():
+        filtered_rosa_dict[version_channel][version_key] = [ver for ver in versions if version_channel in ver]
+
+    return filtered_rosa_dict
+
+
+def get_all_rosa_versions(
+    ocm_token: str, ocm_env: str, rosa_channel: str, version_channel: str, aws_region: str
+) -> Dict[str, Dict[str, List[str]]]:
+    ocm_client = OCMPythonClient(
+        token=ocm_token,
+        endpoint="https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token",
+        api_host=ocm_env,
+        discard_unknown_keys=True,
+    ).client
+    _all_rosa_versions = get_rosa_versions(ocm_client=ocm_client, aws_region=aws_region, channel_group=rosa_channel)
+
+    # To filter 'rc' and 'ec' versions from 'candidate' channel-group versions
+    if not rosa_channel == version_channel:
+        return filter_rosa_versions_by_channel(
+            all_rosa_versions=_all_rosa_versions, rosa_channel=rosa_channel, version_channel=version_channel
+        )
+
+    return _all_rosa_versions
+
+
 def trigger_jobs(config: Dict, jobs: List, logger: logging.Logger, zstream_version: str) -> bool:
     failed_triggers_jobs: List = []
     successful_triggers_jobs: List = []
@@ -141,37 +202,75 @@ def process_and_trigger_jobs(logger: logging.Logger, version: str | None = None)
                 trigger_res[_version] = "No jobs found"
                 continue
 
+            _rosa_env: str = ""
+
+            # If '___' found in any version, it will be considered as ROSA version
+            if "___" in _version:
+                _version, _rosa_env = _version.split("___")[:2]
+
             if "-" in _version:
                 _wanted_version, _version_channel = _version.split("-")
             else:
                 _wanted_version = _version
                 _version_channel = "stable"
 
-            _all_versions = get_accepted_cluster_versions()
-            _latest_version = _all_versions.get(_version_channel)[_wanted_version][0]
+            _base_version = f"{_version}-{_rosa_env}" if _rosa_env else _version
+            _rosa_channel = "candidate" if _rosa_env and _version_channel in ["rc", "ec"] else _version_channel
+
+            if _rosa_env and config.get("gitlab_project"):
+                if not is_rosa_version_enabled(
+                    config=config, version=_wanted_version, channel=_rosa_channel, ocm_env=_rosa_env, logger=logger
+                ):
+                    logger.info(
+                        f"{LOG_PREFIX} Version {_wanted_version}:{_version_channel} not enabled for ROSA {_rosa_env}, skipping"
+                    )
+                    trigger_res[_base_version] = "Not enabled for ROSA"
+                    continue
+
+            _all_versions = (
+                get_all_rosa_versions(
+                    ocm_env=_rosa_env,
+                    ocm_token=config["ocm_token"],
+                    rosa_channel=_rosa_channel,
+                    version_channel=_version_channel,
+                    aws_region=config["aws_region"],
+                )
+                if _rosa_env
+                else get_accepted_cluster_versions()
+            )
+
+            if not (wanted_version_list := _all_versions.get(_version_channel, {}).get(_wanted_version)):
+                logger.info(
+                    f"{LOG_PREFIX} Version {_wanted_version}:{_version_channel} {_rosa_env} not yet released, skipping"
+                )
+                trigger_res[_base_version] = "Not released"
+                continue
+
+            _latest_version = wanted_version_list[0]
             if already_processed_version(
-                base_version=_version,
+                base_version=_base_version,
                 new_version=_latest_version,
                 processed_versions_file_path=_processed_versions_file_path,
                 logger=logger,
             ):
-                logger.info(f"{LOG_PREFIX} Version {_wanted_version}:{_version_channel} already processed, skipping")
-                trigger_res[_version] = "Already processed"
+                logger.info(
+                    f"{LOG_PREFIX} Version {_wanted_version}:{_version_channel} {_rosa_env} already processed, skipping"
+                )
+                trigger_res[_base_version] = "Already processed"
                 continue
 
             logger.info(
-                f"{LOG_PREFIX} New Z-stream version {_latest_version}:{_version_channel} found, triggering jobs: {_jobs}"
+                f"{LOG_PREFIX} New Z-stream version {_latest_version}:{_version_channel} {_rosa_env} found, triggering jobs: {_jobs}"
             )
             if trigger_jobs(config=config, jobs=_jobs, logger=logger, zstream_version=_latest_version):
                 update_processed_version(
-                    base_version=_version,
+                    base_version=_base_version,
                     version=str(_latest_version),
                     processed_versions_file_path=_processed_versions_file_path,
                     logger=logger,
                 )
-                trigger_res[_version] = "Triggered"
+                trigger_res[_base_version] = "Triggered"
                 continue
-
         return trigger_res
 
 

ci_jobs_trigger/tests/zstream_trigger/manifests/versions.py → ci_jobs_trigger/tests/zstream_trigger/manifests/ocp_versions.py

@@ -1,4 +1,4 @@
-VERSIONS = {
+OCP_VERSIONS = {
     "ec": {"4.16": ["4.16.0-ec.2", "4.16.0-ec.1", "4.16.0-ec.0"]},
     "rc": {
         "4.15": [