diff --git a/.flake8 b/.flake8 index 84c55526..00d8a572 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,7 @@ [flake8] # These rules are ignored # - E501 line too long -ignore = E501 -max-line-length = 120 +# - E203 whitespace before ':' (conflicts with Black formatting) +# - D100-D400 docstring style warnings (legacy code, will be addressed separately) +extend-ignore = E501, E203, D +max-line-length = 160 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2be44be7..0b0dc7b2 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ kubectl.exe /build /.vscode /site +/.bob/.bob-errors +/.bob/notes diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 196ea21c..52db1ff0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ default_language_version: python: python repos: - - repo: https://github.com/hhatto/autopep8 - rev: v2.3.2 + - repo: https://github.com/psf/black + rev: 26.5.1 hooks: - - id: autopep8 + - id: black - repo: https://github.com/PyCQA/flake8 rev: 7.3.0 hooks: diff --git a/.secrets.baseline b/.secrets.baseline index 3f9e3a56..146e5aad 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2026-05-21T00:57:00Z", + "generated_at": "2026-06-17T12:31:04Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -87,6 +87,16 @@ "verified_result": null } ], + "bin/mas-devops-create-initial-users-for-saas": [ + { + "hashed_secret": "28ed3a797da3c48c309a4ef792147f3c56cfec40", + "is_secret": false, + "is_verified": false, + "line_number": 116, + "type": "Secret Keyword", + "verified_result": null + } + ], "bin/mas-devops-notify-slack": [ { "hashed_secret": "053f5ed451647be0bbb6f67b80d6726808cad97e", @@ -158,7 +168,7 @@ "hashed_secret": "4dfd3a58b4820476afe7efa2e2c52b267eec876a", "is_secret": false, "is_verified": false, - "line_number": 753, + "line_number": 688, "type": "Secret Keyword", "verified_result": null } @@ -168,7 +178,7 @@ "hashed_secret": "a4b48a81cdab1e1a5dd37907d6c85ca1c61ddc7c", "is_secret": false, "is_verified": false, - "line_number": 290, + "line_number": 245, "type": "Secret Keyword", "verified_result": null } @@ -178,7 +188,7 @@ "hashed_secret": "94f5ed592906089c107208b29e178ddf1f9f5143", "is_secret": false, "is_verified": false, - "line_number": 42, + "line_number": 40, "type": "Secret Keyword", "verified_result": null }, @@ -186,7 +196,33 @@ "hashed_secret": "a9410d9785f49750b9f8672794fc288558c1611c", "is_secret": false, "is_verified": false, - "line_number": 55, + "line_number": 60, + "type": "Secret Keyword", + "verified_result": null + } + ], + "test/src/test_users.py": [ + { + "hashed_secret": "74ba31d41223751c75cc0a453dd7df04889bdc72", + "is_secret": false, + "is_verified": false, + "line_number": 143, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "2edced2e2f44a016a5b2e5ce25fe704e62cbb2e7", + "is_secret": false, + "is_verified": false, + "line_number": 150, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "f66f0353de50f6990a5d761b00268056fa80f95f", + "is_secret": false, + "is_verified": false, + "line_number": 1945, "type": "Secret Keyword", "verified_result": null } diff --git a/README.md b/README.md index 1d88e132..c9d5894c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ mas.devops =============================================================================== -[![Code style: PEP8](https://img.shields.io/badge/code%20style-PEP--8-blue.svg)](https://peps.python.org/pep-0008/) +[![Code Style: Black](https://img.shields.io/badge/Code%20Style-Black-000000.svg)](https://github.com/psf/black) [![Flake8: checked](https://img.shields.io/badge/flake8-checked-blueviolet)](https://flake8.pycqa.org/en/latest/) ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/ibm-mas/python-devops/python-release.yml) [![PyPI - Version](https://img.shields.io/pypi/v/mas.devops)](https://pypi.org/project/mas-devops) @@ -31,7 +31,7 @@ installOpenShiftPipelines(dynamicClient) # Create the pipelines namespace and install the MAS tekton definitions createNamespace(dynamicClient, pipelinesNamespace) -updateTektonDefinitions(pipelinesNamespace, "/mascli/templates/ibm-mas-tekton.yaml") +updateTektonDefinitions(dynamicClient, pipelinesNamespace, "/mascli/templates/ibm-mas-tekton.yaml") # Launch the upgrade pipeline and print the URL to view the pipeline run pipelineURL = launchUpgradePipeline(self.dynamicClient, instanceId) @@ -73,7 +73,7 @@ mas-devops-create-initial-users-for-saas \ --manage-api-port 8443 \ --coreapi-port 8444 \ --admin-dashboard-port 8445 - + mas-devops-create-initial-users-for-saas \ --mas-instance-id tgk01 \ diff --git a/bin/mas-devops-apply-preinstall-rbac-for-saas b/bin/mas-devops-apply-preinstall-rbac-for-saas index ae511745..97b2b4d7 100644 --- a/bin/mas-devops-apply-preinstall-rbac-for-saas +++ b/bin/mas-devops-apply-preinstall-rbac-for-saas @@ -18,32 +18,48 @@ import sys import argparse import logging import urllib3 + urllib3.disable_warnings() if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Apply Pre-Install MAS RBAC') - + parser = argparse.ArgumentParser(description="Apply Pre-Install MAS RBAC") + parser.add_argument("--mas-instance-id", required=True, help="MAS Instance ID") parser.add_argument("--mas-version", required=True, help="MAS Version (e.g., 9.2)") - parser.add_argument("--admin-mode", required=False, default="namespaced", - choices=["cluster", "namespaced", "minimal"], - help="Admin mode: cluster, namespaced, or minimal") - parser.add_argument("--selected-apps", required=False, default="core", - help="Comma-separated list of apps (e.g., core,manage,iot)") - parser.add_argument("--rbac-root-dir", required=False, default="/opt/app-root/rbac", - help="Root directory containing RBAC manifests") - parser.add_argument("--log-level", required=False, - choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - default="INFO") - + parser.add_argument( + "--admin-mode", + required=False, + default="namespaced", + choices=["cluster", "namespaced", "minimal"], + help="Admin mode: cluster, namespaced, or minimal", + ) + parser.add_argument( + "--selected-apps", + required=False, + default="core", + help="Comma-separated list of apps (e.g., core,manage,iot)", + ) + parser.add_argument( + "--rbac-root-dir", + required=False, + default="/opt/app-root/rbac", + help="Root directory containing RBAC manifests", + ) + parser.add_argument( + "--log-level", + required=False, + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + default="INFO", + ) + args = parser.parse_args() - + # Setup logging log_level = getattr(logging, args.log_level) logger = logging.getLogger() logger.setLevel(log_level) - + ch = logging.StreamHandler() ch.setLevel(log_level) chFormatter = logging.Formatter( @@ -51,18 +67,20 @@ if __name__ == "__main__": ) ch.setFormatter(chFormatter) logger.addHandler(ch) - + mas_instance_id = args.mas_instance_id mas_version = ".".join(args.mas_version.split(".")[:2]) admin_mode = args.admin_mode selected_apps_str = args.selected_apps rbac_root_dir = args.rbac_root_dir - + # Parse selected apps selected_apps = None if selected_apps_str: - selected_apps = [app.strip() for app in selected_apps_str.split(',') if app.strip()] - + selected_apps = [ + app.strip() for app in selected_apps_str.split(",") if app.strip() + ] + logger.info("Configuration:") logger.info("--------------") logger.info(f"mas_instance_id: {mas_instance_id}") @@ -72,7 +90,7 @@ if __name__ == "__main__": logger.info(f"rbac_root_dir: {rbac_root_dir}") logger.info(f"log_level: {log_level}") logger.info("") - + try: # Try to load in-cluster configuration config.load_incluster_config() @@ -81,7 +99,7 @@ if __name__ == "__main__": # If that fails, fall back to kubeconfig file config.load_kube_config() logger.debug("Loaded kubeconfig file") - + try: dynClient = DynamicClient(client.api_client.ApiClient()) applyPreInstallMASRBAC( @@ -90,12 +108,13 @@ if __name__ == "__main__": masInstanceId=mas_instance_id, adminMode=admin_mode, selectedApps=selected_apps, - rbacRootDir=rbac_root_dir + rbacRootDir=rbac_root_dir, ) logger.info("Pre-Install MAS RBAC applied successfully") sys.exit(0) except Exception as e: logger.error(f"Error applying Pre-Install MAS RBAC: {e}") import traceback + traceback.print_exc() - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/bin/mas-devops-create-initial-users-for-saas b/bin/mas-devops-create-initial-users-for-saas index 1a05fbf2..ac4925ea 100644 --- a/bin/mas-devops-create-initial-users-for-saas +++ b/bin/mas-devops-create-initial-users-for-saas @@ -21,6 +21,7 @@ from kubernetes.config.config_exception import ConfigException import argparse import logging import urllib3 + urllib3.disable_warnings() @@ -30,7 +31,12 @@ if __name__ == "__main__": # Primary Options parser.add_argument("--mas-instance-id", required=True) parser.add_argument("--mas-workspace-id", required=True) - parser.add_argument("--log-level", required=False, choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="INFO") + parser.add_argument( + "--log-level", + required=False, + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + default="INFO", + ) parser.add_argument("--coreapi-port", required=False, default=443) parser.add_argument("--admin-dashboard-port", required=False, default=443) parser.add_argument("--manage-api-port", required=False, default=443) @@ -86,31 +92,47 @@ if __name__ == "__main__": config.load_kube_config() logger.debug("Loaded kubeconfig file") - user_utils = MASUserUtils(mas_instance_id, mas_workspace_id, client.api_client.ApiClient(), mas_version, coreapi_port=coreapi_port, admin_dashboard_port=admin_dashboard_port, manage_api_port=manage_api_port) + user_utils = MASUserUtils( + mas_instance_id, + mas_workspace_id, + client.api_client.ApiClient(), + mas_version, + coreapi_port=coreapi_port, + admin_dashboard_port=admin_dashboard_port, + manage_api_port=manage_api_port, + ) if initial_users_secret_name is not None: - logger.info(f"Loading initial_users configuration from secret {initial_users_secret_name}") + logger.info( + f"Loading initial_users configuration from secret {initial_users_secret_name}" + ) session = boto3.session.Session() aws_sm_client = session.client( - service_name='secretsmanager', + service_name="secretsmanager", ) try: - initial_users_secret = aws_sm_client.get_secret_value( # pragma: allowlist secret - SecretId=initial_users_secret_name + initial_users_secret = ( + aws_sm_client.get_secret_value( # pragma: allowlist secret + SecretId=initial_users_secret_name + ) ) except ClientError as e: - if e.response['Error']['Code'] == 'ResourceNotFoundException': - logger.info(f"Secret {initial_users_secret_name} was not found, nothing to do, exiting now.") + if e.response["Error"]["Code"] == "ResourceNotFoundException": + logger.info( + f"Secret {initial_users_secret_name} was not found, nothing to do, exiting now." + ) sys.exit(0) - raise Exception(f"Failed to fetch secret {initial_users_secret_name}: {str(e)}") + raise Exception( + f"Failed to fetch secret {initial_users_secret_name}: {str(e)}" + ) - secret_json = json.loads(initial_users_secret['SecretString']) + secret_json = json.loads(initial_users_secret["SecretString"]) initial_users = user_utils.parse_initial_users_from_aws_secret_json(secret_json) elif initial_users_yaml_file is not None: - with open(initial_users_yaml_file, 'r') as file: + with open(initial_users_yaml_file, "r") as file: initial_users = yaml.safe_load(file) else: raise Exception("Something unexpected happened") @@ -122,7 +144,9 @@ if __name__ == "__main__": if initial_users_secret_name is not None: has_updates = False for completed_user in result["completed"]: - logger.info(f"Removing synced user {completed_user['email']} from {initial_users_secret_name} secret") + logger.info( + f"Removing synced user {completed_user['email']} from {initial_users_secret_name} secret" + ) secret_json.pop(completed_user["email"]) has_updates = True @@ -131,10 +155,12 @@ if __name__ == "__main__": try: aws_sm_client.update_secret( # pragma: allowlist secret SecretId=initial_users_secret_name, - SecretString=json.dumps(secret_json) + SecretString=json.dumps(secret_json), ) except ClientError as e: - raise Exception(f"Failed to update secret {initial_users_secret_name}: {str(e)}") + raise Exception( + f"Failed to update secret {initial_users_secret_name}: {str(e)}" + ) if len(result["failed"]) > 0: failed_user_ids = list(map(lambda u: u["email"], result["failed"])) diff --git a/bin/mas-devops-db2-validate-config b/bin/mas-devops-db2-validate-config index 745f872c..4fb49dbc 100644 --- a/bin/mas-devops-db2-validate-config +++ b/bin/mas-devops-db2-validate-config @@ -17,6 +17,7 @@ import argparse import logging import urllib3 + urllib3.disable_warnings() @@ -27,14 +28,19 @@ if __name__ == "__main__": # Primary Options parser.add_argument("--mas-instance-id", required=True) parser.add_argument("--mas-app-id", required=True) - parser.add_argument("--database-role", default='primary', required=False) - parser.add_argument("--log-level", required=False, choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="WARNING") + parser.add_argument("--database-role", default="primary", required=False) + parser.add_argument( + "--log-level", + required=False, + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + default="WARNING", + ) args, unknown = parser.parse_known_args() log_level = getattr(logging, args.log_level) logging.basicConfig() - logging.getLogger('mas.devops.db2').setLevel(level=log_level) + logging.getLogger("mas.devops.db2").setLevel(level=log_level) try: # Try to load in-cluster configuration @@ -49,5 +55,5 @@ if __name__ == "__main__": client.api_client.ApiClient(), args.mas_instance_id, args.mas_app_id, - args.database_role + args.database_role, ) diff --git a/bin/mas-devops-notify-slack b/bin/mas-devops-notify-slack index a139edd0..74108505 100755 --- a/bin/mas-devops-notify-slack +++ b/bin/mas-devops-notify-slack @@ -51,14 +51,14 @@ def notifyProvisionFyre(channels: list[str], rc: int, additionalMsg: str | None SlackUtil.buildHeader(f":glyph-ok: Your IBM DevIT Fyre OCP cluster ({name}) is ready"), SlackUtil.buildSection(f"{url}"), SlackUtil.buildSection(f"- Username: `{username}`\n- Password: `{password}`"), - SlackUtil.buildSection(f"{toolchainLink}") + SlackUtil.buildSection(f"{toolchainLink}"), ] if additionalMsg is not None: message.append(SlackUtil.buildSection(additionalMsg)) else: message = [ SlackUtil.buildHeader(f":glyph-fail: Your IBM DevIT Fyre OCP cluster ({name}) failed to deploy"), - SlackUtil.buildSection(f"{toolchainLink}") + SlackUtil.buildSection(f"{toolchainLink}"), ] response = SlackUtil.postMessageBlocks(channels, message) @@ -81,14 +81,14 @@ def notifyProvisionRoks(channels: list[str], rc: int, additionalMsg: str | None message = [ SlackUtil.buildHeader(f":glyph-ok: Your IBM Cloud ROKS cluster ({name}) is ready"), SlackUtil.buildSection(f"{url}"), - SlackUtil.buildSection(f" | {toolchainLink}") + SlackUtil.buildSection(f" | {toolchainLink}"), ] if additionalMsg is not None: message.append(SlackUtil.buildSection(additionalMsg)) else: message = [ SlackUtil.buildHeader(f":glyph-fail: Your IBM Cloud ROKS cluster ({name}) failed to deploy"), - SlackUtil.buildSection(f" | {toolchainLink}") + SlackUtil.buildSection(f" | {toolchainLink}"), ] response = SlackUtil.postMessageBlocks(channels, message) @@ -97,7 +97,12 @@ def notifyProvisionRoks(channels: list[str], rc: int, additionalMsg: str | None return response.data.get("ok", False) -def notifyPipelineStart(channels: list[str], instanceId: str | None = None, pipelineName: str | None = None, namespace: str | None = None) -> dict | None: +def notifyPipelineStart( + channels: list[str], + instanceId: str | None = None, + pipelineName: str | None = None, + namespace: str | None = None, +) -> dict | None: """Send Slack notification about pipeline start and create thread for all channels.""" # Exit early if no channels provided if not channels or len(channels) == 0: @@ -124,7 +129,7 @@ def notifyPipelineStart(channels: list[str], instanceId: str | None = None, pipe instanceInfo = f"Instance ID: `{instanceId}`" if instanceId else "" message = [ SlackUtil.buildHeader(f"🚀 MAS {pipelineName} Pipeline Started"), - SlackUtil.buildSection(f"Pipeline Run: {pipelineName}\n{instanceInfo}\n{toolchainLink}") + SlackUtil.buildSection(f"Pipeline Run: {pipelineName}\n{instanceInfo}\n{toolchainLink}"), ] response = SlackUtil.postMessageBlocks(channels, message) @@ -159,7 +164,13 @@ def notifyPipelineStart(channels: list[str], instanceId: str | None = None, pipe return SlackUtil.getThreadConfigMap(namespace, instanceId, pipelineName) -def notifyAnsibleStart(channels: list[str], taskName: str, instanceId: str | None = None, pipelineName: str | None = None, namespace: str | None = None) -> bool: +def notifyAnsibleStart( + channels: list[str], + taskName: str, + instanceId: str | None = None, + pipelineName: str | None = None, + namespace: str | None = None, +) -> bool: """Send Slack notification about Ansible task start to all channels.""" # Exit early if no channels provided if not channels or len(channels) == 0: @@ -188,9 +199,7 @@ def notifyAnsibleStart(channels: list[str], taskName: str, instanceId: str | Non return False # Send task start message as thread reply to all channels - taskMessage = [ - SlackUtil.buildSection(f"⏳ *{taskName}* - Started") - ] + taskMessage = [SlackUtil.buildSection(f"⏳ *{taskName}* - Started")] allSuccess = True taskMessageData = {} @@ -220,7 +229,14 @@ def notifyAnsibleStart(channels: list[str], taskName: str, instanceId: str | Non return allSuccess -def notifyAnsibleComplete(channels: list[str], rc: int, taskName: str, instanceId: str | None = None, pipelineName: str | None = None, namespace: str | None = None) -> bool: +def notifyAnsibleComplete( + channels: list[str], + rc: int, + taskName: str, + instanceId: str | None = None, + pipelineName: str | None = None, + namespace: str | None = None, +) -> bool: """Send Slack notification about Ansible task completion status to all channels.""" # Exit early if no channels provided if not channels or len(channels) == 0: @@ -272,6 +288,7 @@ def notifyAnsibleComplete(channels: list[str], rc: int, taskName: str, instanceI durationText = "" if taskMessageTs: from datetime import datetime, timezone + try: # Message timestamp is in format "1234567890.123456" startTime = float(taskMessageTs) @@ -291,9 +308,7 @@ def notifyAnsibleComplete(channels: list[str], rc: int, taskName: str, instanceI print(f"Failed to calculate duration for channel {idx}: {e}") # Build the completion message - taskMessage = [ - SlackUtil.buildSection(f"{emoji} *{taskName}* - {status}{durationText}") - ] + taskMessage = [SlackUtil.buildSection(f"{emoji} *{taskName}* - {status}{durationText}")] if rc != 0: taskMessage.append(SlackUtil.buildSection(f"Return Code: `{rc}`\nCheck logs for details")) @@ -317,7 +332,13 @@ def notifyAnsibleComplete(channels: list[str], rc: int, taskName: str, instanceI return allSuccess -def notifyPipelineComplete(channels: list[str], rc: int, instanceId: str | None = None, pipelineName: str | None = None, namespace: str | None = None) -> bool: +def notifyPipelineComplete( + channels: list[str], + rc: int, + instanceId: str | None = None, + pipelineName: str | None = None, + namespace: str | None = None, +) -> bool: """Send Slack notification about pipeline completion to all channels and cleanup ConfigMap.""" # Exit early if no channels provided if not channels or len(channels) == 0: @@ -351,6 +372,7 @@ def notifyPipelineComplete(channels: list[str], rc: int, instanceId: str | None durationText = "" if startTime: from datetime import datetime, timezone + try: start = datetime.fromisoformat(startTime.replace("Z", "+00:00")) end = datetime.now(timezone.utc) @@ -376,7 +398,7 @@ def notifyPipelineComplete(channels: list[str], rc: int, instanceId: str | None message = [ SlackUtil.buildHeader(f"{emoji} MAS {pipelineName} Pipeline {status}"), - SlackUtil.buildSection(f"Pipeline Run: {pipelineName}\n{instanceInfo}{durationText}{additionalInfo}") + SlackUtil.buildSection(f"Pipeline Run: {pipelineName}\n{instanceInfo}{durationText}{additionalInfo}"), ] allSuccess = True @@ -419,7 +441,12 @@ if __name__ == "__main__": parser.add_argument("--task-name", required=False, default="") parser.add_argument("--instance-id", required=False, default=None) parser.add_argument("--pipeline-name", required=False, default=None) - parser.add_argument("--namespace", required=False, default=None, help="Pipeline namespace (e.g., mas-{instanceId}-pipelines or aiservice-{instanceId}-pipelines)") + parser.add_argument( + "--namespace", + required=False, + default=None, + help="Pipeline namespace (e.g., mas-{instanceId}-pipelines or aiservice-{instanceId}-pipelines)", + ) args, unknown = parser.parse_known_args() @@ -435,6 +462,13 @@ if __name__ == "__main__": elif args.action == "ansible-start": notifyAnsibleStart(channelList, args.task_name, args.instance_id, args.pipeline_name, namespace) elif args.action == "ansible-complete": - notifyAnsibleComplete(channelList, args.rc, args.task_name, args.instance_id, args.pipeline_name, namespace) + notifyAnsibleComplete( + channelList, + args.rc, + args.task_name, + args.instance_id, + args.pipeline_name, + namespace, + ) elif args.action == "pipeline-complete": notifyPipelineComplete(channelList, args.rc, args.instance_id, args.pipeline_name, namespace) diff --git a/bin/mas-devops-saas-job-cleaner b/bin/mas-devops-saas-job-cleaner index 374a7ef3..407dd84e 100644 --- a/bin/mas-devops-saas-job-cleaner +++ b/bin/mas-devops-saas-job-cleaner @@ -18,6 +18,7 @@ import argparse from mas.devops.saas.job_cleaner import JobCleaner import urllib3 + urllib3.disable_warnings() @@ -26,10 +27,29 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() # Primary Options - parser.add_argument("--log-level", required=False, choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], default="WARNING") - parser.add_argument("--label", required=True, help="Kubernetes resource label used to identify Job groups to cleanup") - parser.add_argument("--limit", required=False, help="Limit page sizes fetched from K8S API. Larger values will use more memory but less cpu time / network IO.", default=100) - parser.add_argument("--dry-run", required=False, help="When specified, nothing will actually be deleted from the cluster", action="store_true") + parser.add_argument( + "--log-level", + required=False, + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + default="WARNING", + ) + parser.add_argument( + "--label", + required=True, + help="Kubernetes resource label used to identify Job groups to cleanup", + ) + parser.add_argument( + "--limit", + required=False, + help="Limit page sizes fetched from K8S API. Larger values will use more memory but less cpu time / network IO.", + default=100, + ) + parser.add_argument( + "--dry-run", + required=False, + help="When specified, nothing will actually be deleted from the cluster", + action="store_true", + ) args, unknown = parser.parse_known_args() diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 5676d9b2..d915938c 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -56,7 +56,7 @@ print("OpenShift Pipelines installed successfully") pipelinesNamespace = "mas-myinstance-pipelines" tektonYamlPath = "/path/to/ibm-mas-tekton.yaml" -updateTektonDefinitions(pipelinesNamespace, tektonYamlPath) +updateTektonDefinitions(dynClient, pipelinesNamespace, tektonYamlPath) print("Tekton definitions updated successfully") ``` @@ -96,7 +96,7 @@ createNamespace(dynClient, pipelinesNamespace) # Update Tekton definitions print("Updating Tekton definitions...") -updateTektonDefinitions(pipelinesNamespace, tektonYamlPath) +updateTektonDefinitions(dynClient, pipelinesNamespace, tektonYamlPath) # Launch upgrade pipeline print("Launching upgrade pipeline...") diff --git a/docs/index.md b/docs/index.md index 5aaa1746..2408bbb6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,7 +45,7 @@ installOpenShiftPipelines(dynamicClient) # Create the pipelines namespace and install the MAS tekton definitions createNamespace(dynamicClient, pipelinesNamespace) -updateTektonDefinitions(pipelinesNamespace, "/mascli/templates/ibm-mas-tekton.yaml") +updateTektonDefinitions(dynamicClient, pipelinesNamespace, "/mascli/templates/ibm-mas-tekton.yaml") # Launch the upgrade pipeline and print the URL to view the pipeline run pipelineURL = launchUpgradePipeline(self.dynamicClient, instanceId) diff --git a/pyproject.toml b/pyproject.toml index 2d72e016..c9565db4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,3 +8,8 @@ build-backend = "setuptools.build_meta" markers = [ "openshift: marks tests as requiring an OpenShift cluster (deselect with '-m \"not openshift\"')", ] + +[tool.black] +line-length = 160 +target-version = ['py312'] +include = '(src|tests)/.*\.py$' diff --git a/setup.py b/setup.py index a19c67aa..5fd57e8e 100644 --- a/setup.py +++ b/setup.py @@ -11,11 +11,12 @@ import codecs import sys import os -sys.path.insert(0, 'src') + +sys.path.insert(0, "src") here = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: +with open(os.path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() # Maintain a single source of versioning @@ -24,13 +25,13 @@ def read(rel_path): here = os.path.abspath(os.path.dirname(__file__)) - with codecs.open(os.path.join(here, rel_path), 'r') as fp: + with codecs.open(os.path.join(here, rel_path), "r") as fp: return fp.read() def get_version(rel_path): for line in read(rel_path).splitlines(): - if line.startswith('__version__'): + if line.startswith("__version__"): delim = '"' if '"' in line else "'" return line.split(delim)[1] else: @@ -38,62 +39,62 @@ def get_version(rel_path): setup( - name='mas-devops', + name="mas-devops", version=get_version("src/mas/devops/__init__.py"), - author='David Parker', - author_email='parkerda@uk.ibm.com', - package_dir={'': 'src'}, - packages=find_namespace_packages(where='src'), + author="David Parker", + author_email="parkerda@uk.ibm.com", + package_dir={"": "src"}, + packages=find_namespace_packages(where="src"), include_package_data=True, - url='https://github.com/ibm-mas/python-devops', - license='Eclipse Public License - v1.0', - description='Python for Maximo Application Suite Dev/Ops', + url="https://github.com/ibm-mas/python-devops", + license="Eclipse Public License - v1.0", + description="Python for Maximo Application Suite Dev/Ops", long_description=long_description, - long_description_content_type='text/markdown', + long_description_content_type="text/markdown", install_requires=[ - 'pyyaml', # MIT License - 'openshift', # Apache Software License - 'kubernetes<36', # Apache Software License - 'kubeconfig', # BSD License - 'setuptools', # MIT License (required to install kubeconfig) - 'jinja2', # BSD License - 'jinja2-base64-filters', # MIT License - 'semver', # BSD License - 'boto3', # Apache Software License - 'slack_sdk', # MIT License - "packaging", # Apache Software License + "pyyaml", # MIT License + "openshift", # Apache Software License + "kubernetes<36", # Apache Software License + "kubeconfig", # BSD License + "setuptools", # MIT License (required to install kubeconfig) + "jinja2", # BSD License + "jinja2-base64-filters", # MIT License + "semver", # BSD License + "boto3", # Apache Software License + "slack_sdk", # MIT License + "packaging", # Apache Software License ], extras_require={ - 'dev': [ - 'build', # MIT License - 'flake8', # MIT License - 'pytest', # MIT License - 'pytest-mock', # MIT License - 'requests-mock', # Apache Software License + "dev": [ + "build", # MIT License + "flake8", # MIT License + "pytest", # MIT License + "pytest-mock", # MIT License + "requests-mock", # Apache Software License + ], + "docs": [ + "mkdocs", # BSD License + "mkdocs-material", # MIT License + "mkdocstrings[python]", # ISC License + "pymdown-extensions", # MIT License ], - 'docs': [ - 'mkdocs', # BSD License - 'mkdocs-material', # MIT License - 'mkdocstrings[python]', # ISC License - 'pymdown-extensions', # MIT License - ] }, classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.12', - 'Topic :: Communications', - 'Topic :: Internet', - 'Topic :: Software Development :: Libraries :: Python Modules' + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", + "Topic :: Communications", + "Topic :: Internet", + "Topic :: Software Development :: Libraries :: Python Modules", ], scripts=[ - 'bin/mas-devops-db2-validate-config', - 'bin/mas-devops-create-initial-users-for-saas', - 'bin/mas-devops-saas-job-cleaner', - 'bin/mas-devops-notify-slack', - 'bin/mas-devops-apply-preinstall-rbac-for-saas', - ] + "bin/mas-devops-db2-validate-config", + "bin/mas-devops-create-initial-users-for-saas", + "bin/mas-devops-saas-job-cleaner", + "bin/mas-devops-notify-slack", + "bin/mas-devops-apply-preinstall-rbac-for-saas", + ], ) diff --git a/src/mas/devops/aiservice.py b/src/mas/devops/aiservice.py index 4580c52a..e11f869c 100644 --- a/src/mas/devops/aiservice.py +++ b/src/mas/devops/aiservice.py @@ -10,7 +10,11 @@ import logging from openshift.dynamic import DynamicClient -from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError +from openshift.dynamic.exceptions import ( + NotFoundError, + ResourceNotFoundError, + UnauthorizedError, +) from .ocp import listInstances from .olm import getSubscription @@ -105,7 +109,10 @@ def verifyAiServiceTenantInstance(dynClient: DynamicClient, instanceId: str, ten """ try: aiserviceTenantAPI = dynClient.resources.get(api_version="aiservice.ibm.com/v1", kind="AIServiceTenant") - aiserviceTenantAPI.get(name=f"aiservice-{instanceId}-{tenantId}", namespace=f"aiservice-{instanceId}") + aiserviceTenantAPI.get( + name=f"aiservice-{instanceId}-{tenantId}", + namespace=f"aiservice-{instanceId}", + ) return True except NotFoundError: print("NOT FOUND") diff --git a/src/mas/devops/backup.py b/src/mas/devops/backup.py index 172f26e4..759166da 100644 --- a/src/mas/devops/backup.py +++ b/src/mas/devops/backup.py @@ -42,13 +42,13 @@ class LiteralDumper(yaml.SafeDumper): pass def str_representer(dumper, data): - if '\n' in data: - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') - return dumper.represent_scalar('tag:yaml.org,2002:str', data) + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) LiteralDumper.add_representer(str, str_representer) - with open(file_path, 'w') as yaml_file: + with open(file_path, "w") as yaml_file: yaml.dump(content, yaml_file, default_flow_style=False, Dumper=LiteralDumper) return True except Exception as e: @@ -61,30 +61,30 @@ def filterResourceData(data: dict) -> dict: Filter metadata from Resource data and create minimal dict """ metadata_fields_to_remove = [ - 'annotations', - 'creationTimestamp', - 'generation', - 'resourceVersion', - 'selfLink', - 'ownerReferences', - 'uid', - 'managedFields' + "annotations", + "creationTimestamp", + "generation", + "resourceVersion", + "selfLink", + "ownerReferences", + "uid", + "managedFields", ] filteredCopy = data.copy() - if 'metadata' in filteredCopy: + if "metadata" in filteredCopy: for field in metadata_fields_to_remove: - if field in filteredCopy['metadata']: - del filteredCopy['metadata'][field] + if field in filteredCopy["metadata"]: + del filteredCopy["metadata"][field] - if 'status' in filteredCopy: - del filteredCopy['status'] + if "status" in filteredCopy: + del filteredCopy["status"] # Remove labels with uid # this will cause problem when restoring the backup - if 'metadata' in filteredCopy and 'labels' in filteredCopy['metadata']: - for key in list(filteredCopy['metadata']['labels'].keys()): + if "metadata" in filteredCopy and "labels" in filteredCopy["metadata"]: + for key in list(filteredCopy["metadata"]["labels"].keys()): if "uid" in key.lower(): - filteredCopy['metadata']['labels'].pop(key) + filteredCopy["metadata"]["labels"].pop(key) return filteredCopy @@ -107,12 +107,12 @@ def extract_secrets_from_dict(data, secret_names=None): if isinstance(data, dict): for key, value in data.items(): # Check if this key is 'secretName' and has a string value - if (key == 'secretName' or 'secretname' in key.lower()) and isinstance(value, str) and value: + if (key == "secretName" or "secretname" in key.lower()) and isinstance(value, str) and value: secret_names.add(value) # Check if this key contains 'secretRef' and contains a 'name' field - elif 'SecretRef' in key and isinstance(value, dict): - if 'name' in value and isinstance(value['name'], str) and value['name']: - secret_names.add(value['name']) + elif "SecretRef" in key and isinstance(value, dict): + if "name" in value and isinstance(value["name"], str) and value["name"]: + secret_names.add(value["name"]) # Recursively search nested structures elif isinstance(value, (dict, list)): extract_secrets_from_dict(value, secret_names) @@ -125,7 +125,15 @@ def extract_secrets_from_dict(data, secret_names=None): return secret_names -def backupResources(dynClient: DynamicClient, kind: str, api_version: str, backup_path: str, namespace=None, name=None, labels=None) -> tuple: +def backupResources( + dynClient: DynamicClient, + kind: str, + api_version: str, + backup_path: str, + namespace=None, + name=None, + labels=None, +) -> tuple: """ Backup resources of a given kind. If name is provided, backs up that specific resource. @@ -153,7 +161,7 @@ def backupResources(dynClient: DynamicClient, kind: str, api_version: str, backu # Build label selector string if labels provided label_selector = None if labels: - label_selector = ','.join(labels) + label_selector = ",".join(labels) # Determine scope description for logging scope_desc = f"namespace '{namespace}'" if namespace else "cluster-level" @@ -176,11 +184,21 @@ def backupResources(dynClient: DynamicClient, kind: str, api_version: str, backu else: logger.info(f"{kind} '{name}' not found in {scope_desc}, skipping backup") not_found_count = 1 - return (backed_up_count, not_found_count, failed_count, discovered_secrets) + return ( + backed_up_count, + not_found_count, + failed_count, + discovered_secrets, + ) except NotFoundError: logger.error(f"{kind} '{name}' not found in {scope_desc}, skipping backup") not_found_count = 1 - return (backed_up_count, not_found_count, failed_count, discovered_secrets) + return ( + backed_up_count, + not_found_count, + failed_count, + discovered_secrets, + ) else: # Backup all resources of this kind logger.info(f"Backing up all {kind} resources from {scope_desc} (API version: {api_version}){label_desc}") @@ -188,9 +206,9 @@ def backupResources(dynClient: DynamicClient, kind: str, api_version: str, backu # Build get parameters get_params = {} if namespace: - get_params['namespace'] = namespace + get_params["namespace"] = namespace if label_selector: - get_params['label_selector'] = label_selector + get_params["label_selector"] = label_selector resources = resourceAPI.get(**get_params) resources_to_process = resources.items @@ -201,8 +219,8 @@ def backupResources(dynClient: DynamicClient, kind: str, api_version: str, backu resource_dict = resource.to_dict() # Extract secrets from this resource if it's not a Secret itself - if kind != 'Secret': - secrets = extract_secrets_from_dict(resource_dict.get('spec', {})) + if kind != "Secret": + secrets = extract_secrets_from_dict(resource_dict.get("spec", {})) if secrets: logger.info(f"Found {len(secrets)} secret reference(s) in {kind} '{resource_name}': {', '.join(sorted(secrets))}") discovered_secrets.update(secrets) @@ -246,7 +264,7 @@ def uploadToS3( endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None, - region_name=None + region_name=None, ) -> bool: """ Upload a tar.gz file to S3-compatible storage. @@ -272,7 +290,7 @@ def uploadToS3( logger.error(f"File not found: {file_path}") return False - if not file_path.endswith('.tar.gz'): + if not file_path.endswith(".tar.gz"): logger.warning(f"File does not have .tar.gz extension: {file_path}") # Configure S3 client @@ -280,16 +298,16 @@ def uploadToS3( s3_config = {} if endpoint_url: - s3_config['endpoint_url'] = endpoint_url + s3_config["endpoint_url"] = endpoint_url if aws_access_key_id and aws_secret_access_key: - s3_config['aws_access_key_id'] = aws_access_key_id - s3_config['aws_secret_access_key'] = aws_secret_access_key + s3_config["aws_access_key_id"] = aws_access_key_id + s3_config["aws_secret_access_key"] = aws_secret_access_key if region_name: - s3_config['region_name'] = region_name + s3_config["region_name"] = region_name else: - s3_config['region_name'] = 'us-east-1' + s3_config["region_name"] = "us-east-1" - s3_client = boto3.client('s3', **s3_config) + s3_client = boto3.client("s3", **s3_config) # Upload the file logger.info(f"Uploading {file_path} to s3://{bucket_name}/{object_name}") @@ -309,8 +327,8 @@ def uploadToS3( logger.error("AWS credentials not found. Please provide credentials or configure environment variables.") return False except ClientError as e: - error_code = e.response.get('Error', {}).get('Code', 'Unknown') - error_message = e.response.get('Error', {}).get('Message', str(e)) + error_code = e.response.get("Error", {}).get("Code", "Unknown") + error_message = e.response.get("Error", {}).get("Message", str(e)) logger.error(f"S3 client error ({error_code}): {error_message}") return False except Exception as e: @@ -325,7 +343,7 @@ def downloadFromS3( endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None, - region_name=None + region_name=None, ) -> bool: """ Download a tar.gz file from S3-compatible storage to a backup directory. @@ -355,7 +373,7 @@ def downloadFromS3( file_path = os.path.join(local_dir, object_name) # Warn if file doesn't have .tar.gz extension - if not object_name.endswith('.tar.gz'): + if not object_name.endswith(".tar.gz"): logger.warning(f"Object does not have .tar.gz extension: {object_name}") # Configure S3 client @@ -363,26 +381,26 @@ def downloadFromS3( s3_config = {} if endpoint_url: - s3_config['endpoint_url'] = endpoint_url + s3_config["endpoint_url"] = endpoint_url if aws_access_key_id and aws_secret_access_key: - s3_config['aws_access_key_id'] = aws_access_key_id - s3_config['aws_secret_access_key'] = aws_secret_access_key + s3_config["aws_access_key_id"] = aws_access_key_id + s3_config["aws_secret_access_key"] = aws_secret_access_key if region_name: - s3_config['region_name'] = region_name + s3_config["region_name"] = region_name else: - s3_config['region_name'] = 'us-east-1' + s3_config["region_name"] = "us-east-1" - s3_client = boto3.client('s3', **s3_config) + s3_client = boto3.client("s3", **s3_config) # Check if object exists and get its size logger.info(f"Downloading s3://{bucket_name}/{object_name} to {file_path}") try: response = s3_client.head_object(Bucket=bucket_name, Key=object_name) - file_size = response.get('ContentLength', 0) + file_size = response.get("ContentLength", 0) logger.info(f"Object size: {file_size / (1024 * 1024):.2f} MB") except ClientError as e: - if e.response.get('Error', {}).get('Code') == '404': + if e.response.get("Error", {}).get("Code") == "404": logger.error(f"Object not found in S3: s3://{bucket_name}/{object_name}") return False raise @@ -404,8 +422,8 @@ def downloadFromS3( logger.error("AWS credentials not found. Please provide credentials or configure environment variables.") return False except ClientError as e: - error_code = e.response.get('Error', {}).get('Code', 'Unknown') - error_message = e.response.get('Error', {}).get('Message', str(e)) + error_code = e.response.get("Error", {}).get("Code", "Unknown") + error_message = e.response.get("Error", {}).get("Message", str(e)) logger.error(f"S3 client error ({error_code}): {error_message}") return False except Exception as e: diff --git a/src/mas/devops/data/__init__.py b/src/mas/devops/data/__init__.py index 3101b32c..b738358f 100644 --- a/src/mas/devops/data/__init__.py +++ b/src/mas/devops/data/__init__.py @@ -46,9 +46,7 @@ def getCatalog(name: str) -> dict: pathToCatalog = path.join(modulePath, "catalogs", catalogFileName) if not path.exists(pathToCatalog): - raise NoSuchCatalogError( - f"Catalog {name} is unknown: {pathToCatalog} does not exist" - ) + raise NoSuchCatalogError(f"Catalog {name} is unknown: {pathToCatalog} does not exist") with open(pathToCatalog) as stream: return yaml.safe_load(stream) diff --git a/src/mas/devops/db2.py b/src/mas/devops/db2.py index 1f6ed37b..f7f89b7c 100644 --- a/src/mas/devops/db2.py +++ b/src/mas/devops/db2.py @@ -21,7 +21,12 @@ logger = logging.getLogger(__name__) -def get_db2u_instance_cr(custom_objects_api: client.CustomObjectsApi, mas_instance_id: str, mas_app_id: str, database_role='primary') -> dict: +def get_db2u_instance_cr( + custom_objects_api: client.CustomObjectsApi, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +) -> dict: """ Retrieve the Db2uInstance custom resource for a specific MAS application database. @@ -37,7 +42,10 @@ def get_db2u_instance_cr(custom_objects_api: client.CustomObjectsApi, mas_instan Raises: kubernetes.client.exceptions.ApiException: If the custom resource is not found or cannot be retrieved """ - cr_name = {'primary': f"db2wh-{mas_instance_id}-{mas_app_id}", 'standby': f"db2wh-{mas_instance_id}-{mas_app_id}-sdb"}[database_role] + cr_name = { + "primary": f"db2wh-{mas_instance_id}-{mas_app_id}", + "standby": f"db2wh-{mas_instance_id}-{mas_app_id}-sdb", + }[database_role] namespace = f"db2u-{mas_instance_id}" logger.debug(f"Getting Db2uInstance CR {cr_name} in {namespace}") @@ -46,13 +54,19 @@ def get_db2u_instance_cr(custom_objects_api: client.CustomObjectsApi, mas_instan version="v1", namespace=namespace, plural="db2uinstances", - name=cr_name + name=cr_name, ) return db2u_instance_cr -def db2_pod_exec(core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, command: list, database_role='primary') -> str: +def db2_pod_exec( + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + command: list, + database_role="primary", +) -> str: """ Execute a command in a DB2 pod for a specific MAS application database. @@ -69,12 +83,21 @@ def db2_pod_exec(core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id Raises: Exception: If the command execution fails """ - pod_name = {'primary': f"c-db2wh-{mas_instance_id}-{mas_app_id}-db2u-0", 'standby': f"c-db2wh-{mas_instance_id}-{mas_app_id}-sdb-db2u-0"}[database_role] + pod_name = { + "primary": f"c-db2wh-{mas_instance_id}-{mas_app_id}-db2u-0", + "standby": f"c-db2wh-{mas_instance_id}-{mas_app_id}-sdb-db2u-0", + }[database_role] namespace = f"db2u-{mas_instance_id}" return execInPod(core_v1_api, pod_name, namespace, command) -def db2_pod_exec_db2_get_db_cfg(core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, db_name: str, database_role='primary') -> str: +def db2_pod_exec_db2_get_db_cfg( + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + db_name: str, + database_role="primary", +) -> str: """ Execute 'db2 get db cfg' command in a DB2 pod to retrieve database configuration. @@ -95,7 +118,12 @@ def db2_pod_exec_db2_get_db_cfg(core_v1_api: client.CoreV1Api, mas_instance_id: return db2_pod_exec(core_v1_api, mas_instance_id, mas_app_id, command, database_role) -def db2_pod_exec_db2_get_dbm_cfg(core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, database_role='primary') -> str: +def db2_pod_exec_db2_get_dbm_cfg( + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +) -> str: """ Execute 'db2 get dbm cfg' command in a DB2 pod to retrieve database manager configuration. @@ -115,7 +143,12 @@ def db2_pod_exec_db2_get_dbm_cfg(core_v1_api: client.CoreV1Api, mas_instance_id: return db2_pod_exec(core_v1_api, mas_instance_id, mas_app_id, command, database_role) -def db2_pod_exec_db2set(core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, database_role='primary') -> str: +def db2_pod_exec_db2set( + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +) -> str: """ Execute 'db2set' command in a DB2 pod to retrieve registry configuration variables. @@ -169,7 +202,13 @@ def cr_pod_v_matches(cr_k: str, cr_v: str, pod_v: str) -> bool: return pod_v == cr_v -def check_db_cfgs(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, database_role='primary') -> list: +def check_db_cfgs( + db2u_instance_cr: dict, + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +) -> list: """ Runs check_db_cfg for each database in the provided Db2uInstance CR @@ -190,12 +229,21 @@ def check_db_cfgs(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_ins # Check each db cfg for cr_db in db2u_instance_cr_databases: - failures = [*failures, *check_db_cfg(cr_db, core_v1_api, mas_instance_id, mas_app_id, database_role)] + failures = [ + *failures, + *check_db_cfg(cr_db, core_v1_api, mas_instance_id, mas_app_id, database_role), + ] return failures -def check_db_cfg(db_dr: dict, core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, database_role='primary') -> list: +def check_db_cfg( + db_dr: dict, + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +) -> list: """ Check that the parameters in the provided db dict taken from the Db2uInstance CR align with those in the output of the db2 get db cfg command (i.e. the configuration that is actually active in DB2). @@ -217,7 +265,7 @@ def check_db_cfg(db_dr: dict, core_v1_api: client.CoreV1Api, mas_instance_id: st logger.info(f"Checking db cfg for {db_name}\n{H1_BREAK}") - db_cfg_cr = db_dr.get('dbConfig', None) + db_cfg_cr = db_dr.get("dbConfig", None) if db_cfg_cr is None or len(db_cfg_cr) == 0: logger.info(f"No dbConfig for db {db_name} found in CR, skipping db cfg checks for {db_name}\n") return [] @@ -227,7 +275,7 @@ def check_db_cfg(db_dr: dict, core_v1_api: client.CoreV1Api, mas_instance_id: st logger.debug(f"Running checks\n{H2_BREAK}") for cr_k, cr_v in db_cfg_cr.items(): - matches = re.search(fr"\({cr_k}\)\s=\s(.*)$", db_cfg_pod, re.MULTILINE) + matches = re.search(rf"\({cr_k}\)\s=\s(.*)$", db_cfg_pod, re.MULTILINE) if matches is None: failures.append(f"[db cfg for {db_name}] {cr_k} not found in output of db2 get db cfg command") continue @@ -244,7 +292,13 @@ def check_db_cfg(db_dr: dict, core_v1_api: client.CoreV1Api, mas_instance_id: st return failures -def check_dbm_cfg(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, database_role='primary') -> list: +def check_dbm_cfg( + db2u_instance_cr: dict, + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +) -> list: """ Check that the database manager (dbmConfig) parameters from the Db2uInstance CR align with those in the output of the db2 get dbm cfg command (i.e. the configuration that is actually active in DB2). @@ -275,7 +329,7 @@ def check_dbm_cfg(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_ins logger.debug(f"Running checks\n{H2_BREAK}") for cr_k, cr_v in dbm_cfg_cr.items(): - matches = re.search(fr"\({cr_k}\)\s=\s(.*)$", dbm_cfg_pod, re.MULTILINE) + matches = re.search(rf"\({cr_k}\)\s=\s(.*)$", dbm_cfg_pod, re.MULTILINE) if matches is None: failures.append(f"[dbm cfg] {cr_k} not found in output of db2 get dbm cfg command") continue @@ -293,7 +347,13 @@ def check_dbm_cfg(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_ins return failures -def check_reg_cfg(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_instance_id: str, mas_app_id: str, database_role='primary') -> list: +def check_reg_cfg( + db2u_instance_cr: dict, + core_v1_api: client.CoreV1Api, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +) -> list: """ Check that the registry parameters from the Db2uInstance CR align with those in the output of the db2set command (i.e. the configuration that is actually active in DB2). @@ -326,12 +386,12 @@ def check_reg_cfg(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_ins logger.debug(f"Running checks\n{H2_BREAK}") for cr_k, cr_v in reg_cfg_cr.items(): # regex ignores any trailing [O] (which indicates the param has been overridden I think) - matches = re.search(fr"{cr_k}=(.*?)(?:\s\[O\])?$", reg_cfg_pod, re.MULTILINE) - if matches is None and cr_v != '': + matches = re.search(rf"{cr_k}=(.*?)(?:\s\[O\])?$", reg_cfg_pod, re.MULTILINE) + if matches is None and cr_v != "": failures.append(f"[registry cfg] {cr_k} not found in output of db2set command") continue - pod_v = '' - if cr_v != '': + pod_v = "" + if cr_v != "": pod_v = matches.group(1) if not cr_pod_v_matches(cr_k, cr_v, pod_v): @@ -346,7 +406,12 @@ def check_reg_cfg(db2u_instance_cr: dict, core_v1_api: client.CoreV1Api, mas_ins return failures -def validate_db2_config(k8s_client: client.api_client.ApiClient, mas_instance_id: str, mas_app_id: str, database_role='primary'): +def validate_db2_config( + k8s_client: client.api_client.ApiClient, + mas_instance_id: str, + mas_app_id: str, + database_role="primary", +): """ Validate that the DB2 configuration in the Db2uInstance CR matches the actual configuration in the DB2 pods. @@ -395,9 +460,6 @@ def validate_db2_config(k8s_client: client.api_client.ApiClient, mas_instance_id logger.error(f" {reg_failure}") logger.info("Raising exception:") - raise Exception(dict( - message=f"{len(all_failures)} checks failed", - details=all_failures - )) + raise Exception(dict(message=f"{len(all_failures)} checks failed", details=all_failures)) else: logger.info("All checks passed") diff --git a/src/mas/devops/mas/__init__.py b/src/mas/devops/mas/__init__.py index 57a72a20..171966b0 100644 --- a/src/mas/devops/mas/__init__.py +++ b/src/mas/devops/mas/__init__.py @@ -2,7 +2,7 @@ verifyAppInstance, getAppsSubscriptionChannel, waitForAppReady, - getInstalledApps + getInstalledApps, ) from .suite import ( # noqa: F401 diff --git a/src/mas/devops/mas/apps.py b/src/mas/devops/mas/apps.py index 3324a1b9..7e01b365 100644 --- a/src/mas/devops/mas/apps.py +++ b/src/mas/devops/mas/apps.py @@ -12,7 +12,11 @@ import json from time import sleep from openshift.dynamic import DynamicClient -from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError +from openshift.dynamic.exceptions import ( + NotFoundError, + ResourceNotFoundError, + UnauthorizedError, +) from ..olm import getSubscription @@ -29,7 +33,7 @@ "monitor", "optimizer", "predict", - "visualinspection" + "visualinspection", ] APP_KINDS = dict( predict="PredictApp", @@ -53,7 +57,12 @@ ) -def getAppResource(dynClient: DynamicClient, instanceId: str, applicationId: str, workspaceId: str = None) -> bool: +def getAppResource( + dynClient: DynamicClient, + instanceId: str, + applicationId: str, + workspaceId: str = None, +) -> bool: """ Retrieve a MAS application or workspace custom resource. @@ -109,14 +118,15 @@ def verifyAppInstance(dynClient: DynamicClient, instanceId: str, applicationId: def waitForAppReady( - dynClient: DynamicClient, - instanceId: str, - applicationId: str, - workspaceId: str = None, - retries: int = 100, - delay: int = 600, - debugLogFunction=logger.debug, - infoLogFunction=logger.info) -> bool: + dynClient: DynamicClient, + instanceId: str, + applicationId: str, + workspaceId: str = None, + retries: int = 100, + delay: int = 600, + debugLogFunction=logger.debug, + infoLogFunction=logger.info, +) -> bool: """ Wait for a MAS application or workspace to reach ready state. diff --git a/src/mas/devops/mas/suite.py b/src/mas/devops/mas/suite.py index 0042257a..b9de0e89 100644 --- a/src/mas/devops/mas/suite.py +++ b/src/mas/devops/mas/suite.py @@ -15,7 +15,11 @@ from types import SimpleNamespace from kubernetes.dynamic.resource import ResourceInstance from openshift.dynamic import DynamicClient -from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError +from openshift.dynamic.exceptions import ( + NotFoundError, + ResourceNotFoundError, + UnauthorizedError, +) from jinja2 import Environment, FileSystemLoader from ..ocp import getStorageClasses, listInstances @@ -42,7 +46,10 @@ def isAirgapInstall(dynClient: DynamicClient, checkICSP: bool = False) -> bool: """ if checkICSP: try: - ICSPApi = dynClient.resources.get(api_version="operator.openshift.io/v1alpha1", kind="ImageContentSourcePolicy") + ICSPApi = dynClient.resources.get( + api_version="operator.openshift.io/v1alpha1", + kind="ImageContentSourcePolicy", + ) ICSPApi.get(name="ibm-mas-and-dependencies") return True except NotFoundError: @@ -73,12 +80,7 @@ def getDefaultStorageClasses(dynClient: DynamicClient) -> SimpleNamespace: - rwx (str): Storage class name for RWX volumes All attributes are None if no recognized provider is found. """ - result = SimpleNamespace( - provider=None, - providerName=None, - rwo=None, - rwx=None - ) + result = SimpleNamespace(provider=None, providerName=None, rwo=None, rwx=None) # Iterate through storage classes until we find one that we recognize # We make an assumption that if one of the paired classes if available, both will be @@ -90,13 +92,19 @@ def getDefaultStorageClasses(dynClient: DynamicClient) -> SimpleNamespace: result.rwo = "ibmc-block-gold" result.rwx = "ibmc-file-gold-gid" break - elif storageClass.metadata.name in ["ocs-storagecluster-ceph-rbd", "ocs-storagecluster-cephfs"]: + elif storageClass.metadata.name in [ + "ocs-storagecluster-ceph-rbd", + "ocs-storagecluster-cephfs", + ]: result.provider = "ocs" result.providerName = "OpenShift Container Storage" result.rwo = "ocs-storagecluster-ceph-rbd" result.rwx = "ocs-storagecluster-cephfs" break - elif storageClass.metadata.name in ["ocs-external-storagecluster-ceph-rbd", "ocs-external-storagecluster-cephfs"]: + elif storageClass.metadata.name in [ + "ocs-external-storagecluster-ceph-rbd", + "ocs-external-storagecluster-cephfs", + ]: result.provider = "ocs-external" result.providerName = "OpenShift Container Storage (External)" result.rwo = "ocs-external-storagecluster-ceph-rbd" @@ -153,7 +161,10 @@ def getCurrentCatalog(dynClient: DynamicClient) -> dict: catalogDisplayName = catalog.spec.displayName catalogImage = catalog.spec.image - m = re.match(r".+(?Pv[89]-(?P[0-9]+)-(amd64|s390x|ppc64le))", catalogDisplayName) + m = re.match( + r".+(?Pv[89]-(?P[0-9]+)-(amd64|s390x|ppc64le))", + catalogDisplayName, + ) if m: # catalogId = v9-yymmdd-amd64 # catalogVersion = yymmdd @@ -262,7 +273,15 @@ def getMasChannel(dynClient: DynamicClient, instanceId: str) -> str: return masSubscription.spec.channel -def updateIBMEntitlementKey(dynClient: DynamicClient, namespace: str, icrUsername: str, icrPassword: str, artifactoryUsername: str = None, artifactoryPassword: str = None, secretName: str = "ibm-entitlement") -> ResourceInstance: +def updateIBMEntitlementKey( + dynClient: DynamicClient, + namespace: str, + icrUsername: str, + icrPassword: str, + artifactoryUsername: str = None, + artifactoryPassword: str = None, + secretName: str = "ibm-entitlement", +) -> ResourceInstance: """ Create or update the IBM Entitlement secret for accessing IBM container registries. @@ -291,7 +310,7 @@ def updateIBMEntitlementKey(dynClient: DynamicClient, namespace: str, icrUsernam templateDir = path.join(path.abspath(path.dirname(__file__)), "..", "templates") env = Environment( loader=FileSystemLoader(searchpath=templateDir), - extensions=["jinja2_base64_filters.Base64Filters"] + extensions=["jinja2_base64_filters.Base64Filters"], ) contentTemplate = env.get_template("ibm-entitlement-dockerconfig.json.j2") @@ -299,15 +318,11 @@ def updateIBMEntitlementKey(dynClient: DynamicClient, namespace: str, icrUsernam artifactory_username=artifactoryUsername, artifactory_token=artifactoryPassword, icr_username=icrUsername, - icr_password=icrPassword + icr_password=icrPassword, ) template = env.get_template("ibm-entitlement-secret.yml.j2") - renderedTemplate = template.render( - name=secretName, - namespace=namespace, - docker_config=dockerConfig - ) + renderedTemplate = template.render(name=secretName, namespace=namespace, docker_config=dockerConfig) secret = yaml.safe_load(renderedTemplate) secretsAPI = dynClient.resources.get(api_version="v1", kind="Secret") @@ -337,7 +352,7 @@ def getMasPublicClusterIssuer(dynClient: DynamicClient, instanceId: str) -> str suite = suitesAPI.get(name=instanceId, namespace=f"mas-{instanceId}-core") # Check if spec.certificateIssuer.name exists - if hasattr(suite, 'spec') and hasattr(suite.spec, 'certificateIssuer') and hasattr(suite.spec.certificateIssuer, 'name'): + if hasattr(suite, "spec") and hasattr(suite.spec, "certificateIssuer") and hasattr(suite.spec.certificateIssuer, "name"): issuerName = suite.spec.certificateIssuer.name logger.debug(f"Found custom certificate issuer: {issuerName}") return issuerName @@ -426,13 +441,15 @@ def getPermissionMode(dynClient: DynamicClient, instanceId: str) -> str | None: f"mas:{instanceId}:core:suite:arcgis:essential", f"mas:{instanceId}:core:suite:facilities:essential", f"mas:{instanceId}:core:suite:optimizer:essential", - f"mas:{instanceId}:core:suite:visualinspection:essential" + f"mas:{instanceId}:core:suite:visualinspection:essential", ] for essentialRoleName in essentialRolePatterns: try: roleAPI.get(name=essentialRoleName, namespace=coreNamespace) - logger.info(f"Found essential Role '{essentialRoleName}' in namespace '{coreNamespace}' with no non-essential roles - permission mode is 'minimal'") + logger.info( + f"Found essential Role '{essentialRoleName}' in namespace '{coreNamespace}' with no non-essential roles - permission mode is 'minimal'" + ) return "minimal" except NotFoundError: continue diff --git a/src/mas/devops/ocp.py b/src/mas/devops/ocp.py index b846a29e..ed247e23 100644 --- a/src/mas/devops/ocp.py +++ b/src/mas/devops/ocp.py @@ -54,22 +54,11 @@ def connect(server: str, token: str, skipVerify: bool = False) -> bool: conf.view() logger.debug(f"Starting KubeConfig context: {conf.current_context()}") - conf.set_credentials( - name='my-credentials', - token=token - ) - conf.set_cluster( - name='my-cluster', - server=server, - insecure_skip_tls_verify=skipVerify - ) - conf.set_context( - name='my-context', - cluster='my-cluster', - user='my-credentials' - ) + conf.set_credentials(name="my-credentials", token=token) + conf.set_cluster(name="my-cluster", server=server, insecure_skip_tls_verify=skipVerify) + conf.set_context(name="my-context", cluster="my-cluster", user="my-credentials") - conf.use_context('my-context') + conf.use_context("my-context") conf.view() logger.info(f"KubeConfig context changed to {conf.current_context()}") return True @@ -167,20 +156,16 @@ def createNamespace(dynClient: DynamicClient, namespace: str, kyvernoLabel: str namespaceAPI.patch( name=namespace, body=body, - content_type="application/merge-patch+json" + content_type="application/merge-patch+json", ) except NotFoundError: nsObj = { "apiVersion": "v1", "kind": "Namespace", - "metadata": { - "name": namespace - } + "metadata": {"name": namespace}, } if kyvernoLabel is not None: - nsObj["metadata"]["labels"] = { - "ibm.com/kyverno": kyvernoLabel - } + nsObj["metadata"]["labels"] = {"ibm.com/kyverno": kyvernoLabel} namespaceAPI.create(body=nsObj) logger.debug(f"Created namespace {namespace}") return True @@ -309,7 +294,7 @@ def getNodes(dynClient: DynamicClient) -> dict: list: List of node resources as dictionaries """ nodesAPI = dynClient.resources.get(api_version="v1", kind="Node") - nodes = nodesAPI.get().to_dict()['items'] + nodes = nodesAPI.get().to_dict()["items"] return nodes @@ -394,7 +379,7 @@ def getStorageClassVolumeBindingMode(dynClient: DynamicClient, storageClassName: """ try: storageClass = getStorageClass(dynClient, storageClassName) - if storageClass and hasattr(storageClass, 'volumeBindingMode'): + if storageClass and hasattr(storageClass, "volumeBindingMode"): return storageClass.volumeBindingMode # Default to Immediate if not specified (Kubernetes default) logger.debug(f"Storage class {storageClassName} does not have volumeBindingMode set, defaulting to 'Immediate'") @@ -442,7 +427,13 @@ def crdExists(dynClient: DynamicClient, crdName: str) -> bool: return False -def getCR(dynClient: DynamicClient, cr_api_version: str, cr_kind: str, cr_name: str, namespace: str = None) -> dict: +def getCR( + dynClient: DynamicClient, + cr_api_version: str, + cr_kind: str, + cr_name: str, + namespace: str = None, +) -> dict: """ Get a Custom Resource """ @@ -483,10 +474,10 @@ def apply_resource(dynClient: DynamicClient, resource_yaml: str, namespace: str) If it does not exist, it will be created. """ resource_dict = yaml.safe_load(resource_yaml) - kind = resource_dict['kind'] - api_version = resource_dict['apiVersion'] - metadata = resource_dict['metadata'] - name = metadata['name'] + kind = resource_dict["kind"] + api_version = resource_dict["apiVersion"] + metadata = resource_dict["metadata"] + name = metadata["name"] try: resource = dynClient.resources.get(api_version=api_version, kind=kind) @@ -518,7 +509,7 @@ def listInstances(dynClient: DynamicClient, apiVersion: str, kind: str) -> list: NotFoundError: If the custom resource type is not found """ api = dynClient.resources.get(api_version=apiVersion, kind=kind) - instances = api.get().to_dict()['items'] + instances = api.get().to_dict()["items"] if len(instances) > 0: logger.info(f"There are {len(instances)} {kind} instances installed on this cluster:") for instance in instances: @@ -579,7 +570,13 @@ def waitForPVC(dynClient: DynamicClient, namespace: str, pvcName: str) -> bool: # Assisted by WCA@IBM # Latest GenAI contribution: ibm/granite-8b-code-instruct -def execInPod(core_v1_api: client.CoreV1Api, pod_name: str, namespace, command: list, timeout: int = 60) -> str: +def execInPod( + core_v1_api: client.CoreV1Api, + pod_name: str, + namespace, + command: list, + timeout: int = 60, +) -> str: """ Executes a command in a Kubernetes pod and returns the standard output. If running this function from inside a pod (i.e. config.load_incluster_config()), @@ -627,7 +624,9 @@ def execInPod(core_v1_api: client.CoreV1Api, pod_name: str, namespace, command: if err.get("status") == "Failure": raise Exception(f"Failed to execute {command} on {pod_name} in namespace {namespace}: {err.get('message')}. stdout: {stdout}, stderr: {stderr}") - logger.debug(f"stdout: \n----------------------------------------------------------------\n{stdout}\n----------------------------------------------------------------\n") + logger.debug( + f"stdout: \n----------------------------------------------------------------\n{stdout}\n----------------------------------------------------------------\n" + ) return stdout @@ -661,11 +660,11 @@ def updateGlobalPullSecret(dynClient: DynamicClient, registryUrl: str, username: secretDict = pullSecret.to_dict() # Decode the existing dockerconfigjson - dockerConfigJson = secretDict['data'].get(".dockerconfigjson", "") - dockerConfig = json.loads(base64.b64decode(dockerConfigJson).decode('utf-8')) + dockerConfigJson = secretDict["data"].get(".dockerconfigjson", "") + dockerConfig = json.loads(base64.b64decode(dockerConfigJson).decode("utf-8")) # Create auth string (username:password base64 encoded) - authString = base64.b64encode(f"{username}:{password}".encode('utf-8')).decode('utf-8') + authString = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") # Add or update the registry credentials if "auths" not in dockerConfig: @@ -675,14 +674,14 @@ def updateGlobalPullSecret(dynClient: DynamicClient, registryUrl: str, username: "username": username, "password": password, "email": username, - "auth": authString + "auth": authString, } # Encode back to base64 - updatedDockerConfig = base64.b64encode(json.dumps(dockerConfig).encode('utf-8')).decode('utf-8') + updatedDockerConfig = base64.b64encode(json.dumps(dockerConfig).encode("utf-8")).decode("utf-8") # Update the secret dict - secretDict['data'][".dockerconfigjson"] = updatedDockerConfig + secretDict["data"][".dockerconfigjson"] = updatedDockerConfig # Apply the updated secret updatedSecret = secretsAPI.apply(body=secretDict, namespace="openshift-config") @@ -693,7 +692,7 @@ def updateGlobalPullSecret(dynClient: DynamicClient, registryUrl: str, username: "name": updatedSecret.metadata.name, "namespace": updatedSecret.metadata.namespace, "registry": registryUrl, - "changed": True + "changed": True, } @@ -717,23 +716,17 @@ def configureIngressForPathBasedRouting(dynClient: DynamicClient, ingressControl logger.info(f"Configuring IngressController '{ingressControllerName}' for path-based routing") try: - ingressControllerAPI = dynClient.resources.get( - api_version="operator.openshift.io/v1", - kind="IngressController" - ) + ingressControllerAPI = dynClient.resources.get(api_version="operator.openshift.io/v1", kind="IngressController") try: - ingressController = ingressControllerAPI.get( - name=ingressControllerName, - namespace="openshift-ingress-operator" - ) + ingressController = ingressControllerAPI.get(name=ingressControllerName, namespace="openshift-ingress-operator") except NotFoundError: logger.error(f"IngressController '{ingressControllerName}' not found in namespace 'openshift-ingress-operator'") return False currentPolicy = None - if hasattr(ingressController, 'spec') and hasattr(ingressController.spec, 'routeAdmission'): - if hasattr(ingressController.spec.routeAdmission, 'namespaceOwnership'): + if hasattr(ingressController, "spec") and hasattr(ingressController.spec, "routeAdmission"): + if hasattr(ingressController.spec.routeAdmission, "namespaceOwnership"): currentPolicy = ingressController.spec.routeAdmission.namespaceOwnership logger.debug(f"Current namespaceOwnership policy: {currentPolicy if currentPolicy else 'Not set'}") @@ -744,19 +737,13 @@ def configureIngressForPathBasedRouting(dynClient: DynamicClient, ingressControl logger.info(f"Patching IngressController '{ingressControllerName}' to enable InterNamespaceAllowed") - patch = { - "spec": { - "routeAdmission": { - "namespaceOwnership": "InterNamespaceAllowed" - } - } - } + patch = {"spec": {"routeAdmission": {"namespaceOwnership": "InterNamespaceAllowed"}}} ingressControllerAPI.patch( body=patch, name=ingressControllerName, namespace="openshift-ingress-operator", - content_type="application/merge-patch+json" + content_type="application/merge-patch+json", ) maxRetries = 5 @@ -765,12 +752,14 @@ def configureIngressForPathBasedRouting(dynClient: DynamicClient, ingressControl for attempt in range(maxRetries): sleep(retryDelay) try: - updatedController = ingressControllerAPI.get( - name=ingressControllerName, - namespace="openshift-ingress-operator" - ) - - if (hasattr(updatedController, 'spec') and hasattr(updatedController.spec, 'routeAdmission') and hasattr(updatedController.spec.routeAdmission, 'namespaceOwnership') and updatedController.spec.routeAdmission.namespaceOwnership == "InterNamespaceAllowed"): + updatedController = ingressControllerAPI.get(name=ingressControllerName, namespace="openshift-ingress-operator") + + if ( + hasattr(updatedController, "spec") + and hasattr(updatedController.spec, "routeAdmission") + and hasattr(updatedController.spec.routeAdmission, "namespaceOwnership") + and updatedController.spec.routeAdmission.namespaceOwnership == "InterNamespaceAllowed" + ): logger.info(f"Successfully configured IngressController '{ingressControllerName}' for path-based routing") return True diff --git a/src/mas/devops/olm.py b/src/mas/devops/olm.py index 1cea83b6..4847bb3d 100644 --- a/src/mas/devops/olm.py +++ b/src/mas/devops/olm.py @@ -28,7 +28,11 @@ class OLMException(Exception): pass -def getPackageManifest(dynClient: DynamicClient, packageName: str, catalogSourceNamespace: str = "openshift-marketplace"): +def getPackageManifest( + dynClient: DynamicClient, + packageName: str, + catalogSourceNamespace: str = "openshift-marketplace", +): """ Get the PackageManifest for an operator package. @@ -48,14 +52,21 @@ def getPackageManifest(dynClient: DynamicClient, packageName: str, catalogSource packagemanifestAPI = dynClient.resources.get(api_version="packages.operators.coreos.com/v1", kind="PackageManifest") try: manifestResource = packagemanifestAPI.get(name=packageName, namespace=catalogSourceNamespace) - logger.info(f"Package Manifest Details: {catalogSourceNamespace}:{packageName} - Package is available from {manifestResource.status.catalogSource} (default channel is {manifestResource.status.defaultChannel})") + logger.info( + f"Package Manifest Details: {catalogSourceNamespace}:{packageName} - Package is available from {manifestResource.status.catalogSource} (default channel is {manifestResource.status.defaultChannel})" + ) except NotFoundError: logger.info(f"Package Manifest Details: {catalogSourceNamespace}:{packageName} - Package is not available") manifestResource = None return manifestResource -def ensureOperatorGroupExists(dynClient: DynamicClient, env: Environment, namespace: str, installMode: str = "OwnNamespace"): +def ensureOperatorGroupExists( + dynClient: DynamicClient, + env: Environment, + namespace: str, + installMode: str = "OwnNamespace", +): """ Ensure an OperatorGroup exists in the specified namespace. @@ -78,11 +89,7 @@ def ensureOperatorGroupExists(dynClient: DynamicClient, env: Environment, namesp if len(operatorGroupList.items) == 0: logger.debug(f"Creating new OperatorGroup in namespace {namespace}") template = env.get_template("operatorgroup.yml.j2") - renderedTemplate = template.render( - name="operatorgroup", - namespace=namespace, - installMode=installMode - ) + renderedTemplate = template.render(name="operatorgroup", namespace=namespace, installMode=installMode) operatorGroup = yaml.safe_load(renderedTemplate) operatorGroupsAPI.apply(body=operatorGroup, namespace=namespace) else: @@ -118,7 +125,18 @@ def getSubscription(dynClient: DynamicClient, namespace: str, packageName: str): return subscriptions.items[0] -def applySubscription(dynClient: DynamicClient, namespace: str, packageName: str, packageChannel: Optional[str] = None, catalogSource: Optional[str] = None, catalogSourceNamespace: str = "openshift-marketplace", config: Optional[dict] = None, installMode: str = "OwnNamespace", installPlanApproval: Optional[str] = None, startingCSV: Optional[str] = None): +def applySubscription( + dynClient: DynamicClient, + namespace: str, + packageName: str, + packageChannel: Optional[str] = None, + catalogSource: Optional[str] = None, + catalogSourceNamespace: str = "openshift-marketplace", + config: Optional[dict] = None, + installMode: str = "OwnNamespace", + installPlanApproval: Optional[str] = None, + startingCSV: Optional[str] = None, +): """ Create or update an operator subscription in a namespace. @@ -157,9 +175,7 @@ def applySubscription(dynClient: DynamicClient, namespace: str, packageName: str labelSelector = f"operators.coreos.com/{packageName}.{namespace}" templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) if packageChannel is None or catalogSource is None: logger.debug("Getting PackageManifest to determine defaults") @@ -203,7 +219,7 @@ def applySubscription(dynClient: DynamicClient, namespace: str, packageName: str catalog_name=catalogSource, catalog_namespace=catalogSourceNamespace, install_plan_approval=installPlanApproval, - starting_csv=startingCSV + starting_csv=startingCSV, ) subscription = yaml.safe_load(renderedTemplate) @@ -258,11 +274,8 @@ def applySubscription(dynClient: DynamicClient, namespace: str, packageName: str allInstallPlans = installPlanAPI.get(namespace=namespace) for plan in allInstallPlans.items: # Check if this InstallPlan is owned by our subscription - owner_refs = getattr(plan.metadata, 'ownerReferences', []) - is_owned_by_subscription = any( - ref.kind == "Subscription" and ref.name == name - for ref in owner_refs - ) + owner_refs = getattr(plan.metadata, "ownerReferences", []) + is_owned_by_subscription = any(ref.kind == "Subscription" and ref.name == name for ref in owner_refs) if is_owned_by_subscription: csvNames = getattr(plan.spec, "clusterServiceVersionNames", []) @@ -309,7 +322,7 @@ def applySubscription(dynClient: DynamicClient, namespace: str, packageName: str body=installPlanResource, name=installPlanName, namespace=namespace, - content_type="application/merge-patch+json" + content_type="application/merge-patch+json", ) approved_manual_install = True logger.info(f"InstallPlan {installPlanName} approved successfully") @@ -339,7 +352,9 @@ def applySubscription(dynClient: DynamicClient, namespace: str, packageName: str logger.debug(f"Subscription {name} in {namespace} reached state: {state} with installedCSV: {installedCSV}") return subscriptionResource else: - logger.debug(f"Subscription {name} in {namespace} state is {state} but installedCSV ({installedCSV}) does not match startingCSV ({startingCSV}), retrying...") + logger.debug( + f"Subscription {name} in {namespace} state is {state} but installedCSV ({installedCSV}) does not match startingCSV ({startingCSV}), retrying..." + ) logger.debug(f"Subscription {name} in {namespace} not ready yet (state = {state}), retrying...") sleep(30) diff --git a/src/mas/devops/pre_install.py b/src/mas/devops/pre_install.py index f181d92a..9f0e0f63 100644 --- a/src/mas/devops/pre_install.py +++ b/src/mas/devops/pre_install.py @@ -36,7 +36,7 @@ def _validate_selected_apps(selectedApps: list[str] | None) -> set[str]: "monitor", "optimizer", "predict", - "visualinspection" + "visualinspection", } validatedApps = set() @@ -60,7 +60,7 @@ def _get_selected_operator_dirs(selectedApps: set[str]) -> set[str]: "monitor": "ibm-mas-monitor", "optimizer": "ibm-mas-optimizer", "predict": "ibm-mas-predict", - "visualinspection": "ibm-mas-visualinspection" + "visualinspection": "ibm-mas-visualinspection", } return {appToOperatorDir[app] for app in selectedApps} @@ -88,17 +88,14 @@ def _collect_preinstall_mas_rbac_files_from_source( sourceOperatorsRoot: str, masVersion: str, adminMode: str, - operatorNames: set[str] | None = None + operatorNames: set[str] | None = None, ) -> list[str]: if not path.isdir(sourceOperatorsRoot): logger.debug(f"Skipping missing RBAC source root {sourceOperatorsRoot}") return [] if operatorNames is None: - operatorNames = { - operatorName for operatorName in listdir(sourceOperatorsRoot) - if path.isdir(path.join(sourceOperatorsRoot, operatorName)) - } + operatorNames = {operatorName for operatorName in listdir(sourceOperatorsRoot) if path.isdir(path.join(sourceOperatorsRoot, operatorName))} manifestFiles = [] for operatorName in sorted(operatorNames): @@ -123,12 +120,7 @@ def _collect_preinstall_mas_rbac_files_from_source( return manifestFiles -def _discover_preinstall_mas_rbac_files( - rbacRootDir: str | None, - masVersion: str, - adminMode: str, - selectedApps: set[str] -) -> list[str]: +def _discover_preinstall_mas_rbac_files(rbacRootDir: str | None, masVersion: str, adminMode: str, selectedApps: set[str]) -> list[str]: if not rbacRootDir: rbacRootDir = DEFAULT_PREINSTALL_MAS_RBAC_ROOT @@ -137,12 +129,9 @@ def _discover_preinstall_mas_rbac_files( sourceRoots = [ ( path.join(rbacRootDir, "maximo-operator-catalog", "operators"), - selectedOperatorDirs + selectedOperatorDirs, ), - ( - path.join(rbacRootDir, "openshift-platform", "operators"), - None - ) + (path.join(rbacRootDir, "openshift-platform", "operators"), None), ] manifestFiles = [] @@ -152,7 +141,7 @@ def _discover_preinstall_mas_rbac_files( sourceOperatorsRoot=sourceRoot, masVersion=masVersion, adminMode=adminMode, - operatorNames=operatorNames + operatorNames=operatorNames, ) ) @@ -175,7 +164,7 @@ def _get_preinstall_mas_rbac_namespaces(masInstanceId: str, adminMode: str, sele "monitor": f"mas-{masInstanceId}-monitor", "optimizer": f"mas-{masInstanceId}-optimizer", "predict": f"mas-{masInstanceId}-predict", - "visualinspection": f"mas-{masInstanceId}-visualinspection" + "visualinspection": f"mas-{masInstanceId}-visualinspection", } for app in selectedApps: @@ -190,17 +179,12 @@ def _check_self_subject_access( verb: str, resource: str, group: str = "rbac.authorization.k8s.io", - namespace: str | None = None + namespace: str | None = None, ) -> bool: authAPI = k8s_client.AuthorizationV1Api(dynClient.client) review = k8s_client.V1SelfSubjectAccessReview( spec=k8s_client.V1SelfSubjectAccessReviewSpec( - resource_attributes=k8s_client.V1ResourceAttributes( - namespace=namespace, - verb=verb, - resource=resource, - group=group - ) + resource_attributes=k8s_client.V1ResourceAttributes(namespace=namespace, verb=verb, resource=resource, group=group) ) ) result = authAPI.create_self_subject_access_review(body=review) @@ -218,10 +202,7 @@ def buildClusterAdminPermissionMatrix() -> list[dict[str, str]]: ] -def permissionCheckForRBAC( - dynClient: DynamicClient, - checks: list[dict[str, str]] | None = None -) -> list[dict[str, str | bool]]: +def permissionCheckForRBAC(dynClient: DynamicClient, checks: list[dict[str, str]] | None = None) -> list[dict[str, str | bool]]: if checks is None: checks = buildClusterAdminPermissionMatrix() @@ -238,14 +219,14 @@ def permissionCheckForRBAC( verb=verb, resource=resource, group=group, - namespace=namespace + namespace=namespace, ) result: dict[str, str | bool] = { "verb": verb, "resource": resource, "group": group, - "allowed": allowed + "allowed": allowed, } if namespace is not None: @@ -262,7 +243,7 @@ def applyPreInstallMASRBAC( masInstanceId: str, adminMode: str, selectedApps: list[str] | None = None, - rbacRootDir: str | None = None + rbacRootDir: str | None = None, ) -> None: if not rbacRootDir: rbacRootDir = DEFAULT_PREINSTALL_MAS_RBAC_ROOT @@ -287,7 +268,7 @@ def applyPreInstallMASRBAC( rbacRootDir=rbacRootDir, masVersion=masVersion, adminMode=adminMode, - selectedApps=validatedApps + selectedApps=validatedApps, ) logger.info( @@ -302,21 +283,17 @@ def applyPreInstallMASRBAC( return namespaceAPI = dynClient.resources.get(api_version="v1", kind="Namespace") - requiredNamespaces = _get_preinstall_mas_rbac_namespaces( - masInstanceId=masInstanceId, - adminMode=adminMode, - selectedApps=validatedApps - ) + requiredNamespaces = _get_preinstall_mas_rbac_namespaces(masInstanceId=masInstanceId, adminMode=adminMode, selectedApps=validatedApps) for namespace in sorted(requiredNamespaces): logger.info(f"Ensuring namespace exists for pre-install MAS RBAC: {namespace}") - namespaceAPI.apply(body={ - "apiVersion": "v1", - "kind": "Namespace", - "metadata": { - "name": namespace + namespaceAPI.apply( + body={ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": {"name": namespace}, } - }) + ) env = Environment() appliedResourceCount = 0 @@ -338,14 +315,9 @@ def applyPreInstallMASRBAC( resourceNamespace = metadata.get("namespace") if kind in {"Role", "RoleBinding"} and not resourceNamespace: - raise ValueError( - f"Namespaced RBAC resource {kind}/{resourceName} from {manifestFile} is missing metadata.namespace" - ) + raise ValueError(f"Namespaced RBAC resource {kind}/{resourceName} from {manifestFile} is missing metadata.namespace") - logger.debug( - f"Applying {kind} {resourceName} " - f"(apiVersion={apiVersion}, namespace={resourceNamespace})" - ) + logger.debug(f"Applying {kind} {resourceName} " f"(apiVersion={apiVersion}, namespace={resourceNamespace})") resourceAPI = dynClient.resources.get(api_version=apiVersion, kind=kind) if resourceNamespace: @@ -355,7 +327,4 @@ def applyPreInstallMASRBAC( appliedResourceCount += 1 - logger.info( - f"Pre-install MAS RBAC apply completed: processedFiles={len(manifestFiles)}, " - f"appliedResources={appliedResourceCount}" - ) + logger.info(f"Pre-install MAS RBAC apply completed: processedFiles={len(manifestFiles)}, " f"appliedResources={appliedResourceCount}") diff --git a/src/mas/devops/restore.py b/src/mas/devops/restore.py index d427e628..9bda5ddf 100644 --- a/src/mas/devops/restore.py +++ b/src/mas/devops/restore.py @@ -26,7 +26,7 @@ def loadYamlFile(file_path: str): dict: Parsed YAML content or None if error """ try: - with open(file_path, 'r') as yaml_file: + with open(file_path, "r") as yaml_file: content = yaml.safe_load(yaml_file) return content except Exception as e: @@ -55,16 +55,16 @@ def restoreResource(dynClient: DynamicClient, resource_data: dict, namespace=Non """ try: # Extract resource metadata - kind = resource_data.get('kind') - api_version = resource_data.get('apiVersion') - metadata = resource_data.get('metadata', {}) - resource_name = metadata.get('name') - resource_namespace = namespace or metadata.get('namespace') + kind = resource_data.get("kind") + api_version = resource_data.get("apiVersion") + metadata = resource_data.get("metadata", {}) + resource_name = metadata.get("name") + resource_namespace = namespace or metadata.get("namespace") if not kind or not api_version or not resource_name: error_msg = "Resource missing required fields (kind, apiVersion, or name)" logger.error(error_msg) - return (False, resource_name or 'unknown', error_msg) + return (False, resource_name or "unknown", error_msg) # Get the resource API resourceAPI = dynClient.resources.get(api_version=api_version, kind=kind) @@ -92,9 +92,18 @@ def restoreResource(dynClient: DynamicClient, resource_data: dict, namespace=Non logger.info(f"Patching existing {kind} '{resource_name}' in {scope_desc}") if resource_namespace: - resourceAPI.patch(body=resource_data, name=resource_name, namespace=resource_namespace, content_type='application/merge-patch+json') + resourceAPI.patch( + body=resource_data, + name=resource_name, + namespace=resource_namespace, + content_type="application/merge-patch+json", + ) else: - resourceAPI.patch(body=resource_data, name=resource_name, content_type='application/merge-patch+json') + resourceAPI.patch( + body=resource_data, + name=resource_name, + content_type="application/merge-patch+json", + ) logger.info(f"Successfully patched {kind} '{resource_name}' in {scope_desc}") return (True, resource_name, "updated") else: @@ -119,4 +128,8 @@ def restoreResource(dynClient: DynamicClient, resource_data: dict, namespace=Non except Exception as e: error_msg = f"Error restoring resource: {e}" logger.error(error_msg) - return (False, resource_data.get('metadata', {}).get('name', 'unknown'), error_msg) + return ( + False, + resource_data.get("metadata", {}).get("name", "unknown"), + error_msg, + ) diff --git a/src/mas/devops/saas/job_cleaner.py b/src/mas/devops/saas/job_cleaner.py index ff6c3833..3a6b0211 100644 --- a/src/mas/devops/saas/job_cleaner.py +++ b/src/mas/devops/saas/job_cleaner.py @@ -68,11 +68,7 @@ def _get_all_cleanup_groups(self, label: str, limit: int): _continue = None while True: - jobs_page = self.batch_v1_api.list_job_for_all_namespaces( - label_selector=label, - limit=limit, - _continue=_continue - ) + jobs_page = self.batch_v1_api.list_job_for_all_namespaces(label_selector=label, limit=limit, _continue=_continue) _continue = jobs_page.metadata._continue for job in jobs_page.items: @@ -108,7 +104,7 @@ def _get_all_jobs(self, namespace: str, group_id: str, label: str, limit: int): namespace, label_selector=f"{label}={group_id}", limit=limit, - _continue=_continue + _continue=_continue, ) job_items_iters.append(jobs_page.items) _continue = jobs_page.metadata._continue @@ -161,7 +157,7 @@ def cleanup_jobs(self, label: str, limit: int, dry_run: bool): # we can deal with each one separately; we only have to load the job resources for that particular group into memory at once # (we have to load into memory in order to guarantee the jobs are sorted by creation_date) i = 0 - for (namespace, group_id) in cleanup_groups: + for namespace, group_id in cleanup_groups: self.logger.info("") self.logger.info(f"{i}) {group_id} {namespace}") @@ -172,7 +168,7 @@ def cleanup_jobs(self, label: str, limit: int, dry_run: bool): jobs_sorted = sorted( jobs, key=lambda group_job: group_job.metadata.creation_timestamp, - reverse=True + reverse=True, ) if len(jobs_sorted) == 0: @@ -188,7 +184,12 @@ def cleanup_jobs(self, label: str, limit: int, dry_run: bool): first = False else: try: - self.batch_v1_api.delete_namespaced_job(name, namespace, dry_run=dry_run_param, propagation_policy="Foreground") + self.batch_v1_api.delete_namespaced_job( + name, + namespace, + dry_run=dry_run_param, + propagation_policy="Foreground", + ) result = "SUCCESS" except client.rest.ApiException as e: result = f"FAILED: {e}" diff --git a/src/mas/devops/slack.py b/src/mas/devops/slack.py index 27678926..6f77a3d1 100644 --- a/src/mas/devops/slack.py +++ b/src/mas/devops/slack.py @@ -108,7 +108,13 @@ def postMessageBlocks(cls, channelList: str | list[str], messageBlocks: list, th return responses if len(responses) > 1 else responses[0] - def postMessageText(cls, channelList: str | list[str], message: str, attachments=None, threadId: str = None) -> SlackResponse | list[SlackResponse]: + def postMessageText( + cls, + channelList: str | list[str], + message: str, + attachments=None, + threadId: str = None, + ) -> SlackResponse | list[SlackResponse]: """ Post a plain text message to one or more Slack channels. @@ -166,7 +172,11 @@ def postMessageText(cls, channelList: str | list[str], message: str, attachments return responses if len(responses) > 1 else responses[0] def createMessagePermalink( - cls, slackResponse: SlackResponse = None, channelId: str = None, messageTimestamp: str = None, domain: str = "ibm-mas" + cls, + slackResponse: SlackResponse = None, + channelId: str = None, + messageTimestamp: str = None, + domain: str = "ibm-mas", ) -> str: """ Create a permanent link to a Slack message. @@ -234,7 +244,10 @@ def buildHeader(cls, title: str) -> dict: Returns: dict: Slack block kit header element """ - return {"type": "header", "text": {"type": "plain_text", "text": title, "emoji": True}} + return { + "type": "header", + "text": {"type": "plain_text", "text": title, "emoji": True}, + } def buildSection(cls, text: str) -> dict: """ @@ -271,6 +284,7 @@ def buildDivider(cls) -> dict: Returns: dict: Slack block kit divider element """ + def createThreadConfigMap(cls, namespace: str, instanceId: str, pipelineRunName: str) -> bool: """ Create a ConfigMap to store Slack thread information for a pipeline run. @@ -295,15 +309,12 @@ def createThreadConfigMap(cls, namespace: str, instanceId: str, pipelineRunName: instance_identifier = instanceId if instanceId else "update" configmap_name = f"slack-thread-{instance_identifier}-{pipelineRunName}" configmap = client.V1ConfigMap( - metadata=client.V1ObjectMeta( - name=configmap_name, - namespace=namespace - ), + metadata=client.V1ObjectMeta(name=configmap_name, namespace=namespace), data={ "pipelineName": pipelineRunName, "instanceId": instanceId, - "startTime": datetime.now(timezone.utc) - } + "startTime": datetime.now(timezone.utc), + }, ) v1.create_namespaced_config_map(namespace=namespace, body=configmap) logger.info(f"Created ConfigMap {configmap_name} in namespace {namespace}") diff --git a/src/mas/devops/sls.py b/src/mas/devops/sls.py index e204bc8d..d1393b94 100644 --- a/src/mas/devops/sls.py +++ b/src/mas/devops/sls.py @@ -10,7 +10,11 @@ import logging from openshift.dynamic import DynamicClient -from openshift.dynamic.exceptions import NotFoundError, ResourceNotFoundError, UnauthorizedError +from openshift.dynamic.exceptions import ( + NotFoundError, + ResourceNotFoundError, + UnauthorizedError, +) logger = logging.getLogger(__name__) @@ -36,7 +40,7 @@ def listSLSInstances(dynClient: DynamicClient) -> list: """ try: slsAPI = dynClient.resources.get(api_version="sls.ibm.com/v1", kind="LicenseService") - return slsAPI.get().to_dict()['items'] + return slsAPI.get().to_dict()["items"] except NotFoundError: logger.info("There are no SLS instances installed on this cluster") return [] @@ -74,7 +78,7 @@ def findSLSByNamespace(namespace: str, instances: list = None, dynClient: Dynami instances = listSLSInstances(dynClient) for instance in instances: - if namespace in instance['metadata']['namespace']: + if namespace in instance["metadata"]["namespace"]: return True return False @@ -99,10 +103,10 @@ def getSLSRegistrationDetails(namespace: str, name: str, dynClient: DynamicClien try: slsAPI = dynClient.resources.get(api_version="sls.ibm.com/v1", kind="LicenseService") slsInstance = slsAPI.get(name=name, namespace=namespace) - if hasattr(slsInstance, 'status') and hasattr(slsInstance.status, 'licenseId') and hasattr(slsInstance.status, 'registrationKey'): + if hasattr(slsInstance, "status") and hasattr(slsInstance.status, "licenseId") and hasattr(slsInstance.status, "registrationKey"): return dict( registrationKey=slsInstance.status.registrationKey, - licenseId=slsInstance.status.licenseId + licenseId=slsInstance.status.licenseId, ) except NotFoundError: logger.info(f"No SLS '{name}' found in namespace {namespace}.'") diff --git a/src/mas/devops/tekton.py b/src/mas/devops/tekton.py index b915cac1..5577335c 100644 --- a/src/mas/devops/tekton.py +++ b/src/mas/devops/tekton.py @@ -18,13 +18,25 @@ from time import sleep -from kubeconfig import kubectl from openshift.dynamic import DynamicClient -from openshift.dynamic.exceptions import NotFoundError, UnprocessibleEntityError, ApiException +from openshift.dynamic.exceptions import ( + NotFoundError, + UnprocessibleEntityError, + ApiException, +) from jinja2 import Environment, FileSystemLoader -from .ocp import getConsoleURL, waitForCRD, waitForDeployment, crdExists, waitForPVC, getStorageClasses, getStorageClassVolumeBindingMode, getClusterVersion +from .ocp import ( + getConsoleURL, + waitForCRD, + waitForDeployment, + crdExists, + waitForPVC, + getStorageClasses, + getStorageClassVolumeBindingMode, + getClusterVersion, +) logger = logging.getLogger(__name__) @@ -62,7 +74,10 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: while attempts < max_retries: try: - manifest = packagemanifestAPI.get(name="openshift-pipelines-operator-rh", namespace="openshift-marketplace") + manifest = packagemanifestAPI.get( + name="openshift-pipelines-operator-rh", + namespace="openshift-marketplace", + ) logger.info("Successfully found OpenShift Pipelines Operator package manifest") break except NotFoundError as e: @@ -92,9 +107,7 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: # Create subscription templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) template = env.get_template("subscription.yml.j2") renderedTemplate = template.render( subscription_name="openshift-pipelines-operator", @@ -102,7 +115,7 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: package_name="openshift-pipelines-operator-rh", package_channel=defaultChannel, catalog_name=catalogSource, - catalog_namespace=catalogSourceNamespace + catalog_namespace=catalogSourceNamespace, ) subscription = yaml.safe_load(renderedTemplate) subscriptionsAPI.apply(body=subscription, namespace="openshift-operators") @@ -126,7 +139,11 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: # Wait for the webhook to be ready logger.debug("Waiting for tekton-pipelines-webhook Deployment to be ready") - foundReadyWebhook = waitForDeployment(dynClient, namespace="openshift-pipelines", deploymentName="tekton-pipelines-webhook") + foundReadyWebhook = waitForDeployment( + dynClient, + namespace="openshift-pipelines", + deploymentName="tekton-pipelines-webhook", + ) if foundReadyWebhook: logger.info("OpenShift Pipelines Webhook is installed and ready") else: @@ -175,7 +192,7 @@ def installOpenShiftPipelines(dynClient: DynamicClient, customStorageClassName: dynClient=dynClient, namespace=pvcNamespace, pvcName=pvcName, - storageClassName=customStorageClassName + storageClassName=customStorageClassName, ) if tektonPVCisReady: logger.info("OpenShift Pipelines postgres is installed and ready") @@ -221,7 +238,7 @@ def enablePipelinesConsolePlugin(dynClient: DynamicClient) -> bool: logger.debug(f"Detected OpenShift version: {clusterVersion}") # Parse version (e.g., "4.21.0" -> major=4, minor=21) - versionParts = clusterVersion.split('.') + versionParts = clusterVersion.split(".") if len(versionParts) < 2: logger.warning(f"Unable to parse cluster version '{clusterVersion}', skipping plugin enablement") return True @@ -247,7 +264,7 @@ def enablePipelinesConsolePlugin(dynClient: DynamicClient) -> bool: console = consoleAPI.get(name="cluster") # Check if plugin is already enabled - currentPlugins = console.spec.plugins if hasattr(console.spec, 'plugins') and console.spec.plugins else [] + currentPlugins = console.spec.plugins if hasattr(console.spec, "plugins") and console.spec.plugins else [] pluginName = "pipelines-console-plugin" if pluginName in currentPlugins: @@ -259,17 +276,9 @@ def enablePipelinesConsolePlugin(dynClient: DynamicClient) -> bool: # Create patch to add plugin to the list updatedPlugins = list(currentPlugins) + [pluginName] - patch = { - "spec": { - "plugins": updatedPlugins - } - } + patch = {"spec": {"plugins": updatedPlugins}} - consoleAPI.patch( - name="cluster", - body=patch, - content_type="application/merge-patch+json" - ) + consoleAPI.patch(name="cluster", body=patch, content_type="application/merge-patch+json") logger.info("Successfully enabled Pipelines console plugin") return True @@ -364,13 +373,19 @@ def addMissingStorageClassToTektonPVC(dynClient: DynamicClient, namespace: str, return False -def updateTektonDefinitions(namespace: str, yamlFile: str) -> None: +def updateTektonDefinitions(dynClient: DynamicClient, namespace: str, yamlFile: str) -> None: """ Install or update MAS Tekton pipeline and task definitions from a YAML file. - Uses kubectl to apply a YAML file containing multiple resource types. + Parses a YAML file containing multiple Tekton resources (pipelines, tasks, etc.) + and applies each resource individually using the kubernetes python client. + Includes retry logic to handle intermittent network failures common in OCP clusters. + + This is an all-or-nothing operation - if any resource fails to apply after retries, + the function will raise an exception immediately without processing remaining resources. Parameters: + dynClient (DynamicClient): OpenShift Dynamic Client namespace (str): The namespace to apply the definitions to yamlFile (str): Path to the YAML file containing Tekton definitions @@ -378,14 +393,117 @@ def updateTektonDefinitions(namespace: str, yamlFile: str) -> None: None Raises: - kubeconfig.exceptions.KubectlCommandError: If kubectl command fails + FileNotFoundError: If the YAML file does not exist + ApiException: If resource application fails after all retries or if API resource cannot be retrieved """ - result = kubectl.run(subcmd_args=['apply', '-n', namespace, '-f', yamlFile]) - for line in result.split("\n"): - logger.debug(line) + if not path.isfile(yamlFile): + logger.error(f"Tekton definitions file not found: {yamlFile}") + raise FileNotFoundError(f"Tekton definitions file not found: {yamlFile}") + + # Load all resources from the YAML file + with open(yamlFile, "r") as file: + resources = list(yaml.safe_load_all(file)) + + logger.info(f"Applying {len(resources)} Tekton resources from {yamlFile} to namespace {namespace}") + + # Retry configuration optimized for poor network conditions + maxRetries = 10 + baseDelay = 1 # seconds + maxDelay = 15 # seconds + + appliedCount = 0 + apiCache = {} # Cache API objects by (apiVersion, kind) to avoid repeated discovery + for resourceIndex, resourceBody in enumerate(resources, start=1): + if resourceBody is None: + continue -def preparePipelinesNamespace(dynClient: DynamicClient, instanceId: str = None, storageClass: str = None, accessMode: str = None, waitForBind: bool = True, configureRBAC: bool = True, createConfigPVC: bool = True, createBackupPVC: bool = False, backupStorageSize: str = "20Gi"): + apiVersion = resourceBody.get("apiVersion") + kind = resourceBody.get("kind") + metadata = resourceBody.get("metadata", {}) + name = metadata.get("name", "") + + logger.debug(f"Processing resource {resourceIndex}/{len(resources)}: {kind}/{name}") + + # Get or create cached API object + apiKey = (apiVersion, kind) + if apiKey not in apiCache: + try: + apiCache[apiKey] = dynClient.resources.get(api_version=apiVersion, kind=kind) + except Exception as e: + logger.error(f"Failed to get API resource for {kind} (apiVersion={apiVersion}): {e}") + raise ApiException(f"Cannot proceed: Failed to get API resource for {kind} (apiVersion={apiVersion})") + + resourceAPI = apiCache[apiKey] + + # Apply resource with retry logic for transient failures + for attempt in range(maxRetries): + try: + resourceAPI.apply(body=resourceBody, namespace=namespace) + + # Log success only if there were previous failures + if attempt > 0: + logger.info(f"Successfully applied {kind}/{name} after {attempt + 1} attempts") + else: + logger.debug(f"Applied {kind}/{name}") + + appliedCount += 1 + break # Success, exit retry loop + + except ApiException as e: + # Check if it's a retryable error + errorMessage = str(e).lower() + isRetryable = ( + e.status in [429, 503, 504] + or "tls handshake timeout" in errorMessage + or "eof" in errorMessage + or "connection refused" in errorMessage + or "connection reset" in errorMessage + or "too many requests" in errorMessage + or "apiserver is shutting down" in errorMessage + or "net/http" in errorMessage + ) + + if isRetryable and attempt < maxRetries - 1: + # Exponential backoff with jitter + import random + + waitTime = min(baseDelay * (2**attempt), maxDelay) + jitter = random.uniform(0, 0.1 * waitTime) + totalWait = waitTime + jitter + + logger.warning( + f"Transient error applying {kind}/{name} " f"(attempt {attempt + 1}/{maxRetries}): {str(e)[:150]}. " f"Retrying in {totalWait:.1f}s..." + ) + sleep(totalWait) + else: + # Exhausted retries or non-retryable error - fail immediately + if isRetryable: + logger.error(f"Failed to apply {kind}/{name} after {maxRetries} attempts. " f"Last error: {e.status} - {str(e)[:200]}") + else: + logger.error(f"Failed to apply {kind}/{name}: {e.status} - {str(e)[:200]}") + raise ApiException(f"Failed to apply Tekton resource {kind}/{name} after {appliedCount} successful applications") + + except Exception as e: + # Catch any other unexpected errors - fail immediately + logger.error(f"Unexpected error applying {kind}/{name}: {type(e).__name__} - {str(e)[:200]}") + raise + + # All resources applied successfully + logger.info(f"Successfully applied all {appliedCount} Tekton resources") + + +def preparePipelinesNamespace( + dynClient: DynamicClient, + instanceId: str = None, + storageClass: str = None, + accessMode: str = None, + waitForBind: bool = True, + configureRBAC: bool = True, + createConfigPVC: bool = True, + createBackupPVC: bool = False, + backupStorageSize: str = "20Gi", +): """ Prepare a namespace for MAS pipelines by creating RBAC and PVC resources. @@ -410,9 +528,7 @@ def preparePipelinesNamespace(dynClient: DynamicClient, instanceId: str = None, NotFoundError: If resources cannot be created """ templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) if instanceId is None: namespace = "mas-pipelines" template = env.get_template("pipelines-rbac-cluster.yml.j2") @@ -434,7 +550,7 @@ def preparePipelinesNamespace(dynClient: DynamicClient, instanceId: str = None, # Automatically determine if we should wait for PVC binding based on storage class volumeBindingMode = getStorageClassVolumeBindingMode(dynClient, storageClass) - waitForBind = (volumeBindingMode == "Immediate") + waitForBind = volumeBindingMode == "Immediate" # Create config PVC if requested if createConfigPVC: @@ -443,7 +559,7 @@ def preparePipelinesNamespace(dynClient: DynamicClient, instanceId: str = None, renderedTemplate = template.render( mas_instance_id=instanceId, pipeline_storage_class=storageClass, - pipeline_storage_accessmode=accessMode + pipeline_storage_accessmode=accessMode, ) logger.debug(renderedTemplate) pvc = yaml.safe_load(renderedTemplate) @@ -471,7 +587,7 @@ def preparePipelinesNamespace(dynClient: DynamicClient, instanceId: str = None, mas_instance_id=instanceId, pipeline_storage_class=storageClass, pipeline_storage_accessmode=accessMode, - backup_storage_size=backupStorageSize + backup_storage_size=backupStorageSize, ) logger.debug(renderedBackupTemplate) backupPvc = yaml.safe_load(renderedBackupTemplate) @@ -492,7 +608,14 @@ def preparePipelinesNamespace(dynClient: DynamicClient, instanceId: str = None, logger.info(f"Storage class {storageClass} uses volumeBindingMode={volumeBindingMode}, skipping backup PVC bind wait") -def prepareAiServicePipelinesNamespace(dynClient: DynamicClient, instanceId: str = None, storageClass: str = None, accessMode: str = None, waitForBind: bool = True, configureRBAC: bool = True): +def prepareAiServicePipelinesNamespace( + dynClient: DynamicClient, + instanceId: str = None, + storageClass: str = None, + accessMode: str = None, + waitForBind: bool = True, + configureRBAC: bool = True, +): """ Prepare a namespace for AI Service pipelines by creating RBAC and PVC resources. @@ -514,9 +637,7 @@ def prepareAiServicePipelinesNamespace(dynClient: DynamicClient, instanceId: str NotFoundError: If resources cannot be created """ templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) namespace = f"aiservice-{instanceId}-pipelines" template = env.get_template("aiservice-pipelines-rbac.yml.j2") @@ -532,7 +653,7 @@ def prepareAiServicePipelinesNamespace(dynClient: DynamicClient, instanceId: str renderedTemplate = template.render( aiservice_instance_id=instanceId, pipeline_storage_class=storageClass, - pipeline_storage_accessmode=accessMode + pipeline_storage_accessmode=accessMode, ) logger.debug(renderedTemplate) pvc = yaml.safe_load(renderedTemplate) @@ -541,7 +662,7 @@ def prepareAiServicePipelinesNamespace(dynClient: DynamicClient, instanceId: str # Automatically determine if we should wait for PVC binding based on storage class volumeBindingMode = getStorageClassVolumeBindingMode(dynClient, storageClass) - waitForBind = (volumeBindingMode == "Immediate") + waitForBind = volumeBindingMode == "Immediate" if waitForBind: logger.info(f"Storage class {storageClass} uses volumeBindingMode={volumeBindingMode}, waiting for PVC to bind") @@ -591,14 +712,24 @@ def prepareRestoreSecrets(dynClient: DynamicClient, namespace: str, restoreConfi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-restore-configs" - } + "metadata": {"name": "pipeline-restore-configs"}, } secretsAPI.create(body=restoreConfigs, namespace=namespace) -def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFile: str = None, additionalConfigs: dict = None, certs: str = None, podTemplates: str = None, slack_token: str = None, slack_channel: str = None, aiserviceConfig: str = None, db2LicenseFile: dict | None = None, facilitiesProperties: dict | None = None) -> None: +def prepareInstallSecrets( + dynClient: DynamicClient, + namespace: str, + slsLicenseFile: str = None, + additionalConfigs: dict = None, + certs: str = None, + podTemplates: str = None, + slack_token: str = None, + slack_channel: str = None, + aiserviceConfig: str = None, + db2LicenseFile: dict | None = None, + facilitiesProperties: dict | None = None, +) -> None: """ Create or update secrets required for MAS installation pipelines. @@ -630,7 +761,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi # Extract instance ID from namespace using regex # Supports both formats: mas-{instance_id}-pipelines and aiservice-{instance_id}-pipelines instance_id = None - namespace_pattern = r'^(?:mas|aiservice)-(.+)-pipelines$' + namespace_pattern = r"^(?:mas|aiservice)-(.+)-pipelines$" match = re.match(namespace_pattern, namespace) if match: instance_id = match.group(1) @@ -644,9 +775,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi except NotFoundError: pass - secret_data = { - "MAS_INSTANCE_ID": base64.b64encode(instance_id.encode()).decode() - } + secret_data = {"MAS_INSTANCE_ID": base64.b64encode(instance_id.encode()).decode()} # Add slack_token if provided if slack_token: @@ -660,10 +789,8 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "mas-devops-slack" - }, - "data": secret_data + "metadata": {"name": "mas-devops-slack"}, + "data": secret_data, } secretsAPI.create(body=mas_devops_secret, namespace=namespace) logger.info(f"Created mas-devops-slack secret with MAS_INSTANCE_ID={instance_id} in namespace {namespace}") @@ -681,9 +808,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-additional-configs" - } + "metadata": {"name": "pipeline-additional-configs"}, } secretsAPI.create(body=additionalConfigs, namespace=namespace) @@ -699,9 +824,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-sls-entitlement" - } + "metadata": {"name": "pipeline-sls-entitlement"}, } secretsAPI.create(body=slsLicenseFile, namespace=namespace) @@ -718,9 +841,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-certificates" - } + "metadata": {"name": "pipeline-certificates"}, } secretsAPI.create(body=certs, namespace=namespace) @@ -737,9 +858,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-pod-templates" - } + "metadata": {"name": "pipeline-pod-templates"}, } secretsAPI.create(body=podTemplates, namespace=namespace) @@ -755,9 +874,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-aiservice-config" - } + "metadata": {"name": "pipeline-aiservice-config"}, } secretsAPI.create(body=aiserviceConfig, namespace=namespace) @@ -773,9 +890,7 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-db2-license" - } + "metadata": {"name": "pipeline-db2-license"}, } secretsAPI.create(body=db2LicenseFile, namespace=namespace) @@ -790,7 +905,12 @@ def prepareInstallSecrets(dynClient: DynamicClient, namespace: str, slsLicenseFi secretsAPI.create(body=facilitiesProperties, namespace=namespace) -def prepareUpdateSecrets(dynClient: DynamicClient, slack_token: str = None, slack_channel: str = None, db2LicenseFile: dict | None = None) -> None: +def prepareUpdateSecrets( + dynClient: DynamicClient, + slack_token: str = None, + slack_channel: str = None, + db2LicenseFile: dict | None = None, +) -> None: """ Create or update mas-devops-slack secret in mas-pipelines namespace for update pipeline. @@ -843,10 +963,8 @@ def prepareUpdateSecrets(dynClient: DynamicClient, slack_token: str = None, slac "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "mas-devops-slack" - }, - "data": secret_data + "metadata": {"name": "mas-devops-slack"}, + "data": secret_data, } secretsAPI.create(body=mas_devops_secret, namespace=namespace) @@ -862,9 +980,7 @@ def prepareUpdateSecrets(dynClient: DynamicClient, slack_token: str = None, slac "apiVersion": "v1", "kind": "Secret", "type": "Opaque", - "metadata": { - "name": "pipeline-db2-license" - } + "metadata": {"name": "pipeline-db2-license"}, } secretsAPI.create(body=db2LicenseFile, namespace=namespace) logger.info(f"Created pipeline-db2-license secret in namespace {namespace}") @@ -903,11 +1019,13 @@ def testCLI() -> None: # fi -def launchUpgradePipeline(dynClient: DynamicClient, - instanceId: str, - skipPreCheck: bool = False, - masChannel: str = "", - params: dict = {}) -> str: +def launchUpgradePipeline( + dynClient: DynamicClient, + instanceId: str, + skipPreCheck: bool = False, + masChannel: str = "", + params: dict = {}, +) -> str: """ Create a PipelineRun to upgrade the chosen MAS instance """ @@ -916,16 +1034,14 @@ def launchUpgradePipeline(dynClient: DynamicClient, timestamp = datetime.now().strftime("%y%m%d-%H%M") # Create the PipelineRun templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) template = env.get_template("pipelinerun-upgrade.yml.j2") renderedTemplate = template.render( timestamp=timestamp, mas_instance_id=instanceId, skip_pre_check=skipPreCheck, mas_channel=masChannel, - **params + **params, ) logger.debug(renderedTemplate) pipelineRun = yaml.safe_load(renderedTemplate) @@ -935,15 +1051,17 @@ def launchUpgradePipeline(dynClient: DynamicClient, return pipelineURL -def launchUninstallPipeline(dynClient: DynamicClient, - instanceId: str, - droNamespace: str, - uninstallCertManager: bool = False, - uninstallGrafana: bool = False, - uninstallCatalog: bool = False, - uninstallDRO: bool = False, - uninstallMongoDb: bool = False, - uninstallSLS: bool = False) -> str: +def launchUninstallPipeline( + dynClient: DynamicClient, + instanceId: str, + droNamespace: str, + uninstallCertManager: bool = False, + uninstallGrafana: bool = False, + uninstallCatalog: bool = False, + uninstallDRO: bool = False, + uninstallMongoDb: bool = False, + uninstallSLS: bool = False, +) -> str: """ Create a PipelineRun to uninstall the chosen MAS instance (and selected dependencies) """ @@ -952,9 +1070,7 @@ def launchUninstallPipeline(dynClient: DynamicClient, timestamp = datetime.now().strftime("%y%m%d-%H%M") # Create the PipelineRun templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) template = env.get_template("pipelinerun-uninstall.yml.j2") grafanaAction = "uninstall" if uninstallGrafana else "none" @@ -974,7 +1090,7 @@ def launchUninstallPipeline(dynClient: DynamicClient, mongodb_action=mongoDbAction, sls_action=slsAction, dro_action=droAction, - dro_namespace=droNamespace + dro_namespace=droNamespace, ) logger.debug(renderedTemplate) pipelineRun = yaml.safe_load(renderedTemplate) @@ -1006,16 +1122,11 @@ def launchPipelineRun(dynClient: DynamicClient, namespace: str, templateName: st timestamp = datetime.now().strftime("%y%m%d-%H%M") # Create the PipelineRun templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) template = env.get_template(f"{templateName}.yml.j2") # Render the pipelineRun - renderedTemplate = template.render( - timestamp=timestamp, - **params - ) + renderedTemplate = template.render(timestamp=timestamp, **params) logger.debug(renderedTemplate) pipelineRun = yaml.safe_load(renderedTemplate) pipelineRunsAPI.apply(body=pipelineRun, namespace=namespace) @@ -1112,15 +1223,19 @@ def launchRestorePipeline(dynClient: DynamicClient, params: dict) -> str: namespace = f"mas-{instanceId}-pipelines" timestamp = launchPipelineRun(dynClient, namespace, "pipelinerun-restore", params) - pipelineURL = f"{getConsoleURL(dynClient)}/k8s/ns/mas-{instanceId}-pipelines/tekton.dev~v1beta1~PipelineRun/{instanceId}-restore-{restoreVersion}-{timestamp}" + pipelineURL = ( + f"{getConsoleURL(dynClient)}/k8s/ns/mas-{instanceId}-pipelines/tekton.dev~v1beta1~PipelineRun/{instanceId}-restore-{restoreVersion}-{timestamp}" + ) return pipelineURL -def launchAiServiceUpgradePipeline(dynClient: DynamicClient, - aiserviceInstanceId: str, - skipPreCheck: bool = False, - aiserviceChannel: str = "", - params: dict = {}) -> str: +def launchAiServiceUpgradePipeline( + dynClient: DynamicClient, + aiserviceInstanceId: str, + skipPreCheck: bool = False, + aiserviceChannel: str = "", + params: dict = {}, +) -> str: """ Create a PipelineRun to upgrade the chosen AI Service instance """ @@ -1129,22 +1244,22 @@ def launchAiServiceUpgradePipeline(dynClient: DynamicClient, timestamp = datetime.now().strftime("%y%m%d-%H%M") # Create the PipelineRun templateDir = path.join(path.abspath(path.dirname(__file__)), "templates") - env = Environment( - loader=FileSystemLoader(searchpath=templateDir) - ) + env = Environment(loader=FileSystemLoader(searchpath=templateDir)) template = env.get_template("pipelinerun-aiservice-upgrade.yml.j2") renderedTemplate = template.render( timestamp=timestamp, aiservice_instance_id=aiserviceInstanceId, skip_pre_check=skipPreCheck, aiservice_channel=aiserviceChannel, - **params + **params, ) logger.debug(renderedTemplate) pipelineRun = yaml.safe_load(renderedTemplate) pipelineRunsAPI.apply(body=pipelineRun, namespace=namespace) - pipelineURL = f"{getConsoleURL(dynClient)}/k8s/ns/aiservice-{aiserviceInstanceId}-pipelines/tekton.dev~v1beta1~PipelineRun/{aiserviceInstanceId}-upgrade-{timestamp}" + pipelineURL = ( + f"{getConsoleURL(dynClient)}/k8s/ns/aiservice-{aiserviceInstanceId}-pipelines/tekton.dev~v1beta1~PipelineRun/{aiserviceInstanceId}-upgrade-{timestamp}" + ) return pipelineURL @@ -1218,12 +1333,18 @@ def prepareInstallRBAC(dynClient: DynamicClient, namespace: str, instanceId: str except ApiException as e: # Check if it's a retryable error (429, 503, 504, or API server shutdown) - is_retryable = (e.status in [429, 503, 504] or "apiserver is shutting down" in str(e).lower() or "connection refused" in str(e).lower() or "too many requests" in str(e).lower()) + is_retryable = ( + e.status in [429, 503, 504] + or "apiserver is shutting down" in str(e).lower() + or "connection refused" in str(e).lower() + or "too many requests" in str(e).lower() + ) if is_retryable and attempt < max_retries - 1: # Exponential backoff with jitter to avoid thundering herd import random - wait_time = min(base_delay * (2 ** attempt), max_delay) + + wait_time = min(base_delay * (2**attempt), max_delay) jitter = random.uniform(0, 0.1 * wait_time) # Add up to 10% jitter total_wait = wait_time + jitter diff --git a/src/mas/devops/users.py b/src/mas/devops/users.py index c3d1d1e6..7baa61ac 100644 --- a/src/mas/devops/users.py +++ b/src/mas/devops/users.py @@ -21,7 +21,7 @@ from packaging.version import Version -class MASUserUtils(): +class MASUserUtils: """ Utility class for managing IBM Maximo Application Suite (MAS) users and permissions. @@ -44,7 +44,16 @@ class MASUserUtils(): MXINTADM = "MXINTADM" - def __init__(self, mas_instance_id: str, mas_workspace_id: str, k8s_client: client.api_client.ApiClient, mas_version: str = '9.0', coreapi_port: int = 443, admin_dashboard_port: int = 443, manage_api_port: int = 443): + def __init__( + self, + mas_instance_id: str, + mas_workspace_id: str, + k8s_client: client.api_client.ApiClient, + mas_version: str = "9.0", + coreapi_port: int = 443, + admin_dashboard_port: int = 443, + manage_api_port: int = 443, + ): """ Initialize MASUserUtils for a specific MAS instance and workspace. @@ -70,15 +79,15 @@ def __init__(self, mas_instance_id: str, mas_workspace_id: str, k8s_client: clie self._mas_superuser_credentials = None self._superuser_auth_token = None - self.mas_admin_url_internal = f'https://admin-dashboard.{self.mas_core_namespace}.svc.cluster.local:{admin_dashboard_port}' + self.mas_admin_url_internal = f"https://admin-dashboard.{self.mas_core_namespace}.svc.cluster.local:{admin_dashboard_port}" self._admin_internal_tls_secret = None self._admin_internal_ca_pem_file_path = None - self.mas_api_url_internal = f'https://coreapi.{self.mas_core_namespace}.svc.cluster.local:{coreapi_port}' + self.mas_api_url_internal = f"https://coreapi.{self.mas_core_namespace}.svc.cluster.local:{coreapi_port}" self._core_internal_tls_secret = None self._core_internal_ca_pem_file_path = None - self.manage_api_url_internal = f'https://{self.mas_instance_id}-{self.mas_workspace_id}.{self.manage_namespace}.svc.cluster.local:{manage_api_port}' + self.manage_api_url_internal = f"https://{self.mas_instance_id}-{self.mas_workspace_id}.{self.manage_namespace}.svc.cluster.local:{manage_api_port}" self._manage_internal_tls_secret = None self._manage_internal_ca_pem_file_path = None self._manage_internal_client_pem_file_path = None @@ -88,7 +97,10 @@ def __init__(self, mas_instance_id: str, mas_workspace_id: str, k8s_client: clie @property def mas_superuser_credentials(self): if self._mas_superuser_credentials is None: - k8s_secret = self.v1_secrets.get(name=f"{self.mas_instance_id}-credentials-superuser", namespace=self.mas_core_namespace) + k8s_secret = self.v1_secrets.get( + name=f"{self.mas_instance_id}-credentials-superuser", + namespace=self.mas_core_namespace, + ) self._mas_superuser_credentials = dict( username=base64.b64decode(k8s_secret.data["username"]).decode("utf-8"), password=base64.b64decode(k8s_secret.data["password"]).decode("utf-8"), @@ -98,13 +110,16 @@ def mas_superuser_credentials(self): @property def admin_internal_tls_secret(self): if self._admin_internal_tls_secret is None: - self._admin_internal_tls_secret = self.v1_secrets.get(name=f"{self.mas_instance_id}-admindashboard-cert-internal", namespace=self.mas_core_namespace) + self._admin_internal_tls_secret = self.v1_secrets.get( + name=f"{self.mas_instance_id}-admindashboard-cert-internal", + namespace=self.mas_core_namespace, + ) return self._admin_internal_tls_secret @property def admin_internal_ca_pem_file_path(self): if self._admin_internal_ca_pem_file_path is None: - ca = base64.b64decode(self.admin_internal_tls_secret.data["ca.crt"]).decode('utf-8') + ca = base64.b64decode(self.admin_internal_tls_secret.data["ca.crt"]).decode("utf-8") with tempfile.NamedTemporaryFile(delete=False, suffix=".pem") as pem_file: pem_file.write(ca.encode()) pem_file.flush() @@ -116,13 +131,16 @@ def admin_internal_ca_pem_file_path(self): @property def core_internal_tls_secret(self): if self._core_internal_tls_secret is None: - self._core_internal_tls_secret = self.v1_secrets.get(name=f"{self.mas_instance_id}-coreapi-cert-internal", namespace=self.mas_core_namespace) + self._core_internal_tls_secret = self.v1_secrets.get( + name=f"{self.mas_instance_id}-coreapi-cert-internal", + namespace=self.mas_core_namespace, + ) return self._core_internal_tls_secret @property def core_internal_ca_pem_file_path(self): if self._core_internal_ca_pem_file_path is None: - ca = base64.b64decode(self.core_internal_tls_secret.data["ca.crt"]).decode('utf-8') + ca = base64.b64decode(self.core_internal_tls_secret.data["ca.crt"]).decode("utf-8") with tempfile.NamedTemporaryFile(delete=False, suffix=".pem") as pem_file: pem_file.write(ca.encode()) pem_file.flush() @@ -136,19 +154,15 @@ def superuser_auth_token(self): if self._superuser_auth_token is None: self.logger.debug("Getting superuser auth token") url = f"{self.mas_admin_url_internal}/logininitial" - headers = { - "Content-Type": "application/json" - } - querystring = { - "verify": False - } + headers = {"Content-Type": "application/json"} + querystring = {"verify": False} payload = self.mas_superuser_credentials response = requests.post( url, json=payload, headers=headers, params=querystring, - verify=self.admin_internal_ca_pem_file_path + verify=self.admin_internal_ca_pem_file_path, ) self._superuser_auth_token = response.json()["token"] return self._superuser_auth_token @@ -156,14 +170,17 @@ def superuser_auth_token(self): @property def manage_internal_tls_secret(self): if self._manage_internal_tls_secret is None: - self._manage_internal_tls_secret = self.v1_secrets.get(name=f"{self.mas_instance_id}-internal-manage-tls", namespace=self.manage_namespace) + self._manage_internal_tls_secret = self.v1_secrets.get( + name=f"{self.mas_instance_id}-internal-manage-tls", + namespace=self.manage_namespace, + ) return self._manage_internal_tls_secret @property def manage_internal_client_pem_file_path(self): if self._manage_internal_client_pem_file_path is None: - cert = base64.b64decode(self.manage_internal_tls_secret.data["tls.crt"]).decode('utf-8') - key = base64.b64decode(self.manage_internal_tls_secret.data["tls.key"]).decode('utf-8') + cert = base64.b64decode(self.manage_internal_tls_secret.data["tls.crt"]).decode("utf-8") + key = base64.b64decode(self.manage_internal_tls_secret.data["tls.key"]).decode("utf-8") with tempfile.NamedTemporaryFile(delete=False, suffix=".pem") as pem_file: pem_file.write(key.encode()) pem_file.write(cert.encode()) @@ -176,7 +193,7 @@ def manage_internal_client_pem_file_path(self): @property def manage_internal_ca_pem_file_path(self): if self._manage_internal_ca_pem_file_path is None: - ca = base64.b64decode(self.manage_internal_tls_secret.data["ca.crt"]).decode('utf-8') + ca = base64.b64decode(self.manage_internal_tls_secret.data["ca.crt"]).decode("utf-8") with tempfile.NamedTemporaryFile(delete=False, suffix=".pem") as pem_file: pem_file.write(ca.encode()) pem_file.flush() @@ -215,26 +232,23 @@ def get_user(self, user_id): resource_id = None # For MAS version >= 9.1, use the Manage API masperuser endpoint - if Version(self.mas_version) >= Version('9.1'): + if Version(self.mas_version) >= Version("9.1"): # Get MXINTADM API key for authentication mxintadm_manage_api_key = self.create_or_get_manage_api_key_for_user(MASUserUtils.MXINTADM, temporary=True) # First request: Query to find user and get resource_id from href url = f"{self.manage_api_url_internal}/maximo/api/os/masperuser" - querystring = { - "lean": 1, - "oslc.where": f"user.userid=\"{user_id}\"" - } + querystring = {"lean": 1, "oslc.where": f'user.userid="{user_id}"'} headers = { "Accept": "application/json", - "apikey": mxintadm_manage_api_key["apikey"] + "apikey": mxintadm_manage_api_key["apikey"], } response = requests.get( url, headers=headers, params=querystring, cert=self.manage_internal_client_pem_file_path, - verify=self.manage_internal_ca_pem_file_path + verify=self.manage_internal_ca_pem_file_path, ) user_info = response.json() @@ -251,32 +265,28 @@ def get_user(self, user_id): url = f"{self.manage_api_url_internal}/maximo/api/os/masperuser/" querystring = { "lean": 1, - "oslc.where": f"personid=\"{user_id}\"", - "oslc.select": "personid,displayname" + "oslc.where": f'personid="{user_id}"', + "oslc.select": "personid,displayname", } headers = { "Accept": "application/json", - "apikey": mxintadm_manage_api_key["apikey"] + "apikey": mxintadm_manage_api_key["apikey"], } response = requests.get( url, headers=headers, params=querystring, cert=self.manage_internal_client_pem_file_path, - verify=self.manage_internal_ca_pem_file_path + verify=self.manage_internal_ca_pem_file_path, ) else: # For earlier versions, use the Core API v3/users endpoint url = f"{self.mas_api_url_internal}/v3/users/{user_id}" headers = { "Accept": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } - response = requests.get( - url, - headers=headers, - verify=self.core_internal_ca_pem_file_path - ) + response = requests.get(url, headers=headers, verify=self.core_internal_ca_pem_file_path) if response.status_code == 404: return resource_id, None @@ -285,7 +295,7 @@ def get_user(self, user_id): raise Exception(f"{response.status_code} {response.text}") # Handle response based on version - if Version(self.mas_version) >= Version('9.1'): + if Version(self.mas_version) >= Version("9.1"): # Manage API returns member array user_data = response.json() if "member" in user_data and len(user_data["member"]) > 0: @@ -323,31 +333,29 @@ def get_or_create_user(self, payload): Exception: If user creation fails with an unexpected status code. """ # Determine the user ID field based on version - user_id_field = "personid" if Version(self.mas_version) >= Version('9.1') else "id" + user_id_field = "personid" if Version(self.mas_version) >= Version("9.1") else "id" user_id = payload[user_id_field] resource_id, existing_user = self.get_user(user_id) if existing_user is not None: # Log using the appropriate field based on version - user_identifier = existing_user.get('personid') or existing_user.get('id') + user_identifier = existing_user.get("personid") or existing_user.get("id") self.logger.info(f"Existing user {user_identifier} found") return resource_id, existing_user self.logger.info(f"Creating new user {user_id}") # For MAS version >= 9.1, use the Manage API masapiuser endpoint - if Version(self.mas_version) >= Version('9.1'): + if Version(self.mas_version) >= Version("9.1"): # Get MXINTADM API key for authentication mxintadm_manage_api_key = self.create_or_get_manage_api_key_for_user(MASUserUtils.MXINTADM, temporary=True) url = f"{self.manage_api_url_internal}/maximo/api/os/masperuser" - querystring = { - "lean": 1 - } + querystring = {"lean": 1} headers = { "Content-Type": "application/json", - "apikey": mxintadm_manage_api_key["apikey"] + "apikey": mxintadm_manage_api_key["apikey"], } self.logger.debug(f"Creating new user {user_id} with Manage API with payload {payload}") response = requests.post( @@ -356,7 +364,7 @@ def get_or_create_user(self, payload): headers=headers, params=querystring, cert=self.manage_internal_client_pem_file_path, - verify=self.manage_internal_ca_pem_file_path + verify=self.manage_internal_ca_pem_file_path, ) if response.status_code == 201: # Manage API returns empty response body on success, fetch the user @@ -379,14 +387,14 @@ def get_or_create_user(self, payload): querystring = {} headers = { "Content-Type": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } response = requests.post( url, json=payload, headers=headers, params=querystring, - verify=self.core_internal_ca_pem_file_path + verify=self.core_internal_ca_pem_file_path, ) if response.status_code == 201: # For version < 9.1, resource_id is None @@ -425,23 +433,15 @@ def set_user_group_reassignment_auth(self, user_id, resource_id, groupreassign, # Use Manage API to update the user's grpreassignauth url = f"{self.manage_api_url_internal}/maximo/api/os/masperuser/{resource_id}" - querystring = { - "lean": 1, - "ccm": 1 - } + querystring = {"lean": 1, "ccm": 1} headers = { "Content-Type": "application/json", "apikey": manage_api_key["apikey"], "x-method-override": "PATCH", - "patchtype": "MERGE" + "patchtype": "MERGE", } - payload = { - "maxuser": { - "userid": user_id, - "grpreassignauth": groupreassign - } - } + payload = {"maxuser": {"userid": user_id, "grpreassignauth": groupreassign}} self.logger.debug(f"Sending PATCH request to {url} with payload: {payload}") response = requests.post( @@ -450,7 +450,7 @@ def set_user_group_reassignment_auth(self, user_id, resource_id, groupreassign, headers=headers, params=querystring, cert=self.manage_internal_client_pem_file_path, - verify=self.manage_internal_ca_pem_file_path + verify=self.manage_internal_ca_pem_file_path, ) if response.status_code in [200, 204]: @@ -481,13 +481,13 @@ def update_user(self, payload): url = f"{self.mas_api_url_internal}/v3/users/{user_id}" headers = { "Accept": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } response = requests.put( url, headers=headers, json=payload, - verify=self.core_internal_ca_pem_file_path + verify=self.core_internal_ca_pem_file_path, ) if response.status_code == 200: @@ -516,15 +516,13 @@ def update_user_display_name(self, user_id, display_name): url = f"{self.mas_api_url_internal}/v3/users/{user_id}" headers = { "Accept": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } response = requests.patch( url, headers=headers, - json={ - "displayName": display_name - }, - verify=self.core_internal_ca_pem_file_path + json={"displayName": display_name}, + verify=self.core_internal_ca_pem_file_path, ) if response.status_code == 200: @@ -586,15 +584,12 @@ def link_user_to_local_idp(self, user_id, email_password=False, manage_api_key=N self.logger.info(f"Linking user {user_id} to local IDP using Manage API (version {self.mas_version})") url = f"{self.manage_api_url_internal}/maximo/api/os/masperuser/{resource_id}" - querystring = { - "lean": 1, - "ccm": 1 - } + querystring = {"lean": 1, "ccm": 1} headers = { "Content-Type": "application/json", "apikey": manage_api_key["apikey"], "x-method-override": "PATCH", - "patchtype": "MERGE" + "patchtype": "MERGE", } payload = { @@ -607,9 +602,9 @@ def link_user_to_local_idp(self, user_id, email_password=False, manage_api_key=N "logintype": "0", "idploginid": user_id, "idptype": "local", - "enabled": True + "enabled": True, } - ] + ], } } self.logger.debug(f"Sending PATCH request to {url} with payload: {payload}") @@ -620,7 +615,7 @@ def link_user_to_local_idp(self, user_id, email_password=False, manage_api_key=N headers=headers, params=querystring, cert=self.manage_internal_client_pem_file_path, - verify=self.manage_internal_ca_pem_file_path + verify=self.manage_internal_ca_pem_file_path, ) if response.status_code in [200, 204]: @@ -642,22 +637,20 @@ def link_user_to_local_idp(self, user_id, email_password=False, manage_api_key=N self.logger.info(f"Linking user {user_id} to local IDP using Core API (version {self.mas_version}, email_password: {email_password})") url = f"{self.mas_api_url_internal}/v3/users/{user_id}/idps/local" - querystring = { - "emailPassword": email_password - } + querystring = {"emailPassword": email_password} payload = { "idpUserId": user_id, } headers = { "Content-Type": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } response = requests.put( url, json=payload, headers=headers, params=querystring, - verify=self.core_internal_ca_pem_file_path + verify=self.core_internal_ca_pem_file_path, ) if response.status_code != 200: raise Exception(response.text) @@ -683,13 +676,9 @@ def get_user_workspaces(self, user_id): url = f"{self.mas_api_url_internal}/v3/users/{user_id}/workspaces" headers = { "Accept": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } - response = requests.get( - url, - headers=headers, - verify=self.core_internal_ca_pem_file_path - ) + response = requests.get(url, headers=headers, verify=self.core_internal_ca_pem_file_path) if response.status_code == 404: raise Exception(f"User {user_id} does not exist") @@ -726,21 +715,17 @@ def add_user_to_workspace(self, user_id, is_workspace_admin=False): self.logger.info(f"Adding user {user_id} to {self.mas_workspace_id} (is_workspace_admin: {is_workspace_admin})") url = f"{self.mas_api_url_internal}/workspaces/{self.mas_workspace_id}/users/{user_id}" querystring = {} - payload = { - "permissions": { - "workspaceAdmin": is_workspace_admin - } - } + payload = {"permissions": {"workspaceAdmin": is_workspace_admin}} headers = { "Content-Type": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } response = requests.put( url, json=payload, headers=headers, params=querystring, - verify=self.core_internal_ca_pem_file_path + verify=self.core_internal_ca_pem_file_path, ) if response.status_code == 200: @@ -766,13 +751,9 @@ def get_user_application_permissions(self, user_id, application_id): url = f"{self.mas_api_url_internal}/workspaces/{self.mas_workspace_id}/applications/{application_id}/users/{user_id}" headers = { "Accept": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } - response = requests.get( - url, - headers=headers, - verify=self.core_internal_ca_pem_file_path - ) + response = requests.get(url, headers=headers, verify=self.core_internal_ca_pem_file_path) if response.status_code == 200: return response.json() @@ -810,19 +791,17 @@ def set_user_application_permission(self, user_id, application_id, role): self.logger.info(f"Setting user {user_id} role for {application_id} to {role}") url = f"{self.mas_api_url_internal}/workspaces/{self.mas_workspace_id}/applications/{application_id}/users/{user_id}" querystring = {} - payload = { - "role": role - } + payload = {"role": role} headers = { "Content-Type": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } response = requests.put( url, json=payload, headers=headers, params=querystring, - verify=self.core_internal_ca_pem_file_path + verify=self.core_internal_ca_pem_file_path, ) if response.status_code == 200: @@ -851,11 +830,16 @@ def check_user_sync(self, user_id, application_id, timeout_secs=60 * 10, retry_i Exception: If sync doesn't complete within the timeout period. """ t_end = time.time() + timeout_secs - self.logger.info(f"Awaiting user {user_id} sync status \"SUCCESS\" for app {application_id}: {t_end - time.time():.2f} seconds remaining") + self.logger.info(f'Awaiting user {user_id} sync status "SUCCESS" for app {application_id}: {t_end - time.time():.2f} seconds remaining') while time.time() < t_end: resource_id, user = self.get_user(user_id) - if "applications" not in user or application_id not in user["applications"] or "sync" not in user["applications"][application_id] or "state" not in user["applications"][application_id]["sync"]: + if ( + "applications" not in user + or application_id not in user["applications"] + or "sync" not in user["applications"][application_id] + or "state" not in user["applications"][application_id]["sync"] + ): self.logger.warning(f"User {user_id} does not have any sync state for application {application_id}, triggering resync") self.resync_users([user_id]) time.sleep(retry_interval_secs) @@ -868,7 +852,9 @@ def check_user_sync(self, user_id, application_id, timeout_secs=60 * 10, retry_i self.resync_users([user_id]) time.sleep(retry_interval_secs) else: - self.logger.info(f"User {user_id} sync has not been completed yet for app {application_id} (currrently {sync_state}): {t_end - time.time():.2f} seconds remaining") + self.logger.info( + f"User {user_id} sync has not been completed yet for app {application_id} (currrently {sync_state}): {t_end - time.time():.2f} seconds remaining" + ) time.sleep(retry_interval_secs) raise Exception(f"User {user_id} sync failed to complete for app within {timeout_secs} seconds") @@ -902,7 +888,7 @@ def resync_users(self, user_ids): resource_id, user = self.get_user(user_id) # For version >= 9.1, Manage API uses "displayname" (lowercase) # For version < 9.1, Core API uses "displayName" (camelCase) - display_name = user.get("displayname") if Version(self.mas_version) >= Version('9.1') else user.get("displayName") + display_name = user.get("displayname") if Version(self.mas_version) >= Version("9.1") else user.get("displayName") if display_name: self.update_user_display_name(user_id, display_name) @@ -932,10 +918,7 @@ def create_or_get_manage_api_key_for_user(self, user_id, temporary=False): "lean": 1, } - payload = { - "expiration": -1, - "userid": user_id - } + payload = {"expiration": -1, "userid": user_id} headers = { "Content-Type": "application/json", } @@ -1000,7 +983,7 @@ def get_manage_api_key_for_user(self, user_id): "ccm": 1, "lean": 1, "oslc.select": "*", - "oslc.where": f"userid=\"{user_id}\"", + "oslc.where": f'userid="{user_id}"', } headers = { "Accept": "application/json", @@ -1011,7 +994,7 @@ def get_manage_api_key_for_user(self, user_id): headers=headers, params=querystring, verify=self.manage_internal_ca_pem_file_path, - cert=self.manage_internal_client_pem_file_path + cert=self.manage_internal_client_pem_file_path, ) if response.status_code == 200: @@ -1040,7 +1023,7 @@ def delete_manage_api_key(self, manage_api_key): self.logger.info(f"Deleting Manage API Key for user {manage_api_key['userid']}") # extract the apikey's identifier from the href - match = re.search(r'\/maximo\/api\/os\/mxapiapikey\/(.*)', manage_api_key['href']) + match = re.search(r"\/maximo\/api\/os\/mxapiapikey\/(.*)", manage_api_key["href"]) if match is None: raise Exception(f"Could not parse API Key href: {manage_api_key['href']}") @@ -1085,7 +1068,7 @@ def get_manage_group_id(self, group_name, manage_api_key): "ccm": 1, "lean": 1, "oslc.select": "maxgroupid", - "oslc.where": f"groupname=\"{group_name}\"", + "oslc.where": f'groupname="{group_name}"', } headers = { "Accept": "application/json", @@ -1103,7 +1086,7 @@ def get_manage_group_id(self, group_name, manage_api_key): json = response.json() if "member" in json and len(json["member"]) > 0 and "maxgroupid" in json["member"][0]: - return json["member"][0]['maxgroupid'] + return json["member"][0]["maxgroupid"] return None @@ -1132,7 +1115,7 @@ def is_user_in_manage_group(self, group_name, user_id, manage_api_key): url = f"{self.manage_api_url_internal}/maximo/api/os/mxapigroup/{group_id}/groupuser" querystring = { "lean": 1, - "oslc.where": f"userid=\"{user_id}\"", + "oslc.where": f'userid="{user_id}"', } headers = { "Accept": "application/json", @@ -1190,13 +1173,7 @@ def add_user_to_manage_group(self, user_id, group_name, manage_api_key): "patchtype": "MERGE", "apikey": manage_api_key["apikey"], # <--- careful, don't log headers as-is (apikey is sensitive) } - payload = { - "groupuser": [ - { - "userid": f"{user_id}" - } - ] - } + payload = {"groupuser": [{"userid": f"{user_id}"}]} response = requests.post( url, headers=headers, @@ -1239,7 +1216,7 @@ def get_all_manage_groups(self): params=querystring, # verify=self.manage_internal_ca_pem_file_path, cert=self.manage_internal_client_pem_file_path, - verify=self.manage_internal_ca_pem_file_path + verify=self.manage_internal_ca_pem_file_path, ) if response.status_code != 200: @@ -1269,13 +1246,9 @@ def get_mas_applications_in_workspace(self): url = f"{self.mas_api_url_internal}/workspaces/{self.mas_workspace_id}/applications" headers = { "Accept": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } - response = requests.get( - url, - headers=headers, - verify=self.core_internal_ca_pem_file_path - ) + response = requests.get(url, headers=headers, verify=self.core_internal_ca_pem_file_path) if response.status_code == 200: return response.json() raise Exception(f"{response.status_code} {response.text}") @@ -1297,13 +1270,9 @@ def get_mas_application_availability(self, mas_application_id): url = f"{self.mas_api_url_internal}/workspaces/{self.mas_workspace_id}/applications/{mas_application_id}" headers = { "Accept": "application/json", - "x-access-token": self.superuser_auth_token + "x-access-token": self.superuser_auth_token, } - response = requests.get( - url, - headers=headers, - verify=self.core_internal_ca_pem_file_path - ) + response = requests.get(url, headers=headers, verify=self.core_internal_ca_pem_file_path) if response.status_code == 200: return response.json() raise Exception(f"{response.status_code} {response.text}") @@ -1332,7 +1301,9 @@ def await_mas_application_availability(self, mas_application_id, timeout_secs=60 if "available" in app and "ready" in app and app["ready"] and app["available"]: return else: - self.logger.info(f"{mas_application_id} is not ready or available, retry in {retry_interval_secs} seconds: {t_end - time.time():.2f} seconds remaining") + self.logger.info( + f"{mas_application_id} is not ready or available, retry in {retry_interval_secs} seconds: {t_end - time.time():.2f} seconds remaining" + ) time.sleep(retry_interval_secs) raise Exception(f"{mas_application_id} did not become ready and available in time, aborting") @@ -1356,7 +1327,7 @@ def parse_initial_users_from_aws_secret_json(self, secret_json): """ primary = [] secondary = [] - for (email, csv) in secret_json.items(): + for email, csv in secret_json.items(): values = csv.split(",") if len(values) != 3 and len(values) != 4: @@ -1375,7 +1346,7 @@ def parse_initial_users_from_aws_secret_json(self, secret_json): "email": email, "given_name": given_name, "family_name": family_name, - "id": id + "id": id, } if user_type == "primary": primary.append(user) @@ -1384,12 +1355,7 @@ def parse_initial_users_from_aws_secret_json(self, secret_json): else: raise Exception(f"Unknown user type for {email}: {user_type}") - initial_users = { - "users": { - "primary": primary, - "secondary": secondary - } - } + initial_users = {"users": {"primary": primary, "secondary": secondary}} return initial_users def create_initial_users_for_saas(self, initial_users): @@ -1472,10 +1438,7 @@ def create_initial_users_for_saas(self, initial_users): failed.append(secondary_user) self.logger.info("") - return { - "completed": completed, - "failed": failed - } + return {"completed": completed, "failed": failed} def create_initial_user_for_saas(self, user, user_type, groupreassign=None): """ @@ -1517,10 +1480,17 @@ def create_initial_user_for_saas(self, user, user_type, groupreassign=None): # default to email if no id provided user_id = user_email - if Version(self.mas_version) < Version('9.1'): + if Version(self.mas_version) < Version("9.1"): self.create_initial_user_for_saas_pre_9_1(user_email, user_given_name, user_family_name, user_id, user_type) else: - self.create_initial_user_for_saas_post_9_1(user_email, user_given_name, user_family_name, user_id, user_type, groupreassign) + self.create_initial_user_for_saas_post_9_1( + user_email, + user_given_name, + user_family_name, + user_id, + user_type, + groupreassign, + ) def create_initial_user_for_saas_pre_9_1(self, user_email, user_given_name, user_family_name, user_id, user_type): """ @@ -1567,12 +1537,12 @@ def create_initial_user_for_saas_pre_9_1(self, user_email, user_given_name, user permissions = { "systemAdmin": False, "userAdmin": True, - "apikeyAdmin": False + "apikeyAdmin": False, } entitlement = { "application": "PREMIUM", "admin": "ADMIN_BASE", - "alwaysReserveLicense": True + "alwaysReserveLicense": True, } is_workspace_admin = True application_role = "ADMIN" @@ -1583,12 +1553,12 @@ def create_initial_user_for_saas_pre_9_1(self, user_email, user_given_name, user permissions = { "systemAdmin": False, "userAdmin": False, - "apikeyAdmin": False + "apikeyAdmin": False, } entitlement = { "application": "BASE", "admin": "NONE", - "alwaysReserveLicense": True + "alwaysReserveLicense": True, } is_workspace_admin = False application_role = "USER" @@ -1604,13 +1574,7 @@ def create_initial_user_for_saas_pre_9_1(self, user_email, user_given_name, user "status": {"active": True}, "username": username, "owner": "local", - "emails": [ - { - "value": user_email, - "type": "Work", - "primary": True - } - ], + "emails": [{"value": user_email, "type": "Work", "primary": True}], "phoneNumbers": [], "addresses": [], "displayName": display_name, @@ -1619,7 +1583,6 @@ def create_initial_user_for_saas_pre_9_1(self, user_email, user_given_name, user "entitlement": entitlement, "givenName": user_given_name, "familyName": user_family_name, - } self.get_or_create_user(user_def) @@ -1651,7 +1614,15 @@ def create_initial_user_for_saas_pre_9_1(self, user_email, user_given_name, user for manage_security_group in manage_security_groups: self.add_user_to_manage_group(user_id, manage_security_group, mxintadm_manage_api_key) - def create_initial_user_for_saas_post_9_1(self, user_email, user_given_name, user_family_name, user_id, user_type, groupreassign=None): + def create_initial_user_for_saas_post_9_1( + self, + user_email, + user_given_name, + user_family_name, + user_id, + user_type, + groupreassign=None, + ): """ Create and fully configure a single initial user for MAS SaaS post 9.1 using the Manage APIs @@ -1701,11 +1672,7 @@ def create_initial_user_for_saas_post_9_1(self, user_email, user_given_name, use "isauthorized": 1, "idpadmin": True, "status": "ACTIVE", - "groupuser": [ - { - "groupname": "USERMANAGEMENT" - } - ] + "groupuser": [{"groupname": "USERMANAGEMENT"}], } manage_security_groups = ["USERMANAGEMENT"] elif user_type == "SECONDARY": @@ -1718,7 +1685,7 @@ def create_initial_user_for_saas_post_9_1(self, user_email, user_given_name, use "apikeyadmin": False, "isauthorized": 0, "idpadmin": False, - "status": "ACTIVE" + "status": "ACTIVE", } manage_security_groups = [] else: @@ -1738,7 +1705,12 @@ def create_initial_user_for_saas_post_9_1(self, user_email, user_given_name, use # For version >= 9.1, we always need a Manage API key and resource_id to link user to local IDP mxintadm_manage_api_key = self.create_or_get_manage_api_key_for_user(MASUserUtils.MXINTADM, temporary=True) - self.link_user_to_local_idp(user_id, email_password=True, manage_api_key=mxintadm_manage_api_key, resource_id=resource_id) + self.link_user_to_local_idp( + user_id, + email_password=True, + manage_api_key=mxintadm_manage_api_key, + resource_id=resource_id, + ) if len(manage_security_groups) > 0 and "manage" in self.mas_workspace_application_ids: if user_type == "PRIMARY" and groupreassign is not None: diff --git a/src/mas/devops/utils.py b/src/mas/devops/utils.py index 6ac821e4..5025a3c4 100644 --- a/src/mas/devops/utils.py +++ b/src/mas/devops/utils.py @@ -35,8 +35,8 @@ def isVersionBefore(_compare_to_version, _current_version): return False strippedVersion = _current_version.split("-")[0] - if '.x' in strippedVersion: - strippedVersion = strippedVersion.replace('.x', '.0') + if ".x" in strippedVersion: + strippedVersion = strippedVersion.replace(".x", ".0") current_version = semver.VersionInfo.parse(strippedVersion) compareToVersion = semver.VersionInfo.parse(_compare_to_version) return current_version.compare(compareToVersion) < 0 @@ -69,8 +69,8 @@ def isVersionEqualOrAfter(_compare_to_version, _current_version): return False strippedVersion = _current_version.split("-")[0] - if '.x' in strippedVersion: - strippedVersion = strippedVersion.replace('.x', '.0') + if ".x" in strippedVersion: + strippedVersion = strippedVersion.replace(".x", ".0") current_version = semver.VersionInfo.parse(strippedVersion) compareToVersion = semver.VersionInfo.parse(_compare_to_version) return current_version.compare(compareToVersion) >= 0 diff --git a/test/src/mock/test_mas_mock.py b/test/src/mock/test_mas_mock.py index a50057b2..fbf7adce 100644 --- a/test/src/mock/test_mas_mock.py +++ b/test/src/mock/test_mas_mock.py @@ -16,11 +16,10 @@ from mas.devops import mas - -CATALOG_ID = 'v9-250101-amd64' -CATALOG_DISPLAY_NAME_VALID = f'IBM Maximo Operators {CATALOG_ID}' -CATALOG_DISPLAY_NAME_INVALID = 'invalidCatalogName' -IMAGE = 'testImage' +CATALOG_ID = "v9-250101-amd64" +CATALOG_DISPLAY_NAME_VALID = f"IBM Maximo Operators {CATALOG_ID}" +CATALOG_DISPLAY_NAME_INVALID = "invalidCatalogName" +IMAGE = "testImage" # ----------------------------------------------------------------------------- @@ -30,7 +29,7 @@ @pytest.fixture(autouse=True) -@mock.patch('openshift.dynamic.DynamicClient') +@mock.patch("openshift.dynamic.DynamicClient") def dynamic_client(client): return client @@ -41,8 +40,9 @@ def test_get_current_catalog_success(dynamic_client): # 2. Create a mock kubernetes resources API and attach the mock catalogsource API resources = MagicMock() - resources.get.side_effect = lambda **kwargs: catalog_api if kwargs['api_version'] == 'operators.coreos.com/v1alpha1' \ - and kwargs['kind'] == 'CatalogSource' else None + resources.get.side_effect = lambda **kwargs: ( + catalog_api if kwargs["api_version"] == "operators.coreos.com/v1alpha1" and kwargs["kind"] == "CatalogSource" else None + ) # 3. Create a mock client using the mock resources API client = dynamic_client() @@ -55,24 +55,26 @@ def test_get_current_catalog_success(dynamic_client): catalog = MagicMock() catalog.spec = spec - catalog_api.get.side_effect = lambda **kwargs: catalog if kwargs['name'] == 'ibm-operator-catalog' \ - and kwargs['namespace'] == 'openshift-marketplace' else None + catalog_api.get.side_effect = lambda **kwargs: ( + catalog if kwargs["name"] == "ibm-operator-catalog" and kwargs["namespace"] == "openshift-marketplace" else None + ) # 5. Call the mock API current_catalog = mas.getCurrentCatalog(client) - assert current_catalog['displayName'] == CATALOG_DISPLAY_NAME_VALID - assert current_catalog['catalogId'] == CATALOG_ID - assert current_catalog['image'] == IMAGE + assert current_catalog["displayName"] == CATALOG_DISPLAY_NAME_VALID + assert current_catalog["catalogId"] == CATALOG_ID + assert current_catalog["image"] == IMAGE def test_get_current_catalog_not_found(dynamic_client): client = dynamic_client() resources = MagicMock() catalog_api = MagicMock() - resources.get.side_effect = lambda **kwargs: catalog_api if kwargs['api_version'] == 'operators.coreos.com/v1alpha1' \ - and kwargs['kind'] == 'CatalogSource' else None + resources.get.side_effect = lambda **kwargs: ( + catalog_api if kwargs["api_version"] == "operators.coreos.com/v1alpha1" and kwargs["kind"] == "CatalogSource" else None + ) client.resources = resources - catalog_api.get.side_effect = NotFoundError(ApiException(status='404')) + catalog_api.get.side_effect = NotFoundError(ApiException(status="404")) assert mas.getCurrentCatalog(client) is None @@ -80,17 +82,19 @@ def test_get_current_catalog_invalid_id(dynamic_client): client = dynamic_client() resources = MagicMock() catalog_api = MagicMock() - resources.get.side_effect = lambda **kwargs: catalog_api if kwargs['api_version'] == 'operators.coreos.com/v1alpha1' \ - and kwargs['kind'] == 'CatalogSource' else None + resources.get.side_effect = lambda **kwargs: ( + catalog_api if kwargs["api_version"] == "operators.coreos.com/v1alpha1" and kwargs["kind"] == "CatalogSource" else None + ) client.resources = resources catalog = MagicMock() - catalog_api.get.side_effect = lambda **kwargs: catalog if kwargs['name'] == 'ibm-operator-catalog' \ - and kwargs['namespace'] == 'openshift-marketplace' else None + catalog_api.get.side_effect = lambda **kwargs: ( + catalog if kwargs["name"] == "ibm-operator-catalog" and kwargs["namespace"] == "openshift-marketplace" else None + ) spec = MagicMock() catalog.spec = spec spec.displayName = CATALOG_DISPLAY_NAME_INVALID spec.image = IMAGE current_catalog = mas.getCurrentCatalog(client) - assert current_catalog['displayName'] == CATALOG_DISPLAY_NAME_INVALID - assert current_catalog['image'] == IMAGE - assert current_catalog['catalogId'] is None + assert current_catalog["displayName"] == CATALOG_DISPLAY_NAME_INVALID + assert current_catalog["image"] == IMAGE + assert current_catalog["catalogId"] is None diff --git a/test/src/saas/test_job_cleaner.py b/test/src/saas/test_job_cleaner.py index 5305a2b8..39b600d0 100644 --- a/test/src/saas/test_job_cleaner.py +++ b/test/src/saas/test_job_cleaner.py @@ -27,17 +27,13 @@ def mock_job(name, namespace, labels, creation_timestamp): mock_job("job-xa-1", "x", {"mas.ibm.com/job-cleanup-group": "a"}, 1), mock_job("job-xa-2", "x", {"mas.ibm.com/job-cleanup-group": "a"}, 2), mock_job("job-xa-3", "x", {"mas.ibm.com/job-cleanup-group": "a"}, 3), - mock_job("job-xb-1", "x", {"mas.ibm.com/job-cleanup-group": "b"}, 1), mock_job("job-xb-2", "x", {"mas.ibm.com/job-cleanup-group": "b"}, 2), - mock_job("job-xc-1", "x", {"mas.ibm.com/job-cleanup-group": "c"}, 2), - mock_job("job-ya-2", "y", {"mas.ibm.com/job-cleanup-group": "a"}, 2), mock_job("job-ya-1", "y", {"mas.ibm.com/job-cleanup-group": "a"}, 1), - mock_job("job-yothera-1", "y", {"otherlabel": "a"}, 1), - mock_job("job-zothera-1", "z", {"otherlabel": "a"}, 1) + mock_job("job-zothera-1", "z", {"otherlabel": "a"}, 1), ] @@ -58,19 +54,14 @@ def filter_func(job): filtered_jobs = list(filter(filter_func, jobs_in_cluster)) - jobs_page = filtered_jobs[_continue:_continue + limit] + jobs_page = filtered_jobs[_continue : _continue + limit] if len(jobs_page) == 0: _continue = None else: _continue = _continue + limit - return Mock( - items=jobs_page, - metadata=Mock( - _continue=_continue - ) - ) + return Mock(items=jobs_page, metadata=Mock(_continue=_continue)) def list_job_for_all_namespaces(label_selector, limit, _continue): @@ -86,7 +77,12 @@ def test_get_all_cleanup_groups(mock_batch_v1_api): mock_batch_v1_api.return_value.list_job_for_all_namespaces.side_effect = list_job_for_all_namespaces jc = JobCleaner(None) for limit in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]: - assert jc._get_all_cleanup_groups("mas.ibm.com/job-cleanup-group", limit) == {('x', 'a'), ('x', 'b'), ('x', 'c'), ('y', 'a')} + assert jc._get_all_cleanup_groups("mas.ibm.com/job-cleanup-group", limit) == { + ("x", "a"), + ("x", "b"), + ("x", "c"), + ("y", "a"), + } @patch("kubernetes.client.BatchV1Api") @@ -94,12 +90,45 @@ def test_get_all_jobs(mock_batch_v1_api): mock_batch_v1_api.return_value.list_namespaced_job.side_effect = list_namespaced_job jc = JobCleaner(None) for limit in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]: - assert list(map(lambda job: job.metadata.name, jc._get_all_jobs("x", "a", "mas.ibm.com/job-cleanup-group", limit))) == ["job-xa-1", "job-xa-2", "job-xa-3"] - assert list(map(lambda job: job.metadata.name, jc._get_all_jobs("x", "b", "mas.ibm.com/job-cleanup-group", limit))) == ["job-xb-1", "job-xb-2"] - assert list(map(lambda job: job.metadata.name, jc._get_all_jobs("x", "c", "mas.ibm.com/job-cleanup-group", limit))) == ["job-xc-1"] - assert list(map(lambda job: job.metadata.name, jc._get_all_jobs("y", "a", "mas.ibm.com/job-cleanup-group", limit))) == ["job-ya-2", "job-ya-1"] - assert list(map(lambda job: job.metadata.name, jc._get_all_jobs("y", "b", "mas.ibm.com/job-cleanup-group", limit))) == [] - assert list(map(lambda job: job.metadata.name, jc._get_all_jobs("y", "a", "otherlabel", limit))) == ["job-yothera-1"] + assert list( + map( + lambda job: job.metadata.name, + jc._get_all_jobs("x", "a", "mas.ibm.com/job-cleanup-group", limit), + ) + ) == ["job-xa-1", "job-xa-2", "job-xa-3"] + assert list( + map( + lambda job: job.metadata.name, + jc._get_all_jobs("x", "b", "mas.ibm.com/job-cleanup-group", limit), + ) + ) == ["job-xb-1", "job-xb-2"] + assert list( + map( + lambda job: job.metadata.name, + jc._get_all_jobs("x", "c", "mas.ibm.com/job-cleanup-group", limit), + ) + ) == ["job-xc-1"] + assert list( + map( + lambda job: job.metadata.name, + jc._get_all_jobs("y", "a", "mas.ibm.com/job-cleanup-group", limit), + ) + ) == ["job-ya-2", "job-ya-1"] + assert ( + list( + map( + lambda job: job.metadata.name, + jc._get_all_jobs("y", "b", "mas.ibm.com/job-cleanup-group", limit), + ) + ) + == [] + ) + assert list( + map( + lambda job: job.metadata.name, + jc._get_all_jobs("y", "a", "otherlabel", limit), + ) + ) == ["job-yothera-1"] @patch("kubernetes.client.BatchV1Api") @@ -114,19 +143,16 @@ def test_cleanup_jobs(mock_batch_v1_api): dry_run_param = "All" expected_calls = [ - call('job-ya-1', 'y', dry_run=dry_run_param, propagation_policy='Foreground'), - call('job-xa-2', 'x', dry_run=dry_run_param, propagation_policy='Foreground'), - call('job-xa-1', 'x', dry_run=dry_run_param, propagation_policy='Foreground'), - call('job-xb-1', 'x', dry_run=dry_run_param, propagation_policy='Foreground'), + call("job-ya-1", "y", dry_run=dry_run_param, propagation_policy="Foreground"), + call("job-xa-2", "x", dry_run=dry_run_param, propagation_policy="Foreground"), + call("job-xa-1", "x", dry_run=dry_run_param, propagation_policy="Foreground"), + call("job-xb-1", "x", dry_run=dry_run_param, propagation_policy="Foreground"), ] for limit in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]: mock_batch_v1_api.return_value.delete_namespaced_job.reset_mock() jc.cleanup_jobs("mas.ibm.com/job-cleanup-group", 3, dry_run) - mock_batch_v1_api.return_value.delete_namespaced_job.assert_has_calls( - expected_calls, - any_order=True - ) + mock_batch_v1_api.return_value.delete_namespaced_job.assert_has_calls(expected_calls, any_order=True) assert mock_batch_v1_api.return_value.delete_namespaced_job.call_count == len(expected_calls) diff --git a/test/src/test_backup.py b/test/src/test_backup.py index 5ae37452..6adcc90e 100644 --- a/test/src/test_backup.py +++ b/test/src/test_backup.py @@ -12,7 +12,13 @@ from unittest.mock import MagicMock, Mock from openshift.dynamic.exceptions import NotFoundError -from mas.devops.backup import createBackupDirectories, copyContentsToYamlFile, filterResourceData, backupResources, extract_secrets_from_dict +from mas.devops.backup import ( + createBackupDirectories, + copyContentsToYamlFile, + filterResourceData, + backupResources, + extract_secrets_from_dict, +) class TestCreateBackupDirectories: @@ -29,11 +35,7 @@ def test_create_single_directory(self, tmp_path): def test_create_multiple_directories(self, tmp_path): """Test creating multiple backup directories""" - test_dirs = [ - tmp_path / "backup1", - tmp_path / "backup2", - tmp_path / "backup3" - ] + test_dirs = [tmp_path / "backup1", tmp_path / "backup2", tmp_path / "backup3"] paths = [str(d) for d in test_dirs] result = createBackupDirectories(paths) @@ -68,7 +70,7 @@ def test_create_empty_list(self): def test_create_directory_permission_error(self, mocker): """Test handling of permission errors""" - mock_makedirs = mocker.patch('os.makedirs', side_effect=PermissionError("Permission denied")) + mock_makedirs = mocker.patch("os.makedirs", side_effect=PermissionError("Permission denied")) result = createBackupDirectories(["/invalid/path"]) @@ -77,7 +79,7 @@ def test_create_directory_permission_error(self, mocker): def test_create_directory_os_error(self, mocker): """Test handling of OS errors""" - mocker.patch('os.makedirs', side_effect=OSError("OS error")) + mocker.patch("os.makedirs", side_effect=OSError("OS error")) result = createBackupDirectories(["/some/path"]) @@ -97,26 +99,19 @@ def test_write_simple_dict(self, tmp_path): assert result is True assert test_file.exists() - with open(test_file, 'r') as f: + with open(test_file, "r") as f: loaded_content = yaml.safe_load(f) assert loaded_content == content def test_write_nested_dict(self, tmp_path): """Test writing a nested dictionary to YAML file""" test_file = tmp_path / "nested.yaml" - content = { - "level1": { - "level2": { - "level3": "value" - } - }, - "list": [1, 2, 3] - } + content = {"level1": {"level2": {"level3": "value"}}, "list": [1, 2, 3]} result = copyContentsToYamlFile(str(test_file), content) assert result is True - with open(test_file, 'r') as f: + with open(test_file, "r") as f: loaded_content = yaml.safe_load(f) assert loaded_content == content @@ -128,7 +123,7 @@ def test_write_empty_dict(self, tmp_path): result = copyContentsToYamlFile(str(test_file), content) assert result is True - with open(test_file, 'r') as f: + with open(test_file, "r") as f: loaded_content = yaml.safe_load(f) assert loaded_content == content @@ -139,14 +134,14 @@ def test_overwrite_existing_file(self, tmp_path): new_content = {"new": "data"} # Write initial content - with open(test_file, 'w') as f: + with open(test_file, "w") as f: yaml.dump(old_content, f) # Overwrite with new content result = copyContentsToYamlFile(str(test_file), new_content) assert result is True - with open(test_file, 'r') as f: + with open(test_file, "r") as f: loaded_content = yaml.safe_load(f) assert loaded_content == new_content assert loaded_content != old_content @@ -162,7 +157,7 @@ def test_write_to_nonexistent_directory(self, tmp_path): def test_write_permission_error(self, mocker): """Test handling of permission errors during write""" - mocker.patch('builtins.open', side_effect=PermissionError("Permission denied")) + mocker.patch("builtins.open", side_effect=PermissionError("Permission denied")) result = copyContentsToYamlFile("/invalid/path.yaml", {"key": "value"}) @@ -174,13 +169,13 @@ def test_write_with_special_characters(self, tmp_path): content = { "special": "value with\nnewlines", "unicode": "café ☕", - "quotes": "value with 'quotes' and \"double quotes\"" + "quotes": "value with 'quotes' and \"double quotes\"", } result = copyContentsToYamlFile(str(test_file), content) assert result is True - with open(test_file, 'r') as f: + with open(test_file, "r") as f: loaded_content = yaml.safe_load(f) assert loaded_content == content @@ -202,9 +197,9 @@ def test_filter_all_metadata_fields(self): "resourceVersion": "12345", "selfLink": "/api/v1/namespaces/test/resources/test-resource", "uid": "abc-123-def", - "managedFields": [{"manager": "test"}] + "managedFields": [{"manager": "test"}], }, - "spec": {"replicas": 3} + "spec": {"replicas": 3}, } result = filterResourceData(data) @@ -225,10 +220,7 @@ def test_filter_status_field(self): data = { "metadata": {"name": "test"}, "spec": {"replicas": 3}, - "status": { - "phase": "Running", - "conditions": [] - } + "status": {"phase": "Running", "conditions": []}, } result = filterResourceData(data) @@ -243,7 +235,7 @@ def test_filter_partial_metadata(self): "metadata": { "name": "test-resource", "uid": "abc-123", - "labels": {"app": "test"} + "labels": {"app": "test"}, } } @@ -255,11 +247,7 @@ def test_filter_partial_metadata(self): def test_filter_no_metadata(self): """Test filtering when metadata field is not present""" - data = { - "apiVersion": "v1", - "kind": "Resource", - "spec": {"replicas": 3} - } + data = {"apiVersion": "v1", "kind": "Resource", "spec": {"replicas": 3}} result = filterResourceData(data) @@ -269,10 +257,7 @@ def test_filter_no_metadata(self): def test_filter_empty_metadata(self): """Test filtering with empty metadata""" - data = { - "metadata": {}, - "spec": {"replicas": 3} - } + data = {"metadata": {}, "spec": {"replicas": 3}} result = filterResourceData(data) @@ -284,14 +269,8 @@ def test_filter_preserves_other_fields(self): data = { "apiVersion": "v1", "kind": "ConfigMap", - "metadata": { - "name": "test-config", - "uid": "should-be-removed" - }, - "data": { - "key1": "value1", - "key2": "value2" - } + "metadata": {"name": "test-config", "uid": "should-be-removed"}, + "data": {"key1": "value1", "key2": "value2"}, } result = filterResourceData(data) @@ -304,11 +283,8 @@ def test_filter_preserves_other_fields(self): def test_filter_shallow_copy_behavior(self): """Test that filterResourceData uses shallow copy (modifies nested dicts)""" data = { - "metadata": { - "name": "test", - "uid": "abc-123" - }, - "status": {"phase": "Running"} + "metadata": {"name": "test", "uid": "abc-123"}, + "status": {"phase": "Running"}, } result = filterResourceData(data) @@ -336,16 +312,10 @@ def test_filter_complex_resource(self): "generation": 5, "resourceVersion": "98765", "uid": "xyz-789", - "managedFields": [{"manager": "kubectl"}] - }, - "spec": { - "replicas": 3, - "selector": {"matchLabels": {"app": "myapp"}} + "managedFields": [{"manager": "kubectl"}], }, - "status": { - "availableReplicas": 3, - "readyReplicas": 3 - } + "spec": {"replicas": 3, "selector": {"matchLabels": {"app": "myapp"}}}, + "status": {"availableReplicas": 3, "readyReplicas": 3}, } result = filterResourceData(data) @@ -379,11 +349,7 @@ class TestExtractSecretsFromDict: def test_extract_single_secret(self): """Test extracting a single secret name""" - data = { - "spec": { - "secretName": "my-secret" - } - } + data = {"spec": {"secretName": "my-secret"}} result = extract_secrets_from_dict(data) assert result == {"my-secret"} @@ -391,12 +357,8 @@ def test_extract_multiple_secrets(self): """Test extracting multiple secret names""" data = { "spec": { - "database": { - "secretName": "db-secret" - }, - "auth": { - "secretName": "auth-secret" - } + "database": {"secretName": "db-secret"}, + "auth": {"secretName": "auth-secret"}, } } result = extract_secrets_from_dict(data) @@ -409,7 +371,7 @@ def test_extract_secrets_from_list(self): "volumes": [ {"secretName": "secret1"}, {"secretName": "secret2"}, - {"configMap": "not-a-secret"} + {"configMap": "not-a-secret"}, ] } } @@ -418,26 +380,13 @@ def test_extract_secrets_from_list(self): def test_extract_nested_secrets(self): """Test extracting deeply nested secrets""" - data = { - "level1": { - "level2": { - "level3": { - "secretName": "deep-secret" - } - } - } - } + data = {"level1": {"level2": {"level3": {"secretName": "deep-secret"}}}} result = extract_secrets_from_dict(data) assert result == {"deep-secret"} def test_no_secrets_found(self): """Test when no secrets are present""" - data = { - "spec": { - "replicas": 3, - "image": "myapp:latest" - } - } + data = {"spec": {"replicas": 3, "image": "myapp:latest"}} result = extract_secrets_from_dict(data) assert result == set() @@ -448,27 +397,13 @@ def test_empty_dict(self): def test_ignore_empty_secret_name(self): """Test that empty string secret names are ignored""" - data = { - "spec": { - "secretName": "", - "other": { - "secretName": "valid-secret" - } - } - } + data = {"spec": {"secretName": "", "other": {"secretName": "valid-secret"}}} result = extract_secrets_from_dict(data) assert result == {"valid-secret"} def test_ignore_non_string_secret_name(self): """Test that non-string secret names are ignored""" - data = { - "spec": { - "secretName": 123, - "other": { - "secretName": "valid-secret" - } - } - } + data = {"spec": {"secretName": 123, "other": {"secretName": "valid-secret"}}} result = extract_secrets_from_dict(data) assert result == {"valid-secret"} @@ -478,7 +413,7 @@ def test_duplicate_secrets(self): "spec": { "volume1": {"secretName": "shared-secret"}, "volume2": {"secretName": "shared-secret"}, - "volume3": {"secretName": "unique-secret"} + "volume3": {"secretName": "unique-secret"}, } } result = extract_secrets_from_dict(data) @@ -497,9 +432,9 @@ def test_backup_single_namespaced_resource(self, tmp_path, mocker): "metadata": { "name": "test-resource", "namespace": "test-ns", - "uid": "abc-123" + "uid": "abc-123", }, - "spec": {"replicas": 3} + "spec": {"replicas": 3}, } # Create mock resource object with to_dict method @@ -514,7 +449,7 @@ def test_backup_single_namespaced_resource(self, tmp_path, mocker): mock_client.resources.get.return_value = mock_api # Mock the helper functions - mocker.patch('mas.devops.backup.copyContentsToYamlFile', return_value=True) + mocker.patch("mas.devops.backup.copyContentsToYamlFile", return_value=True) result = backupResources( mock_client, @@ -522,7 +457,7 @@ def test_backup_single_namespaced_resource(self, tmp_path, mocker): api_version="v1", backup_path=backup_path, namespace="test-ns", - name="test-resource" + name="test-resource", ) backed_up, not_found, failed, secrets = result @@ -539,12 +474,12 @@ def test_backup_multiple_namespaced_resources(self, tmp_path, mocker): mock_resources = [ { "metadata": {"name": "resource1", "namespace": "test-ns"}, - "spec": {"data": "value1"} + "spec": {"data": "value1"}, }, { "metadata": {"name": "resource2", "namespace": "test-ns"}, - "spec": {"data": "value2"} - } + "spec": {"data": "value2"}, + }, ] # Create mock resource objects @@ -565,14 +500,14 @@ def test_backup_multiple_namespaced_resources(self, tmp_path, mocker): mock_api.get.return_value = mock_response mock_client.resources.get.return_value = mock_api - mocker.patch('mas.devops.backup.copyContentsToYamlFile', return_value=True) + mocker.patch("mas.devops.backup.copyContentsToYamlFile", return_value=True) result = backupResources( mock_client, kind="ConfigMap", api_version="v1", backup_path=backup_path, - namespace="test-ns" + namespace="test-ns", ) backed_up, not_found, failed, secrets = result @@ -586,7 +521,7 @@ def test_backup_cluster_level_resource(self, tmp_path, mocker): mock_resource = { "metadata": {"name": "cluster-role"}, - "rules": [{"apiGroups": ["*"], "resources": ["*"], "verbs": ["*"]}] + "rules": [{"apiGroups": ["*"], "resources": ["*"], "verbs": ["*"]}], } mock_resource_obj = MagicMock() @@ -598,14 +533,14 @@ def test_backup_cluster_level_resource(self, tmp_path, mocker): mock_api.get.return_value = mock_resource_obj mock_client.resources.get.return_value = mock_api - mocker.patch('mas.devops.backup.copyContentsToYamlFile', return_value=True) + mocker.patch("mas.devops.backup.copyContentsToYamlFile", return_value=True) result = backupResources( mock_client, kind="ClusterRole", api_version="rbac.authorization.k8s.io/v1", backup_path=backup_path, - name="cluster-role" + name="cluster-role", ) backed_up, not_found, failed, secrets = result @@ -621,9 +556,9 @@ def test_backup_with_label_selector(self, tmp_path, mocker): "metadata": { "name": "labeled-resource", "namespace": "test-ns", - "labels": {"app": "myapp", "env": "prod"} + "labels": {"app": "myapp", "env": "prod"}, }, - "spec": {} + "spec": {}, } mock_resource_obj = MagicMock() @@ -638,7 +573,7 @@ def test_backup_with_label_selector(self, tmp_path, mocker): mock_api.get.return_value = mock_response mock_client.resources.get.return_value = mock_api - mocker.patch('mas.devops.backup.copyContentsToYamlFile', return_value=True) + mocker.patch("mas.devops.backup.copyContentsToYamlFile", return_value=True) result = backupResources( mock_client, @@ -646,7 +581,7 @@ def test_backup_with_label_selector(self, tmp_path, mocker): api_version="v1", backup_path=backup_path, namespace="test-ns", - labels=["app=myapp", "env=prod"] + labels=["app=myapp", "env=prod"], ) backed_up, not_found, failed, secrets = result @@ -670,7 +605,7 @@ def test_backup_resource_not_found_by_name(self, mocker): api_version="v1", backup_path="/tmp/backup", namespace="test-ns", - name="nonexistent" + name="nonexistent", ) backed_up, not_found, failed, secrets = result @@ -694,7 +629,7 @@ def test_backup_no_resources_found(self, mocker): kind="ConfigMap", api_version="v1", backup_path="/tmp/backup", - namespace="test-ns" + namespace="test-ns", ) backed_up, not_found, failed, secrets = result @@ -713,11 +648,11 @@ def test_backup_discovers_secrets(self, tmp_path, mocker): "spec": { "volumes": [ {"secretName": "db-credentials"}, - {"secretName": "api-key"} + {"secretName": "api-key"}, ] } } - } + }, } mock_resource_obj = MagicMock() @@ -729,7 +664,7 @@ def test_backup_discovers_secrets(self, tmp_path, mocker): mock_api.get.return_value = mock_resource_obj mock_client.resources.get.return_value = mock_api - mocker.patch('mas.devops.backup.copyContentsToYamlFile', return_value=True) + mocker.patch("mas.devops.backup.copyContentsToYamlFile", return_value=True) result = backupResources( mock_client, @@ -737,7 +672,7 @@ def test_backup_discovers_secrets(self, tmp_path, mocker): api_version="apps/v1", backup_path=backup_path, namespace="test-ns", - name="app-deployment" + name="app-deployment", ) backed_up, not_found, failed, secrets = result @@ -750,7 +685,7 @@ def test_backup_secret_does_not_discover_itself(self, tmp_path, mocker): mock_resource = { "metadata": {"name": "my-secret", "namespace": "test-ns"}, - "data": {"password": "encoded-value"} + "data": {"password": "encoded-value"}, } mock_resource_obj = MagicMock() @@ -762,7 +697,7 @@ def test_backup_secret_does_not_discover_itself(self, tmp_path, mocker): mock_api.get.return_value = mock_resource_obj mock_client.resources.get.return_value = mock_api - mocker.patch('mas.devops.backup.copyContentsToYamlFile', return_value=True) + mocker.patch("mas.devops.backup.copyContentsToYamlFile", return_value=True) result = backupResources( mock_client, @@ -770,7 +705,7 @@ def test_backup_secret_does_not_discover_itself(self, tmp_path, mocker): api_version="v1", backup_path=backup_path, namespace="test-ns", - name="my-secret" + name="my-secret", ) backed_up, not_found, failed, secrets = result @@ -783,7 +718,7 @@ def test_backup_write_failure(self, tmp_path, mocker): mock_resource = { "metadata": {"name": "test-resource", "namespace": "test-ns"}, - "spec": {} + "spec": {}, } mock_resource_obj = MagicMock() @@ -796,7 +731,7 @@ def test_backup_write_failure(self, tmp_path, mocker): mock_client.resources.get.return_value = mock_api # Mock copyContentsToYamlFile to fail - mocker.patch('mas.devops.backup.copyContentsToYamlFile', return_value=False) + mocker.patch("mas.devops.backup.copyContentsToYamlFile", return_value=False) result = backupResources( mock_client, @@ -804,7 +739,7 @@ def test_backup_write_failure(self, tmp_path, mocker): api_version="v1", backup_path=backup_path, namespace="test-ns", - name="test-resource" + name="test-resource", ) backed_up, not_found, failed, secrets = result @@ -822,7 +757,7 @@ def test_backup_api_exception(self, mocker): kind="ConfigMap", api_version="v1", backup_path="/tmp/backup", - namespace="test-ns" + namespace="test-ns", ) backed_up, not_found, failed, secrets = result @@ -835,18 +770,9 @@ def test_backup_mixed_success_and_failure(self, tmp_path, mocker): backup_path = str(tmp_path / "backup") mock_resources = [ - { - "metadata": {"name": "resource1", "namespace": "test-ns"}, - "spec": {} - }, - { - "metadata": {"name": "resource2", "namespace": "test-ns"}, - "spec": {} - }, - { - "metadata": {"name": "resource3", "namespace": "test-ns"}, - "spec": {} - } + {"metadata": {"name": "resource1", "namespace": "test-ns"}, "spec": {}}, + {"metadata": {"name": "resource2", "namespace": "test-ns"}, "spec": {}}, + {"metadata": {"name": "resource3", "namespace": "test-ns"}, "spec": {}}, ] mock_resource_objs = [] @@ -865,7 +791,7 @@ def test_backup_mixed_success_and_failure(self, tmp_path, mocker): mock_client.resources.get.return_value = mock_api # Mock copyContentsToYamlFile to succeed for first two, fail for third - mock_copy = mocker.patch('mas.devops.backup.copyContentsToYamlFile') + mock_copy = mocker.patch("mas.devops.backup.copyContentsToYamlFile") mock_copy.side_effect = [True, True, False] result = backupResources( @@ -873,7 +799,7 @@ def test_backup_mixed_success_and_failure(self, tmp_path, mocker): kind="ConfigMap", api_version="v1", backup_path=backup_path, - namespace="test-ns" + namespace="test-ns", ) backed_up, not_found, failed, secrets = result @@ -891,7 +817,7 @@ def test_backup_resource_kind_not_found(self, mocker): kind="NonExistentKind", api_version="v1", backup_path="/tmp/backup", - namespace="test-ns" + namespace="test-ns", ) backed_up, not_found, failed, secrets = result diff --git a/test/src/test_data.py b/test/src/test_data.py index 653a3b61..ce3a25cb 100644 --- a/test/src/test_data.py +++ b/test/src/test_data.py @@ -36,7 +36,10 @@ def test_get_newest_catalog_tag(): def test_get_newest_catalog_tag_fail(): - with pytest.raises(NoSuchCatalogError, match="There are no known catalogs for the doesntexist platform"): + with pytest.raises( + NoSuchCatalogError, + match="There are no known catalogs for the doesntexist platform", + ): getNewestCatalogTag("doesntexist") diff --git a/test/src/test_db2.py b/test/src/test_db2.py index 8d757606..3c37d43c 100644 --- a/test/src/test_db2.py +++ b/test/src/test_db2.py @@ -16,56 +16,56 @@ from mas.devops import db2 -@pytest.mark.parametrize("cr_k,cr_v,pod_v,expected", [ - ("MIRRORLOGPATH", "/mnt/backup/MIRRORLOGPATH", "/mnt/backup/MIRRORLOGPATH/NODE0000/LOGSTREAM0000/", True), - ("MIRRORLOGPATH", "/mnt/backup/MIRRORLOGPATH", "/notcorrect/NODE0000/LOGSTREAM0000/", False), - ("NOTSPECIAL", "/mnt/backup/MIRRORLOGPATH", "/mnt/backup/MIRRORLOGPATH/NODE0000/LOGSTREAM0000/", False), - - ("X", "22 AUTOMATIC", "AUTOMATIC(22)", True), - ("X", "22 AUTOMATIC", "AUTOMATIC(44)", False), - ("X", "AUTOMATIC", "AUTOMATIC(22)", True), - ("X", "automatic", "AUTOMATIC(22)", True), - ("X", "automatic", "AUTOMATIC", True), - ("X", "automaticx", "AUTOMATIC", False), - - ("X", "otherSTRING", "otherSTRING", True), - ("X", "otherSTRING", "OTHERstring", False), - - ("X", "other string", "other string", True), - - ("X", "22", "22", True), - -]) +@pytest.mark.parametrize( + "cr_k,cr_v,pod_v,expected", + [ + ( + "MIRRORLOGPATH", + "/mnt/backup/MIRRORLOGPATH", + "/mnt/backup/MIRRORLOGPATH/NODE0000/LOGSTREAM0000/", + True, + ), + ( + "MIRRORLOGPATH", + "/mnt/backup/MIRRORLOGPATH", + "/notcorrect/NODE0000/LOGSTREAM0000/", + False, + ), + ( + "NOTSPECIAL", + "/mnt/backup/MIRRORLOGPATH", + "/mnt/backup/MIRRORLOGPATH/NODE0000/LOGSTREAM0000/", + False, + ), + ("X", "22 AUTOMATIC", "AUTOMATIC(22)", True), + ("X", "22 AUTOMATIC", "AUTOMATIC(44)", False), + ("X", "AUTOMATIC", "AUTOMATIC(22)", True), + ("X", "automatic", "AUTOMATIC(22)", True), + ("X", "automatic", "AUTOMATIC", True), + ("X", "automaticx", "AUTOMATIC", False), + ("X", "otherSTRING", "otherSTRING", True), + ("X", "otherSTRING", "OTHERstring", False), + ("X", "other string", "other string", True), + ("X", "22", "22", True), + ], +) def test_cr_pod_v_matches(cr_k, cr_v, pod_v, expected): - assert (db2.cr_pod_v_matches(cr_k, cr_v, pod_v) is expected) + assert db2.cr_pod_v_matches(cr_k, cr_v, pod_v) is expected def test_check_db_cfgs_no_spec(): with pytest.raises(Exception, match="spec.environment.databases not found or empty"): - db2.check_db_cfgs( - dict( - ), None, None, None - ) + db2.check_db_cfgs(dict(), None, None, None) def test_check_db_cfgs_no_environment(): with pytest.raises(Exception, match="spec.environment.databases not found or empty"): - db2.check_db_cfgs( - dict( - spec=dict() - ), None, None, None - ) + db2.check_db_cfgs(dict(spec=dict()), None, None, None) def test_check_db_cfgs_no_databases(): with pytest.raises(Exception, match="spec.environment.databases not found or empty"): - db2.check_db_cfgs( - dict( - spec=dict( - environment=dict() - ) - ), None, None, None - ) + db2.check_db_cfgs(dict(spec=dict(environment=dict())), None, None, None) def test_check_db_cfg_no_dbConfig(mocker): @@ -79,115 +79,87 @@ def test_check_db_cfg_empty_dbConfig(mocker): def test_check_db_cfgs(mocker): - ''' + """ Verifies that check_db_cfg function is called for each db in list - ''' + """ - mock_CoreV1Api = mocker.patch('kubernetes.client.CoreV1Api') + mock_CoreV1Api = mocker.patch("kubernetes.client.CoreV1Api") mock_core_v1_api = mock_CoreV1Api.return_value mock_check_db_cfg = mocker.patch("mas.devops.db2.check_db_cfg") db2.check_db_cfgs( - dict( - spec=dict( - environment=dict( - databases=[ - dict(name="a"), dict(name="b") - ] - ) - ) - ), mock_core_v1_api, "mas_instance_id", "mas_app_id" + dict(spec=dict(environment=dict(databases=[dict(name="a"), dict(name="b")]))), + mock_core_v1_api, + "mas_instance_id", + "mas_app_id", ) assert mock_check_db_cfg.call_args_list == [ mocker.call(dict(name="a"), mock_core_v1_api, "mas_instance_id", "mas_app_id", "primary"), - mocker.call(dict(name="b"), mock_core_v1_api, "mas_instance_id", "mas_app_id", "primary") + mocker.call(dict(name="b"), mock_core_v1_api, "mas_instance_id", "mas_app_id", "primary"), ] def test_check_db_cfg(mocker): mock_db2_pod_exec_db2_get_db_cfg = mocker.patch("mas.devops.db2.db2_pod_exec_db2_get_db_cfg") - mock_db2_pod_exec_db2_get_db_cfg.return_value = ''' + mock_db2_pod_exec_db2_get_db_cfg.return_value = """ Default application heap (4KB) (APPLHEAPSZ) = AUTOMATIC(8192) Changed pages threshold (CHNGPGS_THRESH) = 80 - ''' + """ db_name = "MYDB" db = dict( name=db_name, - dbConfig=dict( - APPLHEAPSZ="8192 AUTOMATIC", - NOTFOUNDINOUTPUT="XXX", - CHNGPGS_THRESH="40" - ) + dbConfig=dict(APPLHEAPSZ="8192 AUTOMATIC", NOTFOUNDINOUTPUT="XXX", CHNGPGS_THRESH="40"), ) - assert set(db2.check_db_cfg(db, None, None, None)) == set([ - f"[db cfg for {db_name}] NOTFOUNDINOUTPUT not found in output of db2 get db cfg command", - f"[db cfg for {db_name}] CHNGPGS_THRESH: 40 != 80" - ]) + assert set(db2.check_db_cfg(db, None, None, None)) == set( + [ + f"[db cfg for {db_name}] NOTFOUNDINOUTPUT not found in output of db2 get db cfg command", + f"[db cfg for {db_name}] CHNGPGS_THRESH: 40 != 80", + ] + ) def test_check_dbm_cfg_no_spec(): - db2_instance_cr = dict( - ) + db2_instance_cr = dict() assert db2.check_dbm_cfg(db2_instance_cr, None, None, None) == [] def test_check_dbm_cfg_no_environment(): - db2_instance_cr = dict( - spec=dict() - ) + db2_instance_cr = dict(spec=dict()) assert db2.check_dbm_cfg(db2_instance_cr, None, None, None) == [] def test_check_dbm_cfg_no_instance(): - db2_instance_cr = dict( - spec=dict( - environment=dict() - ) - ) + db2_instance_cr = dict(spec=dict(environment=dict())) assert db2.check_dbm_cfg(db2_instance_cr, None, None, None) == [] def test_check_dbm_cfg_no_dbmConfig(): - db2_instance_cr = dict( - spec=dict( - environment=dict( - instance=dict() - ) - ) - ) + db2_instance_cr = dict(spec=dict(environment=dict(instance=dict()))) assert db2.check_dbm_cfg(db2_instance_cr, None, None, None) == [] def test_check_dbm_cfg_empty_dbmConfig(): - db2_instance_cr = dict( - spec=dict( - environment=dict( - instance=dict( - dbmConfig=dict() - ) - ) - ) - ) + db2_instance_cr = dict(spec=dict(environment=dict(instance=dict(dbmConfig=dict())))) assert db2.check_dbm_cfg(db2_instance_cr, None, None, None) == [] def test_check_dbm_cfg(mocker): mock_db2_pod_exec_db2_get_dbm_cfg = mocker.patch("mas.devops.db2.db2_pod_exec_db2_get_dbm_cfg") - mock_db2_pod_exec_db2_get_dbm_cfg.return_value = ''' + mock_db2_pod_exec_db2_get_dbm_cfg.return_value = """ Agent stack size (AGENT_STACK_SZ) = 1024 - ''' + """ db2_instance_cr = dict( spec=dict( environment=dict( instance=dict( dbmConfig=dict( - AGENT_STACK_SZ='2048', + AGENT_STACK_SZ="2048", NOTFOUNDINOUTPUT="XXX", ) ) @@ -195,72 +167,53 @@ def test_check_dbm_cfg(mocker): ) ) - assert set(db2.check_dbm_cfg(db2_instance_cr, None, None, None)) == set([ - "[dbm cfg] NOTFOUNDINOUTPUT not found in output of db2 get dbm cfg command", - "[dbm cfg] AGENT_STACK_SZ: 2048 != 1024" - ]) + assert set(db2.check_dbm_cfg(db2_instance_cr, None, None, None)) == set( + [ + "[dbm cfg] NOTFOUNDINOUTPUT not found in output of db2 get dbm cfg command", + "[dbm cfg] AGENT_STACK_SZ: 2048 != 1024", + ] + ) def test_check_reg_cfg_no_spec(): - db2_instance_cr = dict( - ) + db2_instance_cr = dict() assert db2.check_reg_cfg(db2_instance_cr, None, None, None) == [] def test_check_reg_cfg_no_environment(): - db2_instance_cr = dict( - spec=dict() - ) + db2_instance_cr = dict(spec=dict()) assert db2.check_reg_cfg(db2_instance_cr, None, None, None) == [] def test_check_reg_cfg_no_instance(): - db2_instance_cr = dict( - spec=dict( - environment=dict() - ) - ) + db2_instance_cr = dict(spec=dict(environment=dict())) assert db2.check_reg_cfg(db2_instance_cr, None, None, None) == [] def test_check_reg_cfg_no_registry(): - db2_instance_cr = dict( - spec=dict( - environment=dict( - instance=dict() - ) - ) - ) + db2_instance_cr = dict(spec=dict(environment=dict(instance=dict()))) assert db2.check_reg_cfg(db2_instance_cr, None, None, None) == [] def test_check_reg_cfg_empty_registry(): - db2_instance_cr = dict( - spec=dict( - environment=dict( - instance=dict( - registry=dict() - ) - ) - ) - ) + db2_instance_cr = dict(spec=dict(environment=dict(instance=dict(registry=dict())))) assert db2.check_reg_cfg(db2_instance_cr, None, None, None) == [] def test_check_reg_cfg(mocker): mock_db2_pod_exec_db2set = mocker.patch("mas.devops.db2.db2_pod_exec_db2set") - mock_db2_pod_exec_db2set.return_value = ''' + mock_db2_pod_exec_db2set.return_value = """ DB2AUTH=OSAUTHDB,ALLOW_LOCAL_FALLBACK,PLUGIN_AUTO_RELOAD DB2_FMP_COMM_HEAPSZ=65536 [O] - ''' + """ db2_instance_cr = dict( spec=dict( environment=dict( instance=dict( registry=dict( - DB2AUTH='WRONG', + DB2AUTH="WRONG", NOTFOUNDINOUTPUT="XXX", ) ) @@ -268,26 +221,28 @@ def test_check_reg_cfg(mocker): ) ) - assert set(db2.check_reg_cfg(db2_instance_cr, None, None, None)) == set([ - "[registry cfg] NOTFOUNDINOUTPUT not found in output of db2set command", - "[registry cfg] DB2AUTH: WRONG != OSAUTHDB,ALLOW_LOCAL_FALLBACK,PLUGIN_AUTO_RELOAD" - ]) + assert set(db2.check_reg_cfg(db2_instance_cr, None, None, None)) == set( + [ + "[registry cfg] NOTFOUNDINOUTPUT not found in output of db2set command", + "[registry cfg] DB2AUTH: WRONG != OSAUTHDB,ALLOW_LOCAL_FALLBACK,PLUGIN_AUTO_RELOAD", + ] + ) def test_check_reg_cfg_with_empty_value_in_cr(mocker): mock_db2_pod_exec_db2set = mocker.patch("mas.devops.db2.db2_pod_exec_db2set") - mock_db2_pod_exec_db2set.return_value = ''' + mock_db2_pod_exec_db2set.return_value = """ DB2AUTH=OSAUTHDB,ALLOW_LOCAL_FALLBACK,PLUGIN_AUTO_RELOAD DB2_FMP_COMM_HEAPSZ=65536 [O] - ''' + """ db2_instance_cr = dict( spec=dict( environment=dict( instance=dict( registry=dict( - DB2AUTH='WRONG', + DB2AUTH="WRONG", NOTFOUNDINOUTPUT="", ) ) @@ -295,98 +250,108 @@ def test_check_reg_cfg_with_empty_value_in_cr(mocker): ) ) - assert set(db2.check_reg_cfg(db2_instance_cr, None, None, None)) == set([ - "[registry cfg] DB2AUTH: WRONG != OSAUTHDB,ALLOW_LOCAL_FALLBACK,PLUGIN_AUTO_RELOAD" - ]) - + assert set(db2.check_reg_cfg(db2_instance_cr, None, None, None)) == set( + ["[registry cfg] DB2AUTH: WRONG != OSAUTHDB,ALLOW_LOCAL_FALLBACK,PLUGIN_AUTO_RELOAD"] + ) -@pytest.mark.parametrize("test_case_name, expected_failures", [ - # This test case simulates what will happen when we run the validate_db2_config using the IoT Db2uInstance CR - # as we have it today in fvtsaas after the CR settings have been applied successfully to DB2 - # (there are currently no custom settings set for IoT, so the validate should skip all the checks) - ( - "iot", [ - ] - ), - # This test case simulates what will happen when we run the validate_db2_config using the Manage Db2uInstance CR - # as we have it today in fvtsaas after the CR settings have been applied successfully to DB2 - ( - "manage_pass", [ - ] - ), - - # This test case simulates what will happen when we run the validate_db2_config using the Manage Db2uInstance CR - # as we have it today in fvtsaas against default DB2 configuration values - ("manage_fail", [ - "[db cfg for BLUDB] APPLHEAPSZ: WRONG != AUTOMATIC(256)", - "[db cfg for BLUDB] AUTHN_CACHE_DURATION: 10 != 3", - "[db cfg for BLUDB] AUTHN_CACHE_USERS: 100 != 0", - "[db cfg for BLUDB] AUTO_REORG: OFF != ON", - "[db cfg for BLUDB] CATALOGCACHE_SZ: 800 != 742", - "[db cfg for BLUDB] CHNGPGS_THRESH: 40 != 80", - "[db cfg for BLUDB] DDL_CONSTRAINT_DEF: YES != NO", - "[db cfg for BLUDB] LOCKTIMEOUT: 300 != -1", - "[db cfg for BLUDB] LOGBUFSZ: 1024 != 2152", - "[db cfg for BLUDB] LOGFILSIZ: 32768 != 50000", - "[db cfg for BLUDB] LOGPRIMARY: 100 != 20", - "[db cfg for BLUDB] LOGSECOND: 156 != 30", - "[db cfg for BLUDB] MIRRORLOGPATH: /mnt/backup != ", - "[db cfg for BLUDB] NUM_DB_BACKUPS: 60 != 2", - "[db cfg for BLUDB] REC_HIS_RETENTN: 60 != 0", - "[db cfg for BLUDB] SHEAPTHRES_SHR: automatic != 1336548", - "[db cfg for BLUDB] SORTHEAP: automatic != 66827", - "[db cfg for BLUDB] STMTHEAP: 20000 != AUTOMATIC(16384)", - "[db cfg for BLUDB] STMT_CONC: LITERALS != OFF", - "[db cfg for BLUDB] WLM_ADMISSION_CTRL: NO != YES", - "[dbm cfg] AGENT_STACK_SZ: WRONG != 1024", - "[dbm cfg] FENCED_POOL: 50 != AUTOMATIC(MAX_COORDAGENTS)", - "[dbm cfg] KEEPFENCED: NO != YES", - "[registry cfg] DB2AUTH: WRONG != OSAUTHDB", - "[registry cfg] DB2_4K_DEVICE_SUPPORT not found in output of db2set command", - "[registry cfg] DB2_BCKP_PAGE_VERIFICATION not found in output of db2set command", - "[registry cfg] DB2_CDE_REDUCED_LOGGING not found in output of db2set command", - "[registry cfg] DB2_EVALUNCOMMITTED not found in output of db2set command", - "[registry cfg] DB2_FMP_COMM_HEAPSZ not found in output of db2set command", - "[registry cfg] DB2_FMP_RUN_AS_CONNECTED_USER: NO != YES", - "[registry cfg] DB2_INLIST_TO_NLJN not found in output of db2set command", - "[registry cfg] DB2_MINIMIZE_LISTPREFETCH not found in output of db2set command", - "[registry cfg] DB2_OBJECT_STORAGE_LOCAL_STAGING_PATH: /mnt/backup/staging != /mnt/bludata0/scratch/db2/RemoteStorage", - "[registry cfg] DB2_SKIPDELETED not found in output of db2set command", - "[registry cfg] DB2_SKIPINSERTED not found in output of db2set command", - "[registry cfg] DB2_USE_ALTERNATE_PAGE_CLEANING: ON != ON [DB2_WORKLOAD]", - "[registry cfg] DB2_WORKLOAD: MAXIMO != ANALYTICS", - ]), -]) +@pytest.mark.parametrize( + "test_case_name, expected_failures", + [ + # This test case simulates what will happen when we run the validate_db2_config using the IoT Db2uInstance CR + # as we have it today in fvtsaas after the CR settings have been applied successfully to DB2 + # (there are currently no custom settings set for IoT, so the validate should skip all the checks) + ("iot", []), + # This test case simulates what will happen when we run the validate_db2_config using the Manage Db2uInstance CR + # as we have it today in fvtsaas after the CR settings have been applied successfully to DB2 + ("manage_pass", []), + # This test case simulates what will happen when we run the validate_db2_config using the Manage Db2uInstance CR + # as we have it today in fvtsaas against default DB2 configuration values + ( + "manage_fail", + [ + "[db cfg for BLUDB] APPLHEAPSZ: WRONG != AUTOMATIC(256)", + "[db cfg for BLUDB] AUTHN_CACHE_DURATION: 10 != 3", + "[db cfg for BLUDB] AUTHN_CACHE_USERS: 100 != 0", + "[db cfg for BLUDB] AUTO_REORG: OFF != ON", + "[db cfg for BLUDB] CATALOGCACHE_SZ: 800 != 742", + "[db cfg for BLUDB] CHNGPGS_THRESH: 40 != 80", + "[db cfg for BLUDB] DDL_CONSTRAINT_DEF: YES != NO", + "[db cfg for BLUDB] LOCKTIMEOUT: 300 != -1", + "[db cfg for BLUDB] LOGBUFSZ: 1024 != 2152", + "[db cfg for BLUDB] LOGFILSIZ: 32768 != 50000", + "[db cfg for BLUDB] LOGPRIMARY: 100 != 20", + "[db cfg for BLUDB] LOGSECOND: 156 != 30", + "[db cfg for BLUDB] MIRRORLOGPATH: /mnt/backup != ", + "[db cfg for BLUDB] NUM_DB_BACKUPS: 60 != 2", + "[db cfg for BLUDB] REC_HIS_RETENTN: 60 != 0", + "[db cfg for BLUDB] SHEAPTHRES_SHR: automatic != 1336548", + "[db cfg for BLUDB] SORTHEAP: automatic != 66827", + "[db cfg for BLUDB] STMTHEAP: 20000 != AUTOMATIC(16384)", + "[db cfg for BLUDB] STMT_CONC: LITERALS != OFF", + "[db cfg for BLUDB] WLM_ADMISSION_CTRL: NO != YES", + "[dbm cfg] AGENT_STACK_SZ: WRONG != 1024", + "[dbm cfg] FENCED_POOL: 50 != AUTOMATIC(MAX_COORDAGENTS)", + "[dbm cfg] KEEPFENCED: NO != YES", + "[registry cfg] DB2AUTH: WRONG != OSAUTHDB", + "[registry cfg] DB2_4K_DEVICE_SUPPORT not found in output of db2set command", + "[registry cfg] DB2_BCKP_PAGE_VERIFICATION not found in output of db2set command", + "[registry cfg] DB2_CDE_REDUCED_LOGGING not found in output of db2set command", + "[registry cfg] DB2_EVALUNCOMMITTED not found in output of db2set command", + "[registry cfg] DB2_FMP_COMM_HEAPSZ not found in output of db2set command", + "[registry cfg] DB2_FMP_RUN_AS_CONNECTED_USER: NO != YES", + "[registry cfg] DB2_INLIST_TO_NLJN not found in output of db2set command", + "[registry cfg] DB2_MINIMIZE_LISTPREFETCH not found in output of db2set command", + "[registry cfg] DB2_OBJECT_STORAGE_LOCAL_STAGING_PATH: /mnt/backup/staging != /mnt/bludata0/scratch/db2/RemoteStorage", + "[registry cfg] DB2_SKIPDELETED not found in output of db2set command", + "[registry cfg] DB2_SKIPINSERTED not found in output of db2set command", + "[registry cfg] DB2_USE_ALTERNATE_PAGE_CLEANING: ON != ON [DB2_WORKLOAD]", + "[registry cfg] DB2_WORKLOAD: MAXIMO != ANALYTICS", + ], + ), + ], +) def test_validate_db2_config(test_case_name, expected_failures, mocker): - ''' + """ Each test case corresponds to a folder under test/test_cases. Each folder must contain a file db2uinstance.yaml and optionally db2getdbcfg.txt, db2getdbmcfg.txt and db2set.txt. - ''' + """ current_dir = os.path.dirname(os.path.abspath(__file__)) mock_get_db2u_instance_cr = mocker.patch("mas.devops.db2.get_db2u_instance_cr") - with open(os.path.join(current_dir, "..", "test_cases", test_case_name, "db2uinstance.yaml"), "r") as f: + with open( + os.path.join(current_dir, "..", "test_cases", test_case_name, "db2uinstance.yaml"), + "r", + ) as f: mock_get_db2u_instance_cr.return_value = yaml.load(f, Loader=yaml.FullLoader) mock_db2_pod_exec_db2_get_db_cfg = mocker.patch("mas.devops.db2.db2_pod_exec_db2_get_db_cfg") try: - with open(os.path.join(current_dir, "..", "test_cases", test_case_name, "db2getdbcfg.txt"), "r") as f: + with open( + os.path.join(current_dir, "..", "test_cases", test_case_name, "db2getdbcfg.txt"), + "r", + ) as f: mock_db2_pod_exec_db2_get_db_cfg.return_value = f.read() except FileNotFoundError: mock_db2_pod_exec_db2_get_db_cfg.return_value = None mock_db2_pod_exec_db2_get_dbm_cfg = mocker.patch("mas.devops.db2.db2_pod_exec_db2_get_dbm_cfg") try: - with open(os.path.join(current_dir, "..", "test_cases", test_case_name, "db2getdbmcfg.txt"), "r") as f: + with open( + os.path.join(current_dir, "..", "test_cases", test_case_name, "db2getdbmcfg.txt"), + "r", + ) as f: mock_db2_pod_exec_db2_get_dbm_cfg.return_value = f.read() except FileNotFoundError: mock_db2_pod_exec_db2_get_dbm_cfg.return_value = None mock_db2_pod_exec_db2set = mocker.patch("mas.devops.db2.db2_pod_exec_db2set") try: - with open(os.path.join(current_dir, "..", "test_cases", test_case_name, "db2set.txt"), "r") as f: + with open( + os.path.join(current_dir, "..", "test_cases", test_case_name, "db2set.txt"), + "r", + ) as f: mock_db2_pod_exec_db2set.return_value = f.read() except FileNotFoundError: mock_db2_pod_exec_db2set.return_value = None diff --git a/test/src/test_mas.py b/test/src/test_mas.py index 5a775e30..bc7c8511 100644 --- a/test/src/test_mas.py +++ b/test/src/test_mas.py @@ -22,9 +22,7 @@ @pytest.fixture(scope="module") def dynClient(): """Create DynamicClient for OpenShift cluster access.""" - return dynamic.DynamicClient( - api_client.ApiClient(configuration=config.load_kube_config()) - ) + return dynamic.DynamicClient(api_client.ApiClient(configuration=config.load_kube_config())) def test_entitlement(dynClient): @@ -44,7 +42,14 @@ def test_entitlement_with_artifactory(dynClient): icrUsername = "testing-i" icrPassword = "not-a-real-password-i" - secret = mas.updateIBMEntitlementKey(dynClient, "default", icrUsername, icrPassword, artifactoryUsername, artifactoryPassword) + secret = mas.updateIBMEntitlementKey( + dynClient, + "default", + icrUsername, + icrPassword, + artifactoryUsername, + artifactoryPassword, + ) assert secret is not None assert isinstance(secret, ResourceInstance) assert secret.metadata.name == "ibm-entitlement" diff --git a/test/src/test_ocp.py b/test/src/test_ocp.py index 6ef47a69..1600d638 100644 --- a/test/src/test_ocp.py +++ b/test/src/test_ocp.py @@ -28,11 +28,11 @@ def test_is_cluster_in_range(): def test_execInPod_success(mocker): - mock_CoreV1Api = mocker.patch('kubernetes.client.CoreV1Api') + mock_CoreV1Api = mocker.patch("kubernetes.client.CoreV1Api") mock_core_v1_api = mock_CoreV1Api.return_value # Mock the `stream` function and the request object it returns - mock_stream = mocker.patch('mas.devops.ocp.stream') + mock_stream = mocker.patch("mas.devops.ocp.stream") # Mock the response of the `stream` function mock_req = mock_stream.return_value @@ -41,17 +41,17 @@ def test_execInPod_success(mocker): mock_req.read_stderr.return_value = "mock_stderr" mock_req.read_channel.return_value = yaml.dump({"status": "Success"}) - o = ocp.execInPod(mock_core_v1_api, 'pod_name', 'namespace', ['command']) + o = ocp.execInPod(mock_core_v1_api, "pod_name", "namespace", ["command"]) assert o == "mock_stdout" def test_execInPod_failure(mocker): - mock_CoreV1Api = mocker.patch('kubernetes.client.CoreV1Api') + mock_CoreV1Api = mocker.patch("kubernetes.client.CoreV1Api") mock_core_v1_api = mock_CoreV1Api.return_value # Mock the `stream` function and the request object it returns - mock_stream = mocker.patch('mas.devops.ocp.stream') + mock_stream = mocker.patch("mas.devops.ocp.stream") # Mock the response of the `stream` function mock_req = mock_stream.return_value @@ -60,5 +60,8 @@ def test_execInPod_failure(mocker): mock_req.read_stderr.return_value = "mock_stderr" mock_req.read_channel.return_value = yaml.dump({"status": "Failure"}) - with pytest.raises(Exception, match=r"Failed to execute \['command'\] on pod_name in namespace namespace: None. stdout: mock_stdout, stderr: mock_stderr"): - ocp.execInPod(mock_core_v1_api, 'pod_name', 'namespace', ['command']) + with pytest.raises( + Exception, + match=r"Failed to execute \['command'\] on pod_name in namespace namespace: None. stdout: mock_stdout, stderr: mock_stderr", + ): + ocp.execInPod(mock_core_v1_api, "pod_name", "namespace", ["command"]) diff --git a/test/src/test_olm.py b/test/src/test_olm.py index e0357356..e0839742 100644 --- a/test/src/test_olm.py +++ b/test/src/test_olm.py @@ -21,9 +21,7 @@ @pytest.fixture(scope="module") def dynClient(): """Create DynamicClient for OpenShift cluster access.""" - return dynamic.DynamicClient( - api_client.ApiClient(configuration=config.load_kube_config()) - ) + return dynamic.DynamicClient(api_client.ApiClient(configuration=config.load_kube_config())) def test_get_manifest(dynClient): @@ -72,11 +70,7 @@ def test_crud(dynClient): def test_crud_with_config(dynClient): namespace = "cli-fvt-2" # We don't need this, just want to test that it works - testConfig = { - "env": [ - {"name": "DUMMY_ENV_VAR", "value": "testing"} - ] - } + testConfig = {"env": [{"name": "DUMMY_ENV_VAR", "value": "testing"}]} subscription = olm.applySubscription(dynClient, namespace, "ibm-sls", packageChannel="3.x", config=testConfig) assert subscription.metadata.name == "ibm-sls" assert subscription.metadata.namespace == namespace @@ -102,7 +96,7 @@ def test_crud_with_manual_approval(dynClient): namespace, "ibm-sls", packageChannel="3.x", - installPlanApproval="Manual" + installPlanApproval="Manual", ) # If we get here, the test should fail assert False, "Expected OLMException to be raised when installPlanApproval is Manual without startingCSV" @@ -121,7 +115,7 @@ def test_crud_with_starting_csv(dynClient): namespace, "ibm-sls", packageChannel="3.x", - startingCSV="ibm-sls.v3.8.0" + startingCSV="ibm-sls.v3.8.0", ) assert subscription.metadata.name == "ibm-sls" assert subscription.metadata.namespace == namespace @@ -151,7 +145,7 @@ def test_crud_with_manual_approval_and_starting_csv(dynClient): "ibm-sls", packageChannel="3.x", installPlanApproval="Manual", - startingCSV="ibm-sls.v3.8.0" + startingCSV="ibm-sls.v3.8.0", ) assert subscription.metadata.name == "ibm-sls" assert subscription.metadata.namespace == namespace diff --git a/test/src/test_olm_installplan_selection.py b/test/src/test_olm_installplan_selection.py index 1101742c..2097364c 100644 --- a/test/src/test_olm_installplan_selection.py +++ b/test/src/test_olm_installplan_selection.py @@ -82,21 +82,24 @@ def create_owner_ref(kind, name): return ref -@patch('mas.devops.olm.createNamespace') -@patch('mas.devops.olm.ensureOperatorGroupExists') -@patch('mas.devops.olm.getPackageManifest') -@patch('mas.devops.olm.sleep') +@patch("mas.devops.olm.createNamespace") +@patch("mas.devops.olm.ensureOperatorGroupExists") +@patch("mas.devops.olm.getPackageManifest") +@patch("mas.devops.olm.sleep") def test_automatic_approval_uses_label_selector_only( - mock_sleep, mock_get_manifest, mock_ensure_og, mock_create_ns, mock_dyn_client, mock_env + mock_sleep, + mock_get_manifest, + mock_ensure_og, + mock_create_ns, + mock_dyn_client, + mock_env, ): """ Test that automatic approval uses only the label selector (standard behavior). Should NOT query all InstallPlans. """ # Setup mocks - mock_get_manifest.return_value = Mock( - status=Mock(defaultChannel="stable", catalogSource="test-catalog") - ) + mock_get_manifest.return_value = Mock(status=Mock(defaultChannel="stable", catalogSource="test-catalog")) # Mock subscription API sub_api = Mock() @@ -111,7 +114,7 @@ def test_automatic_approval_uses_label_selector_only( # First call returns empty list (no existing subscription), subsequent calls return the subscription sub_api.get.side_effect = [ MockResourceList([]), # Initial check for existing subscription - mock_subscription # Subsequent calls when waiting for subscription to complete + mock_subscription, # Subsequent calls when waiting for subscription to complete ] sub_api.apply.return_value = Mock() @@ -121,7 +124,7 @@ def test_automatic_approval_uses_label_selector_only( name="install-plan-1", labels={"operators.coreos.com/test-operator.test-namespace": ""}, csv_names=["test-operator.v1.0.0"], - phase="Complete" + phase="Complete", ) install_plan_api.get.return_value = MockResourceList([install_plan]) @@ -131,14 +134,14 @@ def test_automatic_approval_uses_label_selector_only( ("operators.coreos.com/v1alpha1", "InstallPlan"): install_plan_api, }.get((kwargs.get("api_version"), kwargs.get("kind"))) - with patch('mas.devops.olm.Environment', return_value=mock_env): + with patch("mas.devops.olm.Environment", return_value=mock_env): # Call applySubscription with Automatic approval (default) olm.applySubscription( mock_dyn_client, "test-namespace", "test-operator", packageChannel="stable", - installPlanApproval="Automatic" + installPlanApproval="Automatic", ) # Verify InstallPlan API was called with label selector only @@ -147,25 +150,28 @@ def test_automatic_approval_uses_label_selector_only( # Should only use label selector, never query all InstallPlans for call_args in install_plan_calls: args, kwargs = call_args - assert 'label_selector' in kwargs, "Should use label selector" - assert kwargs.get('namespace') == "test-namespace" + assert "label_selector" in kwargs, "Should use label selector" + assert kwargs.get("namespace") == "test-namespace" -@patch('mas.devops.olm.createNamespace') -@patch('mas.devops.olm.ensureOperatorGroupExists') -@patch('mas.devops.olm.getPackageManifest') -@patch('mas.devops.olm.sleep') +@patch("mas.devops.olm.createNamespace") +@patch("mas.devops.olm.ensureOperatorGroupExists") +@patch("mas.devops.olm.getPackageManifest") +@patch("mas.devops.olm.sleep") def test_manual_approval_without_starting_csv_uses_label_selector_only( - mock_sleep, mock_get_manifest, mock_ensure_og, mock_create_ns, mock_dyn_client, mock_env + mock_sleep, + mock_get_manifest, + mock_ensure_og, + mock_create_ns, + mock_dyn_client, + mock_env, ): """ Test that Manual approval with startingCSV uses only label selector when it finds a match. Should NOT query all InstallPlans. """ # Setup mocks - mock_get_manifest.return_value = Mock( - status=Mock(defaultChannel="stable", catalogSource="test-catalog") - ) + mock_get_manifest.return_value = Mock(status=Mock(defaultChannel="stable", catalogSource="test-catalog")) # Mock subscription API sub_api = Mock() @@ -180,7 +186,7 @@ def test_manual_approval_without_starting_csv_uses_label_selector_only( # First call returns empty list (no existing subscription), subsequent calls return the subscription sub_api.get.side_effect = [ MockResourceList([]), # Initial check for existing subscription - mock_subscription # Subsequent calls when waiting for subscription to complete + mock_subscription, # Subsequent calls when waiting for subscription to complete ] sub_api.apply.return_value = Mock() @@ -190,18 +196,18 @@ def test_manual_approval_without_starting_csv_uses_label_selector_only( name="install-plan-1", labels={"operators.coreos.com/test-operator.test-namespace": ""}, csv_names=["test-operator.v1.0.0"], - phase="RequiresApproval" + phase="RequiresApproval", ) install_plan_complete = MockResource( name="install-plan-1", labels={"operators.coreos.com/test-operator.test-namespace": ""}, csv_names=["test-operator.v1.0.0"], - phase="Complete" + phase="Complete", ) # Simulate phase transition: first returns RequiresApproval, then Complete after patch def get_install_plan_side_effect(*args, **kwargs): - if 'name' in kwargs: + if "name" in kwargs: # After patch is called, return Complete phase return install_plan_complete else: @@ -216,7 +222,7 @@ def get_install_plan_side_effect(*args, **kwargs): ("operators.coreos.com/v1alpha1", "InstallPlan"): install_plan_api, }.get((kwargs.get("api_version"), kwargs.get("kind"))) - with patch('mas.devops.olm.Environment', return_value=mock_env): + with patch("mas.devops.olm.Environment", return_value=mock_env): # Call with Manual approval with startingCSV olm.applySubscription( mock_dyn_client, @@ -224,7 +230,7 @@ def get_install_plan_side_effect(*args, **kwargs): "test-operator", packageChannel="stable", installPlanApproval="Manual", - startingCSV="test-operator.v1.0.0" + startingCSV="test-operator.v1.0.0", ) # Verify only label selector was used @@ -232,24 +238,27 @@ def get_install_plan_side_effect(*args, **kwargs): for call_args in install_plan_calls: args, kwargs = call_args # Should only use label selector or get by name, never query all - assert 'label_selector' in kwargs or 'name' in kwargs + assert "label_selector" in kwargs or "name" in kwargs -@patch('mas.devops.olm.createNamespace') -@patch('mas.devops.olm.ensureOperatorGroupExists') -@patch('mas.devops.olm.getPackageManifest') -@patch('mas.devops.olm.sleep') +@patch("mas.devops.olm.createNamespace") +@patch("mas.devops.olm.ensureOperatorGroupExists") +@patch("mas.devops.olm.getPackageManifest") +@patch("mas.devops.olm.sleep") def test_manual_approval_with_starting_csv_label_selector_finds_match( - mock_sleep, mock_get_manifest, mock_ensure_og, mock_create_ns, mock_dyn_client, mock_env + mock_sleep, + mock_get_manifest, + mock_ensure_og, + mock_create_ns, + mock_dyn_client, + mock_env, ): """ Test Manual approval with startingCSV when label selector returns the correct InstallPlan. Should use label selector result, NOT query all InstallPlans. """ # Setup mocks - mock_get_manifest.return_value = Mock( - status=Mock(defaultChannel="stable", catalogSource="test-catalog") - ) + mock_get_manifest.return_value = Mock(status=Mock(defaultChannel="stable", catalogSource="test-catalog")) # Mock subscription API sub_api = Mock() @@ -264,7 +273,7 @@ def test_manual_approval_with_starting_csv_label_selector_finds_match( # First call returns empty list (no existing subscription), subsequent calls return the subscription sub_api.get.side_effect = [ MockResourceList([]), # Initial check for existing subscription - mock_subscription # Subsequent calls when waiting for subscription to complete + mock_subscription, # Subsequent calls when waiting for subscription to complete ] sub_api.apply.return_value = Mock() @@ -274,18 +283,18 @@ def test_manual_approval_with_starting_csv_label_selector_finds_match( name="install-plan-1", labels={"operators.coreos.com/test-operator.test-namespace": ""}, csv_names=["test-operator.v1.0.0"], # Matches startingCSV - phase="RequiresApproval" + phase="RequiresApproval", ) install_plan_complete = MockResource( name="install-plan-1", labels={"operators.coreos.com/test-operator.test-namespace": ""}, csv_names=["test-operator.v1.0.0"], - phase="Complete" + phase="Complete", ) # Simulate phase transition def get_install_plan_side_effect(*args, **kwargs): - if 'name' in kwargs: + if "name" in kwargs: return install_plan_complete else: return MockResourceList([install_plan_requires_approval]) @@ -298,14 +307,14 @@ def get_install_plan_side_effect(*args, **kwargs): ("operators.coreos.com/v1alpha1", "InstallPlan"): install_plan_api, }.get((kwargs.get("api_version"), kwargs.get("kind"))) - with patch('mas.devops.olm.Environment', return_value=mock_env): + with patch("mas.devops.olm.Environment", return_value=mock_env): olm.applySubscription( mock_dyn_client, "test-namespace", "test-operator", packageChannel="stable", installPlanApproval="Manual", - startingCSV="test-operator.v1.0.0" + startingCSV="test-operator.v1.0.0", ) # Verify we found the InstallPlan via label selector @@ -315,16 +324,20 @@ def get_install_plan_side_effect(*args, **kwargs): # Check that we never queried without a label_selector or name for call_args in install_plan_calls: args, kwargs = call_args - assert 'label_selector' in kwargs or 'name' in kwargs, \ - "Should only use label selector or get by name, not query all" + assert "label_selector" in kwargs or "name" in kwargs, "Should only use label selector or get by name, not query all" -@patch('mas.devops.olm.createNamespace') -@patch('mas.devops.olm.ensureOperatorGroupExists') -@patch('mas.devops.olm.getPackageManifest') -@patch('mas.devops.olm.sleep') +@patch("mas.devops.olm.createNamespace") +@patch("mas.devops.olm.ensureOperatorGroupExists") +@patch("mas.devops.olm.getPackageManifest") +@patch("mas.devops.olm.sleep") def test_manual_approval_with_starting_csv_fallback_to_ownership_search( - mock_sleep, mock_get_manifest, mock_ensure_og, mock_create_ns, mock_dyn_client, mock_env + mock_sleep, + mock_get_manifest, + mock_ensure_og, + mock_create_ns, + mock_dyn_client, + mock_env, ): """ Test Manual approval with startingCSV when label selector misses the completed InstallPlan. @@ -332,9 +345,7 @@ def test_manual_approval_with_starting_csv_fallback_to_ownership_search( This is the key scenario the bug fix addresses. """ # Setup mocks - mock_get_manifest.return_value = Mock( - status=Mock(defaultChannel="stable", catalogSource="test-catalog") - ) + mock_get_manifest.return_value = Mock(status=Mock(defaultChannel="stable", catalogSource="test-catalog")) # Mock subscription API sub_api = Mock() @@ -349,7 +360,7 @@ def test_manual_approval_with_starting_csv_fallback_to_ownership_search( # First call returns empty list (no existing subscription), subsequent calls return the subscription sub_api.get.side_effect = [ MockResourceList([]), # Initial check for existing subscription - mock_subscription # Subsequent calls when waiting for subscription to complete + mock_subscription, # Subsequent calls when waiting for subscription to complete ] sub_api.apply.return_value = Mock() @@ -361,7 +372,7 @@ def test_manual_approval_with_starting_csv_fallback_to_ownership_search( name="install-plan-2", labels={"operators.coreos.com/test-operator.test-namespace": ""}, csv_names=["test-operator.v2.0.0"], # Does NOT match startingCSV - phase="Installing" + phase="Installing", ) # All InstallPlans query returns both (including the completed one) @@ -370,7 +381,7 @@ def test_manual_approval_with_starting_csv_fallback_to_ownership_search( labels={}, # Label might be removed from completed plan owner_refs=[create_owner_ref("Subscription", "test-operator")], csv_names=["test-operator.v1.0.0"], # Matches startingCSV - phase="RequiresApproval" + phase="RequiresApproval", ) correct_install_plan_complete = MockResource( @@ -378,15 +389,15 @@ def test_manual_approval_with_starting_csv_fallback_to_ownership_search( labels={}, owner_refs=[create_owner_ref("Subscription", "test-operator")], csv_names=["test-operator.v1.0.0"], - phase="Complete" + phase="Complete", ) # Setup the mock to return different results based on parameters def get_side_effect(*args, **kwargs): - if 'label_selector' in kwargs: + if "label_selector" in kwargs: # Label selector query - returns only wrong InstallPlan return MockResourceList([wrong_install_plan]) - elif 'name' in kwargs: + elif "name" in kwargs: # Get by name - return the correct one (Complete after patch) return correct_install_plan_complete else: @@ -401,14 +412,14 @@ def get_side_effect(*args, **kwargs): ("operators.coreos.com/v1alpha1", "InstallPlan"): install_plan_api, }.get((kwargs.get("api_version"), kwargs.get("kind"))) - with patch('mas.devops.olm.Environment', return_value=mock_env): + with patch("mas.devops.olm.Environment", return_value=mock_env): olm.applySubscription( mock_dyn_client, "test-namespace", "test-operator", packageChannel="stable", installPlanApproval="Manual", - startingCSV="test-operator.v1.0.0" + startingCSV="test-operator.v1.0.0", ) # Verify the fallback behavior occurred @@ -417,34 +428,31 @@ def get_side_effect(*args, **kwargs): # Should have: # 1. Called with label_selector (initial query) # 2. Called without label_selector (fallback to query all) - has_label_selector_call = any( - 'label_selector' in call_args[1] - for call_args in install_plan_calls - ) - has_all_query_call = any( - 'label_selector' not in call_args[1] and 'name' not in call_args[1] - for call_args in install_plan_calls - ) + has_label_selector_call = any("label_selector" in call_args[1] for call_args in install_plan_calls) + has_all_query_call = any("label_selector" not in call_args[1] and "name" not in call_args[1] for call_args in install_plan_calls) assert has_label_selector_call, "Should have tried label selector first" assert has_all_query_call, "Should have fallen back to querying all InstallPlans" -@patch('mas.devops.olm.createNamespace') -@patch('mas.devops.olm.ensureOperatorGroupExists') -@patch('mas.devops.olm.getPackageManifest') -@patch('mas.devops.olm.sleep') +@patch("mas.devops.olm.createNamespace") +@patch("mas.devops.olm.ensureOperatorGroupExists") +@patch("mas.devops.olm.getPackageManifest") +@patch("mas.devops.olm.sleep") def test_manual_approval_filters_by_subscription_ownership( - mock_sleep, mock_get_manifest, mock_ensure_og, mock_create_ns, mock_dyn_client, mock_env + mock_sleep, + mock_get_manifest, + mock_ensure_og, + mock_create_ns, + mock_dyn_client, + mock_env, ): """ Test that when querying all InstallPlans, we correctly filter by subscription ownership. This ensures we don't accidentally use InstallPlans from other subscriptions. """ # Setup mocks - mock_get_manifest.return_value = Mock( - status=Mock(defaultChannel="stable", catalogSource="test-catalog") - ) + mock_get_manifest.return_value = Mock(status=Mock(defaultChannel="stable", catalogSource="test-catalog")) # Mock subscription API sub_api = Mock() @@ -459,7 +467,7 @@ def test_manual_approval_filters_by_subscription_ownership( # First call returns empty list (no existing subscription), subsequent calls return the subscription sub_api.get.side_effect = [ MockResourceList([]), # Initial check for existing subscription - mock_subscription # Subsequent calls when waiting for subscription to complete + mock_subscription, # Subsequent calls when waiting for subscription to complete ] sub_api.apply.return_value = Mock() @@ -471,7 +479,7 @@ def test_manual_approval_filters_by_subscription_ownership( name="install-plan-wrong", labels={"operators.coreos.com/test-operator.test-namespace": ""}, csv_names=["test-operator.v2.0.0"], - phase="Installing" + phase="Installing", ) # All InstallPlans includes: @@ -481,7 +489,7 @@ def test_manual_approval_filters_by_subscription_ownership( labels={}, owner_refs=[create_owner_ref("Subscription", "test-operator")], csv_names=["test-operator.v1.0.0"], - phase="RequiresApproval" + phase="RequiresApproval", ) correct_install_plan_complete = MockResource( @@ -489,7 +497,7 @@ def test_manual_approval_filters_by_subscription_ownership( labels={}, owner_refs=[create_owner_ref("Subscription", "test-operator")], csv_names=["test-operator.v1.0.0"], - phase="Complete" + phase="Complete", ) # 2. One owned by a different subscription (should be ignored) @@ -498,17 +506,23 @@ def test_manual_approval_filters_by_subscription_ownership( labels={}, owner_refs=[create_owner_ref("Subscription", "other-operator")], csv_names=["test-operator.v1.0.0"], # Same CSV but wrong subscription - phase="Complete" + phase="Complete", ) def get_side_effect(*args, **kwargs): - if 'label_selector' in kwargs: + if "label_selector" in kwargs: return MockResourceList([wrong_install_plan]) - elif 'name' in kwargs: + elif "name" in kwargs: return correct_install_plan_complete else: # Return all three InstallPlans - return MockResourceList([correct_install_plan_requires_approval, other_subscription_plan, wrong_install_plan]) + return MockResourceList( + [ + correct_install_plan_requires_approval, + other_subscription_plan, + wrong_install_plan, + ] + ) install_plan_api.get.side_effect = get_side_effect install_plan_api.patch.return_value = Mock() @@ -518,18 +532,19 @@ def get_side_effect(*args, **kwargs): ("operators.coreos.com/v1alpha1", "InstallPlan"): install_plan_api, }.get((kwargs.get("api_version"), kwargs.get("kind"))) - with patch('mas.devops.olm.Environment', return_value=mock_env): + with patch("mas.devops.olm.Environment", return_value=mock_env): olm.applySubscription( mock_dyn_client, "test-namespace", "test-operator", packageChannel="stable", installPlanApproval="Manual", - startingCSV="test-operator.v1.0.0" + startingCSV="test-operator.v1.0.0", ) # The test passes if it completes without error # The code should have found the correct InstallPlan by checking ownership # and ignored the one from the other subscription + # Made with Bob diff --git a/test/src/test_restore.py b/test/src/test_restore.py index 3666b6b4..26e60001 100644 --- a/test/src/test_restore.py +++ b/test/src/test_restore.py @@ -21,23 +21,20 @@ class TestLoadYamlFile: def test_load_valid_yaml_file(self, tmp_path): """Test loading a valid YAML file""" yaml_content = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config', - 'namespace': 'test-ns' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config", "namespace": "test-ns"}, } yaml_file = tmp_path / "test.yaml" - with open(yaml_file, 'w') as f: + with open(yaml_file, "w") as f: yaml.dump(yaml_content, f) result = loadYamlFile(str(yaml_file)) assert result is not None - assert result['kind'] == 'ConfigMap' - assert result['metadata']['name'] == 'test-config' + assert result["kind"] == "ConfigMap" + assert result["metadata"]["name"] == "test-config" def test_load_empty_yaml_file(self, tmp_path): """Test loading an empty YAML file""" @@ -86,15 +83,10 @@ def setup_method(self): def test_create_new_namespaced_resource(self): """Test creating a new namespaced resource""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config', - 'namespace': 'test-ns' - }, - 'data': { - 'key': 'value' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config", "namespace": "test-ns"}, + "data": {"key": "value"}, } # Resource doesn't exist @@ -103,21 +95,16 @@ def test_create_new_namespaced_resource(self): success, name, status = restoreResource(self.mock_client, resource_data) assert success is True - assert name == 'test-config' + assert name == "test-config" assert status is None - self.mock_resource_api.create.assert_called_once_with( - body=resource_data, - namespace='test-ns' - ) + self.mock_resource_api.create.assert_called_once_with(body=resource_data, namespace="test-ns") def test_create_new_cluster_resource(self): """Test creating a new cluster-scoped resource""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'Namespace', - 'metadata': { - 'name': 'test-namespace' - } + "apiVersion": "v1", + "kind": "Namespace", + "metadata": {"name": "test-namespace"}, } # Resource doesn't exist @@ -126,149 +113,107 @@ def test_create_new_cluster_resource(self): success, name, status = restoreResource(self.mock_client, resource_data) assert success is True - assert name == 'test-namespace' + assert name == "test-namespace" assert status is None - self.mock_resource_api.create.assert_called_once_with( - body=resource_data - ) + self.mock_resource_api.create.assert_called_once_with(body=resource_data) def test_update_existing_resource_with_replace_true(self): """Test updating an existing resource when replace_resource is True""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config', - 'namespace': 'test-ns' - }, - 'data': { - 'key': 'new-value' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config", "namespace": "test-ns"}, + "data": {"key": "new-value"}, } # Resource exists - existing_resource = { - 'metadata': { - 'name': 'test-config', - 'resourceVersion': '12345' - } - } + existing_resource = {"metadata": {"name": "test-config", "resourceVersion": "12345"}} self.mock_resource_api.get.return_value = existing_resource success, name, status = restoreResource(self.mock_client, resource_data, replace_resource=True) assert success is True - assert name == 'test-config' - assert status == 'updated' + assert name == "test-config" + assert status == "updated" self.mock_resource_api.patch.assert_called_once_with( body=resource_data, - name='test-config', - namespace='test-ns', - content_type='application/merge-patch+json' + name="test-config", + namespace="test-ns", + content_type="application/merge-patch+json", ) def test_skip_existing_resource_with_replace_false(self): """Test skipping an existing resource when replace_resource is False""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config', - 'namespace': 'test-ns' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config", "namespace": "test-ns"}, } # Resource exists - existing_resource = {'metadata': {'name': 'test-config'}} + existing_resource = {"metadata": {"name": "test-config"}} self.mock_resource_api.get.return_value = existing_resource success, name, status = restoreResource(self.mock_client, resource_data, replace_resource=False) assert success is True - assert name == 'test-config' - assert status == 'skipped' + assert name == "test-config" + assert status == "skipped" self.mock_resource_api.patch.assert_not_called() self.mock_resource_api.create.assert_not_called() def test_namespace_override(self): """Test that namespace parameter overrides resource namespace""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config', - 'namespace': 'original-ns' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config", "namespace": "original-ns"}, } # Resource doesn't exist self.mock_resource_api.get.side_effect = NotFoundError(Mock()) - success, name, status = restoreResource( - self.mock_client, - resource_data, - namespace='override-ns' - ) + success, name, status = restoreResource(self.mock_client, resource_data, namespace="override-ns") assert success is True - self.mock_resource_api.create.assert_called_once_with( - body=resource_data, - namespace='override-ns' - ) + self.mock_resource_api.create.assert_called_once_with(body=resource_data, namespace="override-ns") def test_missing_kind_field(self): """Test handling resource missing kind field""" - resource_data = { - 'apiVersion': 'v1', - 'metadata': { - 'name': 'test-resource' - } - } + resource_data = {"apiVersion": "v1", "metadata": {"name": "test-resource"}} success, name, status = restoreResource(self.mock_client, resource_data) assert success is False - assert name == 'test-resource' - assert 'missing required fields' in status.lower() + assert name == "test-resource" + assert "missing required fields" in status.lower() def test_missing_api_version_field(self): """Test handling resource missing apiVersion field""" - resource_data = { - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-resource' - } - } + resource_data = {"kind": "ConfigMap", "metadata": {"name": "test-resource"}} success, name, status = restoreResource(self.mock_client, resource_data) assert success is False - assert name == 'test-resource' - assert 'missing required fields' in status.lower() + assert name == "test-resource" + assert "missing required fields" in status.lower() def test_missing_name_field(self): """Test handling resource missing name field""" - resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': {} - } + resource_data = {"apiVersion": "v1", "kind": "ConfigMap", "metadata": {}} success, name, status = restoreResource(self.mock_client, resource_data) assert success is False - assert name == 'unknown' - assert 'missing required fields' in status.lower() + assert name == "unknown" + assert "missing required fields" in status.lower() def test_create_failure(self): """Test handling create operation failure""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config', - 'namespace': 'test-ns' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config", "namespace": "test-ns"}, } # Resource doesn't exist @@ -279,23 +224,20 @@ def test_create_failure(self): success, name, status = restoreResource(self.mock_client, resource_data) assert success is False - assert name == 'test-config' - assert 'Failed to create' in status - assert 'Create failed' in status + assert name == "test-config" + assert "Failed to create" in status + assert "Create failed" in status def test_patch_failure(self): """Test handling patch operation failure""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config', - 'namespace': 'test-ns' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config", "namespace": "test-ns"}, } # Resource exists - existing_resource = {'metadata': {'name': 'test-config'}} + existing_resource = {"metadata": {"name": "test-config"}} self.mock_resource_api.get.return_value = existing_resource # Patch fails self.mock_resource_api.patch.side_effect = Exception("Patch failed") @@ -303,18 +245,16 @@ def test_patch_failure(self): success, name, status = restoreResource(self.mock_client, resource_data, replace_resource=True) assert success is False - assert name == 'test-config' - assert 'Failed to update' in status - assert 'Patch failed' in status + assert name == "test-config" + assert "Failed to update" in status + assert "Patch failed" in status def test_resource_api_get_failure(self): """Test handling failure to get resource API""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': { - 'name': 'test-config' - } + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": "test-config"}, } # Getting resource API fails @@ -323,67 +263,58 @@ def test_resource_api_get_failure(self): success, name, status = restoreResource(self.mock_client, resource_data) assert success is False - assert name == 'test-config' - assert 'Error restoring resource' in status + assert name == "test-config" + assert "Error restoring resource" in status def test_update_cluster_scoped_resource(self): """Test updating a cluster-scoped resource""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'Namespace', - 'metadata': { - 'name': 'test-namespace' - } + "apiVersion": "v1", + "kind": "Namespace", + "metadata": {"name": "test-namespace"}, } # Resource exists - existing_resource = {'metadata': {'name': 'test-namespace'}} + existing_resource = {"metadata": {"name": "test-namespace"}} self.mock_resource_api.get.return_value = existing_resource success, name, status = restoreResource(self.mock_client, resource_data, replace_resource=True) assert success is True - assert name == 'test-namespace' - assert status == 'updated' + assert name == "test-namespace" + assert status == "updated" self.mock_resource_api.patch.assert_called_once_with( body=resource_data, - name='test-namespace', - content_type='application/merge-patch+json' + name="test-namespace", + content_type="application/merge-patch+json", ) def test_malformed_resource_data(self): """Test handling malformed resource data""" resource_data = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap' + "apiVersion": "v1", + "kind": "ConfigMap", # Missing metadata entirely } success, name, status = restoreResource(self.mock_client, resource_data) assert success is False - assert name == 'unknown' - assert 'missing required fields' in status.lower() + assert name == "unknown" + assert "missing required fields" in status.lower() def test_resource_with_complex_metadata(self): """Test resource with complex metadata structure""" resource_data = { - 'apiVersion': 'apps/v1', - 'kind': 'Deployment', - 'metadata': { - 'name': 'test-deployment', - 'namespace': 'test-ns', - 'labels': { - 'app': 'test', - 'version': 'v1' - }, - 'annotations': { - 'description': 'Test deployment' - } + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "test-deployment", + "namespace": "test-ns", + "labels": {"app": "test", "version": "v1"}, + "annotations": {"description": "Test deployment"}, }, - 'spec': { - 'replicas': 3 - } + "spec": {"replicas": 3}, } # Resource doesn't exist @@ -392,6 +323,6 @@ def test_resource_with_complex_metadata(self): success, name, status = restoreResource(self.mock_client, resource_data) assert success is True - assert name == 'test-deployment' + assert name == "test-deployment" assert status is None self.mock_resource_api.create.assert_called_once() diff --git a/test/src/test_slack.py b/test/src/test_slack.py index a3dfab83..d6d530e3 100644 --- a/test/src/test_slack.py +++ b/test/src/test_slack.py @@ -16,9 +16,9 @@ from mas.devops.slack import SlackUtil # Import functions from the notify-slack script -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../bin')) -script_path = os.path.join(os.path.dirname(__file__), '../../bin/mas-devops-notify-slack') -notify_slack = SourceFileLoader('notify_slack', script_path).load_module() +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../bin")) +script_path = os.path.join(os.path.dirname(__file__), "../../bin/mas-devops-notify-slack") +notify_slack = SourceFileLoader("notify_slack", script_path).load_module() def testSendMessage(): @@ -49,9 +49,9 @@ def testBroadcast(): # Tests for _getClusterName function def test_getClusterName_success(): """Test _getClusterName returns cluster name when env var is set""" - with patch.dict(os.environ, {'CLUSTER_NAME': 'test-cluster'}): + with patch.dict(os.environ, {"CLUSTER_NAME": "test-cluster"}): result = notify_slack._getClusterName() - assert result == 'test-cluster' + assert result == "test-cluster" def test_getClusterName_missing(): @@ -64,7 +64,7 @@ def test_getClusterName_missing(): def test_getClusterName_empty(): """Test _getClusterName exits when CLUSTER_NAME is empty""" - with patch.dict(os.environ, {'CLUSTER_NAME': ''}): + with patch.dict(os.environ, {"CLUSTER_NAME": ""}): with pytest.raises(SystemExit) as exc_info: notify_slack._getClusterName() assert exc_info.value.code == 1 @@ -73,156 +73,175 @@ def test_getClusterName_empty(): # Tests for _getToolchainLink function def test_getToolchainLink_both_set(): """Test _getToolchainLink returns formatted link when both env vars are set""" - with patch.dict(os.environ, { - 'TOOLCHAIN_PIPELINERUN_URL': 'https://example.com/pipeline', - 'TOOLCHAIN_TRIGGER_NAME': 'test-trigger' - }): + with patch.dict( + os.environ, + { + "TOOLCHAIN_PIPELINERUN_URL": "https://example.com/pipeline", + "TOOLCHAIN_TRIGGER_NAME": "test-trigger", + }, + ): result = notify_slack._getToolchainLink() - assert result == '' + assert result == "" def test_getToolchainLink_url_only(): """Test _getToolchainLink returns empty string when only URL is set""" - with patch.dict(os.environ, {'TOOLCHAIN_PIPELINERUN_URL': 'https://example.com/pipeline'}, clear=True): + with patch.dict( + os.environ, + {"TOOLCHAIN_PIPELINERUN_URL": "https://example.com/pipeline"}, + clear=True, + ): result = notify_slack._getToolchainLink() - assert result == '' + assert result == "" def test_getToolchainLink_trigger_only(): """Test _getToolchainLink returns empty string when only trigger name is set""" - with patch.dict(os.environ, {'TOOLCHAIN_TRIGGER_NAME': 'test-trigger'}, clear=True): + with patch.dict(os.environ, {"TOOLCHAIN_TRIGGER_NAME": "test-trigger"}, clear=True): result = notify_slack._getToolchainLink() - assert result == '' + assert result == "" def test_getToolchainLink_none_set(): """Test _getToolchainLink returns empty string when neither env var is set""" with patch.dict(os.environ, {}, clear=True): result = notify_slack._getToolchainLink() - assert result == '' + assert result == "" # Tests for notifyProvisionFyre function -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyProvisionFyre_success(mock_post): """Test notifyProvisionFyre with successful provisioning (rc=0)""" mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} mock_post.return_value = mock_response - with patch.dict(os.environ, { - 'CLUSTER_NAME': 'test-cluster', - 'OCP_CONSOLE_URL': 'https://console.example.com', - 'OCP_USERNAME': 'admin', - 'OCP_PASSWORD': 'password123' # pragma: allowlist secret - }): - result = notify_slack.notifyProvisionFyre(['#test-channel'], 0) + with patch.dict( + os.environ, + { + "CLUSTER_NAME": "test-cluster", + "OCP_CONSOLE_URL": "https://console.example.com", + "OCP_USERNAME": "admin", + "OCP_PASSWORD": "password123", # pragma: allowlist secret + }, + ): + result = notify_slack.notifyProvisionFyre(["#test-channel"], 0) assert result is True mock_post.assert_called_once() call_args = mock_post.call_args assert len(call_args[0][1]) == 4 # 4 message blocks -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyProvisionFyre_success_with_additional_msg(mock_post): """Test notifyProvisionFyre with successful provisioning and additional message""" mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} mock_post.return_value = mock_response - with patch.dict(os.environ, { - 'CLUSTER_NAME': 'test-cluster', - 'OCP_CONSOLE_URL': 'https://console.example.com', - 'OCP_USERNAME': 'admin', - 'OCP_PASSWORD': 'password123' # pragma: allowlist secret - }): - result = notify_slack.notifyProvisionFyre(['#test-channel'], 0, 'Additional info') + with patch.dict( + os.environ, + { + "CLUSTER_NAME": "test-cluster", + "OCP_CONSOLE_URL": "https://console.example.com", + "OCP_USERNAME": "admin", + "OCP_PASSWORD": "password123", # pragma: allowlist secret + }, + ): + result = notify_slack.notifyProvisionFyre(["#test-channel"], 0, "Additional info") assert result is True call_args = mock_post.call_args assert len(call_args[0][1]) == 5 # 5 message blocks with additional message -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyProvisionFyre_failure(mock_post): """Test notifyProvisionFyre with failed provisioning (rc!=0)""" mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} mock_post.return_value = mock_response - with patch.dict(os.environ, {'CLUSTER_NAME': 'test-cluster'}): - result = notify_slack.notifyProvisionFyre(['#test-channel'], 1) + with patch.dict(os.environ, {"CLUSTER_NAME": "test-cluster"}): + result = notify_slack.notifyProvisionFyre(["#test-channel"], 1) assert result is True call_args = mock_post.call_args assert len(call_args[0][1]) == 2 # 2 message blocks for failure -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyProvisionFyre_multiple_channels(mock_post): """Test notifyProvisionFyre with multiple channels""" mock_response1 = Mock() - mock_response1.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} + mock_response1.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} mock_response2 = Mock() - mock_response2.data = {'ok': True, 'channel': 'C456', 'ts': '1234567890.123457'} + mock_response2.data = {"ok": True, "channel": "C456", "ts": "1234567890.123457"} mock_post.return_value = [mock_response1, mock_response2] - with patch.dict(os.environ, {'CLUSTER_NAME': 'test-cluster'}): - result = notify_slack.notifyProvisionFyre(['#channel1', '#channel2'], 1) + with patch.dict(os.environ, {"CLUSTER_NAME": "test-cluster"}): + result = notify_slack.notifyProvisionFyre(["#channel1", "#channel2"], 1) assert result is True def test_notifyProvisionFyre_missing_env_vars(): """Test notifyProvisionFyre exits when required env vars are missing for success case""" - with patch.dict(os.environ, {'CLUSTER_NAME': 'test-cluster'}, clear=True): + with patch.dict(os.environ, {"CLUSTER_NAME": "test-cluster"}, clear=True): with pytest.raises(SystemExit) as exc_info: - notify_slack.notifyProvisionFyre(['#test-channel'], 0) + notify_slack.notifyProvisionFyre(["#test-channel"], 0) assert exc_info.value.code == 1 # Tests for notifyProvisionRoks function -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyProvisionRoks_success(mock_post): """Test notifyProvisionRoks with successful provisioning (rc=0)""" mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} mock_post.return_value = mock_response - with patch.dict(os.environ, { - 'CLUSTER_NAME': 'test-cluster', - 'OCP_CONSOLE_URL': 'https://console.example.com' - }): - result = notify_slack.notifyProvisionRoks(['#test-channel'], 0) + with patch.dict( + os.environ, + { + "CLUSTER_NAME": "test-cluster", + "OCP_CONSOLE_URL": "https://console.example.com", + }, + ): + result = notify_slack.notifyProvisionRoks(["#test-channel"], 0) assert result is True mock_post.assert_called_once() call_args = mock_post.call_args assert len(call_args[0][1]) == 3 # 3 message blocks -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyProvisionRoks_success_with_additional_msg(mock_post): """Test notifyProvisionRoks with successful provisioning and additional message""" mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} mock_post.return_value = mock_response - with patch.dict(os.environ, { - 'CLUSTER_NAME': 'test-cluster', - 'OCP_CONSOLE_URL': 'https://console.example.com' - }): - result = notify_slack.notifyProvisionRoks(['#test-channel'], 0, 'Extra details') + with patch.dict( + os.environ, + { + "CLUSTER_NAME": "test-cluster", + "OCP_CONSOLE_URL": "https://console.example.com", + }, + ): + result = notify_slack.notifyProvisionRoks(["#test-channel"], 0, "Extra details") assert result is True call_args = mock_post.call_args assert len(call_args[0][1]) == 4 # 4 message blocks with additional message -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyProvisionRoks_failure(mock_post): """Test notifyProvisionRoks with failed provisioning (rc!=0)""" mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} mock_post.return_value = mock_response - with patch.dict(os.environ, {'CLUSTER_NAME': 'test-cluster'}): - result = notify_slack.notifyProvisionRoks(['#test-channel'], 1) + with patch.dict(os.environ, {"CLUSTER_NAME": "test-cluster"}): + result = notify_slack.notifyProvisionRoks(["#test-channel"], 1) assert result is True call_args = mock_post.call_args assert len(call_args[0][1]) == 2 # 2 message blocks for failure @@ -230,33 +249,33 @@ def test_notifyProvisionRoks_failure(mock_post): def test_notifyProvisionRoks_missing_url(): """Test notifyProvisionRoks exits when OCP_CONSOLE_URL is missing for success case""" - with patch.dict(os.environ, {'CLUSTER_NAME': 'test-cluster'}, clear=True): + with patch.dict(os.environ, {"CLUSTER_NAME": "test-cluster"}, clear=True): with pytest.raises(SystemExit) as exc_info: - notify_slack.notifyProvisionRoks(['#test-channel'], 0) + notify_slack.notifyProvisionRoks(["#test-channel"], 0) assert exc_info.value.code == 1 # Tests for notifyPipelineStart function -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'createThreadConfigMap') -@patch.object(SlackUtil, 'updateThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "createThreadConfigMap") +@patch.object(SlackUtil, "updateThreadConfigMap") def test_notifyPipelineStart_new_thread(mock_update, mock_create, mock_post, mock_get): """Test notifyPipelineStart creates new thread when none exists""" # First call returns None, second call returns the created thread info thread_info = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_get.side_effect = [None, thread_info] mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} - mock_response.__getitem__ = lambda self, key: mock_response.data[key] if key in ['ts', 'channel'] else None + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} + mock_response.__getitem__ = lambda self, key: (mock_response.data[key] if key in ["ts", "channel"] else None) mock_post.return_value = mock_response - result = notify_slack.notifyPipelineStart(['#test-channel'], 'test-instance', 'Install') + result = notify_slack.notifyPipelineStart(["#test-channel"], "test-instance", "Install") assert result is not None assert result == thread_info @@ -265,18 +284,18 @@ def test_notifyPipelineStart_new_thread(mock_update, mock_create, mock_post, moc mock_update.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") def test_notifyPipelineStart_existing_thread(mock_get): """Test notifyPipelineStart returns existing thread info""" existing_thread = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_get.return_value = existing_thread - result = notify_slack.notifyPipelineStart(['#test-channel'], 'test-instance', 'Install') + result = notify_slack.notifyPipelineStart(["#test-channel"], "test-instance", "Install") assert result == existing_thread @@ -286,89 +305,93 @@ def test_notifyPipelineStart_existing_thread(mock_get): # test_notifyPipelineStart_update_pipeline_empty_instance_id -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'createThreadConfigMap') -@patch.object(SlackUtil, 'updateThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "createThreadConfigMap") +@patch.object(SlackUtil, "updateThreadConfigMap") def test_notifyPipelineStart_multiple_channels(mock_update, mock_create, mock_post, mock_get): """Test notifyPipelineStart with multiple channels""" # First call returns None, second call returns the created thread info thread_info = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_1': 'C456', - 'threadId_1': '1234567890.123457', - 'channel_count': '2' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_1": "C456", + "threadId_1": "1234567890.123457", + "channel_count": "2", } mock_get.side_effect = [None, thread_info] mock_response1 = Mock() - mock_response1.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} - mock_response1.__getitem__ = lambda self, key: mock_response1.data[key] if key in ['ts', 'channel'] else None + mock_response1.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} + mock_response1.__getitem__ = lambda self, key: (mock_response1.data[key] if key in ["ts", "channel"] else None) mock_response2 = Mock() - mock_response2.data = {'ok': True, 'channel': 'C456', 'ts': '1234567890.123457'} - mock_response2.__getitem__ = lambda self, key: mock_response2.data[key] if key in ['ts', 'channel'] else None + mock_response2.data = {"ok": True, "channel": "C456", "ts": "1234567890.123457"} + mock_response2.__getitem__ = lambda self, key: (mock_response2.data[key] if key in ["ts", "channel"] else None) mock_post.return_value = [mock_response1, mock_response2] - result = notify_slack.notifyPipelineStart(['#channel1', '#channel2'], 'test-instance', 'Install') + result = notify_slack.notifyPipelineStart(["#channel1", "#channel2"], "test-instance", "Install") assert result is not None # Verify that channel_count is set to 2 update_call_args = mock_update.call_args[0][2] - assert update_call_args['channel_count'] == '2' + assert update_call_args["channel_count"] == "2" # Tests for notifyAnsibleStart function -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'updateThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "updateThreadConfigMap") def test_notifyAnsibleStart_success(mock_update, mock_post, mock_get): """Test notifyAnsibleStart sends task start message""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True, 'ts': '1234567890.123457'} + mock_response.data = {"ok": True, "ts": "1234567890.123457"} mock_post.return_value = mock_response - result = notify_slack.notifyAnsibleStart(['#test-channel'], 'install-mas', 'test-instance', 'Install') + result = notify_slack.notifyAnsibleStart(["#test-channel"], "install-mas", "test-instance", "Install") assert result is True mock_post.assert_called_once() mock_update.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'updateThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "updateThreadConfigMap") def test_notifyAnsibleStart_creates_thread_if_missing(mock_update, mock_post, mock_get): """Test notifyAnsibleStart creates pipeline thread if it doesn't exist""" # First call returns None (no thread), second call returns None (checking again in notifyPipelineStart), # third call returns thread info (after creation), fourth call returns thread info (for ansible start) thread_info = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_get.side_effect = [None, None, thread_info, thread_info] # Mock for notifyPipelineStart's postMessageBlocks call mock_pipeline_response = Mock() - mock_pipeline_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} - mock_pipeline_response.__getitem__ = lambda self, key: mock_pipeline_response.data[key] if key in ['ts', 'channel'] else None + mock_pipeline_response.data = { + "ok": True, + "channel": "C123", + "ts": "1234567890.123456", + } + mock_pipeline_response.__getitem__ = lambda self, key: (mock_pipeline_response.data[key] if key in ["ts", "channel"] else None) # Mock for notifyAnsibleStart's postMessageBlocks call mock_task_response = Mock() - mock_task_response.data = {'ok': True, 'ts': '1234567890.123457'} + mock_task_response.data = {"ok": True, "ts": "1234567890.123457"} mock_post.side_effect = [mock_pipeline_response, mock_task_response] - with patch.object(SlackUtil, 'createThreadConfigMap'): - result = notify_slack.notifyAnsibleStart(['#test-channel'], 'install-mas', 'test-instance', 'Install') + with patch.object(SlackUtil, "createThreadConfigMap"): + result = notify_slack.notifyAnsibleStart(["#test-channel"], "install-mas", "test-instance", "Install") assert result is True assert mock_post.call_count == 2 # Once for pipeline start, once for task start @@ -378,57 +401,54 @@ def test_notifyAnsibleStart_creates_thread_if_missing(mock_update, mock_post, mo # See new test: test_notifyAnsibleStart_update_pipeline_no_instance_id -@patch.object(SlackUtil, 'getThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") def test_notifyAnsibleStart_no_channels(mock_get): """Test notifyAnsibleStart returns False when no channels found""" - mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_count': '0' - } + mock_get.return_value = {"instanceId": "test-instance", "channel_count": "0"} - result = notify_slack.notifyAnsibleStart(['#test-channel'], 'task-name', 'test-instance', 'Install') + result = notify_slack.notifyAnsibleStart(["#test-channel"], "task-name", "test-instance", "Install") assert result is False # Tests for notifyAnsibleComplete function -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'updateMessageBlocks') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "updateMessageBlocks") def test_notifyAnsibleComplete_success(mock_update, mock_get): """Test notifyAnsibleComplete with successful task (rc=0)""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'task_install-mas_0': '1234567890.123457', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "task_install-mas_0": "1234567890.123457", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_update.return_value = mock_response - result = notify_slack.notifyAnsibleComplete(['#test-channel'], 0, 'install-mas', 'test-instance', 'Install') + result = notify_slack.notifyAnsibleComplete(["#test-channel"], 0, "install-mas", "test-instance", "Install") assert result is True mock_update.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'updateMessageBlocks') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "updateMessageBlocks") def test_notifyAnsibleComplete_failure(mock_update, mock_get): """Test notifyAnsibleComplete with failed task (rc!=0)""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'task_install-mas_0': '1234567890.123457', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "task_install-mas_0": "1234567890.123457", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_update.return_value = mock_response - result = notify_slack.notifyAnsibleComplete(['#test-channel'], 1, 'install-mas', 'test-instance', 'Install') + result = notify_slack.notifyAnsibleComplete(["#test-channel"], 1, "install-mas", "test-instance", "Install") assert result is True # Verify failure message includes return code @@ -436,21 +456,21 @@ def test_notifyAnsibleComplete_failure(mock_update, mock_get): assert len(call_args) == 2 # Should have 2 blocks for failure (status + error details) -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyAnsibleComplete_no_start_message(mock_post, mock_get): """Test notifyAnsibleComplete posts new message when start message not found""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_post.return_value = mock_response - result = notify_slack.notifyAnsibleComplete(['#test-channel'], 0, 'install-mas', 'test-instance', 'Install') + result = notify_slack.notifyAnsibleComplete(["#test-channel"], 0, "install-mas", "test-instance", "Install") assert result is True mock_post.assert_called_once() @@ -460,88 +480,92 @@ def test_notifyAnsibleComplete_no_start_message(mock_post, mock_get): # See new test: test_notifyAnsibleComplete_update_pipeline_no_instance_id -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") def test_notifyAnsibleComplete_creates_thread_if_missing(mock_post, mock_get): """Test notifyAnsibleComplete creates pipeline thread if it doesn't exist""" # First call returns None (no thread), second call returns None (checking again in notifyPipelineStart), # third call returns thread info (after creation), fourth call returns thread info (for ansible complete) thread_info = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_get.side_effect = [None, None, thread_info, thread_info] # Mock for notifyPipelineStart's postMessageBlocks call mock_pipeline_response = Mock() - mock_pipeline_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} - mock_pipeline_response.__getitem__ = lambda self, key: mock_pipeline_response.data[key] if key in ['ts', 'channel'] else None + mock_pipeline_response.data = { + "ok": True, + "channel": "C123", + "ts": "1234567890.123456", + } + mock_pipeline_response.__getitem__ = lambda self, key: (mock_pipeline_response.data[key] if key in ["ts", "channel"] else None) # Mock for notifyAnsibleComplete's postMessageBlocks call mock_complete_response = Mock() - mock_complete_response.data = {'ok': True} + mock_complete_response.data = {"ok": True} mock_post.side_effect = [mock_pipeline_response, mock_complete_response] - with patch.object(SlackUtil, 'createThreadConfigMap'), patch.object(SlackUtil, 'updateThreadConfigMap'): - result = notify_slack.notifyAnsibleComplete(['#test-channel'], 0, 'install-mas', 'test-instance', 'Install') + with patch.object(SlackUtil, "createThreadConfigMap"), patch.object(SlackUtil, "updateThreadConfigMap"): + result = notify_slack.notifyAnsibleComplete(["#test-channel"], 0, "install-mas", "test-instance", "Install") assert result is True assert mock_post.call_count == 2 # Once for pipeline start, once for task complete # Tests for notifyPipelineComplete function -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'deleteThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "deleteThreadConfigMap") def test_notifyPipelineComplete_success(mock_delete, mock_post, mock_get): """Test notifyPipelineComplete with successful pipeline (rc=0)""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_post.return_value = mock_response - result = notify_slack.notifyPipelineComplete(['#test-channel'], 0, 'test-instance', 'Install') + result = notify_slack.notifyPipelineComplete(["#test-channel"], 0, "test-instance", "Install") assert result is True mock_post.assert_called_once() mock_delete.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'deleteThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "deleteThreadConfigMap") def test_notifyPipelineComplete_failure(mock_delete, mock_post, mock_get): """Test notifyPipelineComplete with failed pipeline (rc!=0)""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_post.return_value = mock_response - result = notify_slack.notifyPipelineComplete(['#test-channel'], 1, 'test-instance', 'Install') + result = notify_slack.notifyPipelineComplete(["#test-channel"], 1, "test-instance", "Install") assert result is True mock_delete.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") def test_notifyPipelineComplete_no_thread_info(mock_get): """Test notifyPipelineComplete returns False when no thread info found""" mock_get.return_value = None - result = notify_slack.notifyPipelineComplete(['#test-channel'], 0, 'test-instance', 'Install') + result = notify_slack.notifyPipelineComplete(["#test-channel"], 0, "test-instance", "Install") assert result is False @@ -550,60 +574,57 @@ def test_notifyPipelineComplete_no_thread_info(mock_get): # See new test: test_notifyPipelineComplete_update_pipeline_no_instance_id -@patch.object(SlackUtil, 'getThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") def test_notifyPipelineComplete_no_channels(mock_get): """Test notifyPipelineComplete returns False when no channels found""" - mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_count': '0' - } + mock_get.return_value = {"instanceId": "test-instance", "channel_count": "0"} - result = notify_slack.notifyPipelineComplete(['#test-channel'], 0, 'test-instance', 'Install') + result = notify_slack.notifyPipelineComplete(["#test-channel"], 0, "test-instance", "Install") assert result is False -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'deleteThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "deleteThreadConfigMap") def test_notifyPipelineComplete_multiple_channels(mock_delete, mock_post, mock_get): """Test notifyPipelineComplete with multiple channels""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_1': 'C456', - 'threadId_1': '1234567890.123457', - 'channel_count': '2' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_1": "C456", + "threadId_1": "1234567890.123457", + "channel_count": "2", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_post.return_value = mock_response - result = notify_slack.notifyPipelineComplete(['#channel1', '#channel2'], 0, 'test-instance', 'Install') + result = notify_slack.notifyPipelineComplete(["#channel1", "#channel2"], 0, "test-instance", "Install") assert result is True assert mock_post.call_count == 2 mock_delete.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'deleteThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "deleteThreadConfigMap") def test_notifyPipelineComplete_with_duration(mock_delete, mock_post, mock_get): """Test notifyPipelineComplete includes duration when startTime is available""" mock_get.return_value = { - 'instanceId': 'test-instance', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1', - 'startTime': '2026-03-10T18:00:00Z' + "instanceId": "test-instance", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", + "startTime": "2026-03-10T18:00:00Z", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_post.return_value = mock_response - result = notify_slack.notifyPipelineComplete(['#test-channel'], 0, 'test-instance', 'Install') + result = notify_slack.notifyPipelineComplete(["#test-channel"], 0, "test-instance", "Install") assert result is True # Verify that postMessageBlocks was called @@ -612,26 +633,26 @@ def test_notifyPipelineComplete_with_duration(mock_delete, mock_post, mock_get): # Tests for update pipeline (no instance ID) scenarios -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'createThreadConfigMap') -@patch.object(SlackUtil, 'updateThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "createThreadConfigMap") +@patch.object(SlackUtil, "updateThreadConfigMap") def test_notifyPipelineStart_update_pipeline_no_instance_id(mock_update, mock_create, mock_post, mock_get): """Test notifyPipelineStart for update pipeline with no instance ID""" # First call returns None, second call returns the created thread info thread_info = { - 'instanceId': '', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_get.side_effect = [None, thread_info] mock_response = Mock() - mock_response.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} - mock_response.__getitem__ = lambda self, key: mock_response.data[key] if key in ['ts', 'channel'] else None + mock_response.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} + mock_response.__getitem__ = lambda self, key: (mock_response.data[key] if key in ["ts", "channel"] else None) mock_post.return_value = mock_response - result = notify_slack.notifyPipelineStart(['#test-channel'], None, 'Update') + result = notify_slack.notifyPipelineStart(["#test-channel"], None, "Update") assert result is not None assert result == thread_info @@ -642,22 +663,22 @@ def test_notifyPipelineStart_update_pipeline_no_instance_id(mock_update, mock_cr mock_update.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'updateThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "updateThreadConfigMap") def test_notifyAnsibleStart_update_pipeline_no_instance_id(mock_update, mock_post, mock_get): """Test notifyAnsibleStart for update pipeline with no instance ID""" mock_get.return_value = { - 'instanceId': '', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True, 'ts': '1234567890.123457'} + mock_response.data = {"ok": True, "ts": "1234567890.123457"} mock_post.return_value = mock_response - result = notify_slack.notifyAnsibleStart(['#test-channel'], 'update-catalog', None, 'Update') + result = notify_slack.notifyAnsibleStart(["#test-channel"], "update-catalog", None, "Update") assert result is True mock_post.assert_called_once() @@ -666,43 +687,43 @@ def test_notifyAnsibleStart_update_pipeline_no_instance_id(mock_update, mock_pos assert mock_update.call_args[0][1] is None -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'updateMessageBlocks') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "updateMessageBlocks") def test_notifyAnsibleComplete_update_pipeline_no_instance_id(mock_update, mock_get): """Test notifyAnsibleComplete for update pipeline with no instance ID""" mock_get.return_value = { - 'instanceId': '', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'task_update-catalog_0': '1234567890.123457', - 'channel_count': '1' + "instanceId": "", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "task_update-catalog_0": "1234567890.123457", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_update.return_value = mock_response - result = notify_slack.notifyAnsibleComplete(['#test-channel'], 0, 'update-catalog', None, 'Update') + result = notify_slack.notifyAnsibleComplete(["#test-channel"], 0, "update-catalog", None, "Update") assert result is True mock_update.assert_called_once() -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'deleteThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "deleteThreadConfigMap") def test_notifyPipelineComplete_update_pipeline_no_instance_id(mock_delete, mock_post, mock_get): """Test notifyPipelineComplete for update pipeline with no instance ID""" mock_get.return_value = { - 'instanceId': '', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_post.return_value = mock_response - result = notify_slack.notifyPipelineComplete(['#test-channel'], 0, None, 'Update') + result = notify_slack.notifyPipelineComplete(["#test-channel"], 0, None, "Update") assert result is True mock_post.assert_called_once() @@ -711,59 +732,59 @@ def test_notifyPipelineComplete_update_pipeline_no_instance_id(mock_delete, mock assert mock_delete.call_args[0][1] is None -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'deleteThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "deleteThreadConfigMap") def test_notifyPipelineComplete_update_pipeline_empty_instance_id(mock_delete, mock_post, mock_get): """Test notifyPipelineComplete for update pipeline with empty string instance ID""" mock_get.return_value = { - 'instanceId': '', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_count': '1' + "instanceId": "", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_count": "1", } mock_response = Mock() - mock_response.data = {'ok': True} + mock_response.data = {"ok": True} mock_post.return_value = mock_response - result = notify_slack.notifyPipelineComplete(['#test-channel'], 0, '', 'Update') + result = notify_slack.notifyPipelineComplete(["#test-channel"], 0, "", "Update") assert result is True mock_post.assert_called_once() mock_delete.assert_called_once() # Verify that deleteThreadConfigMap was called with empty string for instanceId - assert mock_delete.call_args[0][1] == '' + assert mock_delete.call_args[0][1] == "" -@patch.object(SlackUtil, 'getThreadConfigMap') -@patch.object(SlackUtil, 'postMessageBlocks') -@patch.object(SlackUtil, 'createThreadConfigMap') -@patch.object(SlackUtil, 'updateThreadConfigMap') +@patch.object(SlackUtil, "getThreadConfigMap") +@patch.object(SlackUtil, "postMessageBlocks") +@patch.object(SlackUtil, "createThreadConfigMap") +@patch.object(SlackUtil, "updateThreadConfigMap") def test_notifyPipelineStart_update_pipeline_multiple_channels(mock_update, mock_create, mock_post, mock_get): """Test notifyPipelineStart for update pipeline with multiple channels and no instance ID""" # First call returns None, second call returns the created thread info thread_info = { - 'instanceId': '', - 'channel_0': 'C123', - 'threadId_0': '1234567890.123456', - 'channel_1': 'C456', - 'threadId_1': '1234567890.123457', - 'channel_count': '2' + "instanceId": "", + "channel_0": "C123", + "threadId_0": "1234567890.123456", + "channel_1": "C456", + "threadId_1": "1234567890.123457", + "channel_count": "2", } mock_get.side_effect = [None, thread_info] mock_response1 = Mock() - mock_response1.data = {'ok': True, 'channel': 'C123', 'ts': '1234567890.123456'} - mock_response1.__getitem__ = lambda self, key: mock_response1.data[key] if key in ['ts', 'channel'] else None + mock_response1.data = {"ok": True, "channel": "C123", "ts": "1234567890.123456"} + mock_response1.__getitem__ = lambda self, key: (mock_response1.data[key] if key in ["ts", "channel"] else None) mock_response2 = Mock() - mock_response2.data = {'ok': True, 'channel': 'C456', 'ts': '1234567890.123457'} - mock_response2.__getitem__ = lambda self, key: mock_response2.data[key] if key in ['ts', 'channel'] else None + mock_response2.data = {"ok": True, "channel": "C456", "ts": "1234567890.123457"} + mock_response2.__getitem__ = lambda self, key: (mock_response2.data[key] if key in ["ts", "channel"] else None) mock_post.return_value = [mock_response1, mock_response2] - result = notify_slack.notifyPipelineStart(['#channel1', '#channel2'], None, 'Update') + result = notify_slack.notifyPipelineStart(["#channel1", "#channel2"], None, "Update") assert result is not None # Verify that channel_count is set to 2 update_call_args = mock_update.call_args[0][2] - assert update_call_args['channel_count'] == '2' + assert update_call_args["channel_count"] == "2" # Verify namespace is mas-pipelines (not mas-None-pipelines) - assert mock_create.call_args[0][0] == 'mas-pipelines' + assert mock_create.call_args[0][0] == "mas-pipelines" diff --git a/test/src/test_users.py b/test/src/test_users.py index 0307adf3..3636f962 100644 --- a/test/src/test_users.py +++ b/test/src/test_users.py @@ -39,8 +39,8 @@ MANAGE_API_PORT = 3 MAS_ADMIN_URL = f"https://admin-dashboard.{MAS_CORE_NAMESPACE}.svc.cluster.local:{ADMIN_DASHBOARD_PORT}" -MAS_API_URL = f'https://coreapi.{MAS_CORE_NAMESPACE}.svc.cluster.local:{COREAPI_PORT}' -MANAGE_API_URL = f'https://{MAS_INSTANCE_ID}-{MAS_WORKSPACE_ID}.{MANAGE_NAMESPACE}.svc.cluster.local:{MANAGE_API_PORT}' +MAS_API_URL = f"https://coreapi.{MAS_CORE_NAMESPACE}.svc.cluster.local:{COREAPI_PORT}" +MANAGE_API_URL = f"https://{MAS_INSTANCE_ID}-{MAS_WORKSPACE_ID}.{MANAGE_NAMESPACE}.svc.cluster.local:{MANAGE_API_PORT}" PEM_PATH = "pempath" @@ -61,14 +61,10 @@ def get_secret(name, namespace): } if name == f"{MAS_INSTANCE_ID}-admindashboard-cert-internal": - data = { - "ca.crt": base64.b64encode(ADMINDASHBOARD_CA_CRT.encode("utf-8")) - } + data = {"ca.crt": base64.b64encode(ADMINDASHBOARD_CA_CRT.encode("utf-8"))} if name == f"{MAS_INSTANCE_ID}-coreapi-cert-internal": - data = { - "ca.crt": base64.b64encode(COREAPI_CA_CRT.encode("utf-8")) - } + data = {"ca.crt": base64.b64encode(COREAPI_CA_CRT.encode("utf-8"))} if name == f"{MAS_INSTANCE_ID}-internal-manage-tls": data = { @@ -77,20 +73,18 @@ def get_secret(name, namespace): "tls.key": base64.b64encode(MANAGE_TLS_KEY.encode("utf-8")), } - return MagicMock( - data=data - ) + return MagicMock(data=data) @fixture def mock_atexit(): - with patch('atexit.register') as mock_atexit: + with patch("atexit.register") as mock_atexit: yield mock_atexit @fixture def mock_named_temporary_file(mock_atexit): - with patch('tempfile.NamedTemporaryFile') as mock_named_temporary_file: + with patch("tempfile.NamedTemporaryFile") as mock_named_temporary_file: mock_file = MagicMock() mock_file.name = PEM_PATH mock_named_temporary_file.return_value.__enter__.return_value = mock_file @@ -99,7 +93,7 @@ def mock_named_temporary_file(mock_atexit): @fixture def mock_v1_secrets(): - with patch('mas.devops.users.DynamicClient') as mock_DynamicClientCls: + with patch("mas.devops.users.DynamicClient") as mock_DynamicClientCls: mock_DynamicClient = mock_DynamicClientCls.return_value mock_v1_secrets = mock_DynamicClient.resources.get.return_value mock_v1_secrets.get.side_effect = get_secret @@ -111,12 +105,18 @@ def mock_logininitial_endpoint(requests_mock): yield requests_mock.post( f"{MAS_ADMIN_URL}/logininitial", json=dict(token=TOKEN), - additional_matcher=lambda req: additional_matcher(req, json={"username": SUPERUSER_USERNAME, "password": SUPERUSER_PASSWORD}) + additional_matcher=lambda req: additional_matcher(req, json={"username": SUPERUSER_USERNAME, "password": SUPERUSER_PASSWORD}), ) -@fixture(params=['9.0', '9.1']) -def user_utils(request, mock_v1_secrets, mock_logininitial_endpoint, mock_named_temporary_file, mock_atexit): +@fixture(params=["9.0", "9.1"]) +def user_utils( + request, + mock_v1_secrets, + mock_logininitial_endpoint, + mock_named_temporary_file, + mock_atexit, +): k8s_client = MagicMock() # DynamicClient is mocked out, no methods will be called on the k8s_client mas_version = request.param user_utils = MASUserUtils( @@ -126,7 +126,7 @@ def user_utils(request, mock_v1_secrets, mock_logininitial_endpoint, mock_named_ mas_version=mas_version, coreapi_port=COREAPI_PORT, admin_dashboard_port=ADMIN_DASHBOARD_PORT, - manage_api_port=MANAGE_API_PORT + manage_api_port=MANAGE_API_PORT, ) yield user_utils @@ -134,14 +134,22 @@ def user_utils(request, mock_v1_secrets, mock_logininitial_endpoint, mock_named_ @fixture def mock_manage_api_key(requests_mock): - ''' + """ Setup mock Manage APIs for setting up an API Key - ''' + """ user_id = "user1" - apikey = {"userid": user_id, "apikey": "test-api-key-12345", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "test-api-key-12345", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret # Also setup for MXINTADM user - mxintadm_apikey = {"userid": "MXINTADM", "apikey": "mxintadm-api-key-67890", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/mxintadmapikeyid"} # pragma: allowlist secret + mxintadm_apikey = { + "userid": "MXINTADM", + "apikey": "mxintadm-api-key-67890", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/mxintadmapikeyid", + } # pragma: allowlist secret def mxintadm_matcher(req): return req.json().get("userid") == "MXINTADM" and req.verify == PEM_PATH and req.cert == PEM_PATH @@ -153,9 +161,14 @@ def user1_matcher(req): requests_mock.post( f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1", request_headers={"content-type": "application/json"}, - json={"Error": {"reasonCode": "BMXAA10051E", "message": "Only one API key allowed per user"}}, + json={ + "Error": { + "reasonCode": "BMXAA10051E", + "message": "Only one API key allowed per user", + } + }, status_code=400, - additional_matcher=mxintadm_matcher + additional_matcher=mxintadm_matcher, ) # Mock for user1 API key creation @@ -164,25 +177,25 @@ def user1_matcher(req): request_headers={"content-type": "application/json"}, json={"id": user_id}, status_code=201, - additional_matcher=user1_matcher + additional_matcher=user1_matcher, ) # Mock for user1 API key retrieval requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json"}, json={"member": [apikey]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) # Mock for MXINTADM API key retrieval (returns existing key) requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"MXINTADM\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="MXINTADM"', request_headers={"accept": "application/json"}, json={"member": [mxintadm_apikey]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) yield mxintadm_apikey @@ -190,12 +203,20 @@ def user1_matcher(req): def test_admin_internal_ca_pem_file_path(user_utils, mock_named_temporary_file, mock_atexit): assert str(user_utils.admin_internal_ca_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(ADMINDASHBOARD_CA_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(ADMINDASHBOARD_CA_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] # verify caching assert str(user_utils.admin_internal_ca_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(ADMINDASHBOARD_CA_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(ADMINDASHBOARD_CA_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] @@ -206,7 +227,7 @@ def mock_get_user(requests_mock, user_id, json, status_code, mock_manage_api_key request_headers={"x-access-token": TOKEN}, json=json, status_code=status_code, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) # Use separate JSON for Manage API if provided, otherwise use the same @@ -219,7 +240,7 @@ def mock_get_user(requests_mock, user_id, json, status_code, mock_manage_api_key request_headers={"apikey": mock_manage_api_key["apikey"]}, json=manage_json, status_code=status_code, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) # Second request: Mock the query-based request with personid @@ -229,7 +250,7 @@ def mock_get_user(requests_mock, user_id, json, status_code, mock_manage_api_key request_headers={"apikey": mock_manage_api_key["apikey"]}, json=manage_json, status_code=status_code, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) return core_mock, manage_query_mock, manage_personid_mock @@ -237,76 +258,90 @@ def mock_get_user(requests_mock, user_id, json, status_code, mock_manage_api_key def mock_get_user_200(requests_mock, user_id, mock_manage_api_key): # Core API response for version < 9.1 - core_json = { - "id": user_id, - "displayName": user_id - } + core_json = {"id": user_id, "displayName": user_id} # Manage API response for version >= 9.1 # Include member array with href containing resource_id resource_id = f"{user_id}_resource_id" manage_json = { - "member": [{ - "href": f"api/os/masperuser/{resource_id}", - "personid": user_id, - "displayname": user_id - }] + "member": [ + { + "href": f"api/os/masperuser/{resource_id}", + "personid": user_id, + "displayname": user_id, + } + ] } return mock_get_user( - requests_mock, user_id, core_json, 200, mock_manage_api_key, json_manage=manage_json + requests_mock, + user_id, + core_json, + 200, + mock_manage_api_key, + json_manage=manage_json, ) def mock_get_user_404(requests_mock, user_id, mock_manage_api_key): - return mock_get_user( - requests_mock, user_id, {"error": "notfound"}, 404, mock_manage_api_key - ) + return mock_get_user(requests_mock, user_id, {"error": "notfound"}, 404, mock_manage_api_key) def mock_get_user_500(requests_mock, user_id, mock_manage_api_key): - return mock_get_user( - requests_mock, user_id, {"error": "internal"}, 500, mock_manage_api_key - ) + return mock_get_user(requests_mock, user_id, {"error": "internal"}, 500, mock_manage_api_key) def test_mas_superuser_credentials(user_utils, mock_v1_secrets): assert mock_v1_secrets.get.call_count == 0 - assert user_utils.mas_superuser_credentials == {"username": SUPERUSER_USERNAME, "password": SUPERUSER_PASSWORD} + assert user_utils.mas_superuser_credentials == { + "username": SUPERUSER_USERNAME, + "password": SUPERUSER_PASSWORD, + } assert mock_v1_secrets.get.call_count == 1 # verify caching is working - assert user_utils.mas_superuser_credentials == {"username": SUPERUSER_USERNAME, "password": SUPERUSER_PASSWORD} + assert user_utils.mas_superuser_credentials == { + "username": SUPERUSER_USERNAME, + "password": SUPERUSER_PASSWORD, + } assert mock_v1_secrets.get.call_count == 1 def test_admin_internal_tls_secret(user_utils, mock_v1_secrets): assert mock_v1_secrets.get.call_count == 0 - assert user_utils.admin_internal_tls_secret.data["ca.crt"] == base64.b64encode(ADMINDASHBOARD_CA_CRT.encode('utf-8')) + assert user_utils.admin_internal_tls_secret.data["ca.crt"] == base64.b64encode(ADMINDASHBOARD_CA_CRT.encode("utf-8")) assert mock_v1_secrets.get.call_count == 1 - assert user_utils.admin_internal_tls_secret.data["ca.crt"] == base64.b64encode(ADMINDASHBOARD_CA_CRT.encode('utf-8')) + assert user_utils.admin_internal_tls_secret.data["ca.crt"] == base64.b64encode(ADMINDASHBOARD_CA_CRT.encode("utf-8")) assert mock_v1_secrets.get.call_count == 1 def test_core_internal_tls_secret(user_utils, mock_v1_secrets): assert mock_v1_secrets.get.call_count == 0 - assert user_utils.core_internal_tls_secret.data["ca.crt"] == base64.b64encode(COREAPI_CA_CRT.encode('utf-8')) + assert user_utils.core_internal_tls_secret.data["ca.crt"] == base64.b64encode(COREAPI_CA_CRT.encode("utf-8")) assert mock_v1_secrets.get.call_count == 1 - assert user_utils.core_internal_tls_secret.data["ca.crt"] == base64.b64encode(COREAPI_CA_CRT.encode('utf-8')) + assert user_utils.core_internal_tls_secret.data["ca.crt"] == base64.b64encode(COREAPI_CA_CRT.encode("utf-8")) assert mock_v1_secrets.get.call_count == 1 def test_core_internal_ca_pem_file_path(user_utils, mock_named_temporary_file, mock_atexit): - ''' + """ Check the correct content is written to core_internal_ca_pem_file_path tempfile, that an exit handler is registered to delete the temp file, and that the tempfile is only written once (with its path cached) - ''' + """ assert str(user_utils.core_internal_ca_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(COREAPI_CA_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(COREAPI_CA_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] # verify caching assert str(user_utils.core_internal_ca_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(COREAPI_CA_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(COREAPI_CA_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] @@ -322,35 +357,53 @@ def test_superuser_auth_token(user_utils, mock_logininitial_endpoint): def test_manage_internal_tls_secret(user_utils, mock_v1_secrets): assert mock_v1_secrets.get.call_count == 0 - assert user_utils.manage_internal_tls_secret.data["ca.crt"] == base64.b64encode(MANAGE_CA_CRT.encode('utf-8')) - assert user_utils.manage_internal_tls_secret.data["tls.crt"] == base64.b64encode(MANAGE_TLS_CRT.encode('utf-8')) - assert user_utils.manage_internal_tls_secret.data["tls.key"] == base64.b64encode(MANAGE_TLS_KEY.encode('utf-8')) + assert user_utils.manage_internal_tls_secret.data["ca.crt"] == base64.b64encode(MANAGE_CA_CRT.encode("utf-8")) + assert user_utils.manage_internal_tls_secret.data["tls.crt"] == base64.b64encode(MANAGE_TLS_CRT.encode("utf-8")) + assert user_utils.manage_internal_tls_secret.data["tls.key"] == base64.b64encode(MANAGE_TLS_KEY.encode("utf-8")) assert mock_v1_secrets.get.call_count == 1 - assert user_utils.manage_internal_tls_secret.data["ca.crt"] == base64.b64encode(MANAGE_CA_CRT.encode('utf-8')) - assert user_utils.manage_internal_tls_secret.data["tls.crt"] == base64.b64encode(MANAGE_TLS_CRT.encode('utf-8')) - assert user_utils.manage_internal_tls_secret.data["tls.key"] == base64.b64encode(MANAGE_TLS_KEY.encode('utf-8')) + assert user_utils.manage_internal_tls_secret.data["ca.crt"] == base64.b64encode(MANAGE_CA_CRT.encode("utf-8")) + assert user_utils.manage_internal_tls_secret.data["tls.crt"] == base64.b64encode(MANAGE_TLS_CRT.encode("utf-8")) + assert user_utils.manage_internal_tls_secret.data["tls.key"] == base64.b64encode(MANAGE_TLS_KEY.encode("utf-8")) assert mock_v1_secrets.get.call_count == 1 def test_manage_internal_client_pem_file_path(user_utils, mock_named_temporary_file, mock_atexit): assert str(user_utils.manage_internal_client_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(MANAGE_TLS_KEY.encode()), call.write(MANAGE_TLS_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(MANAGE_TLS_KEY.encode()), + call.write(MANAGE_TLS_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] # verify caching assert str(user_utils.manage_internal_client_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(MANAGE_TLS_KEY.encode()), call.write(MANAGE_TLS_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(MANAGE_TLS_KEY.encode()), + call.write(MANAGE_TLS_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] def test_manage_internal_ca_pem_file_path(user_utils, mock_named_temporary_file, mock_atexit): assert str(user_utils.manage_internal_ca_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(MANAGE_CA_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(MANAGE_CA_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] # verify caching assert str(user_utils.manage_internal_ca_pem_file_path) == PEM_PATH - assert mock_named_temporary_file.mock_calls == [call.write(MANAGE_CA_CRT.encode()), call.flush(), call.close()] + assert mock_named_temporary_file.mock_calls == [ + call.write(MANAGE_CA_CRT.encode()), + call.flush(), + call.close(), + ] assert mock_atexit.mock_calls == [call(os.remove, PEM_PATH)] @@ -359,7 +412,7 @@ def test_mas_workspace_application_ids(user_utils, requests_mock): f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications", request_headers={"x-access-token": TOKEN}, json=[{"id": "manage"}, {"id": "iot"}], - status_code=200 + status_code=200, ) assert user_utils.mas_workspace_application_ids == ["manage", "iot"] assert get.call_count == 1 @@ -374,7 +427,7 @@ def test_mas_workspace_application_ids_filters_health(user_utils, requests_mock) f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications", request_headers={"x-access-token": TOKEN}, json=[{"id": "manage"}, {"id": "health"}, {"id": "iot"}], - status_code=200 + status_code=200, ) # health should be filtered out assert user_utils.mas_workspace_application_ids == ["manage", "iot"] @@ -387,21 +440,21 @@ def test_get_user_exists(user_utils, requests_mock, mock_manage_api_key): resource_id, user_data = user_utils.get_user(user_id) # For version >= 9.1, Manage API uses "personid" and "displayname" # For version < 9.1, Core API uses "id" and "displayName" - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert user_data["personid"] == user_id assert user_data["displayname"] == user_id else: assert user_data["id"] == user_id assert user_data["displayName"] == user_id # For version >= 9.1, resource_id should be extracted; for < 9.1, it should be None - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert resource_id is not None assert resource_id == f"{user_id}_resource_id" else: assert resource_id is None # Check that the correct endpoint was called based on version - if user_utils.mas_version >= '9.1': + if user_utils.mas_version >= "9.1": assert get_core.call_count == 0 assert get_manage.call_count == 1 else: @@ -417,7 +470,7 @@ def test_get_user_notfound(user_utils, requests_mock, mock_manage_api_key): assert user_data is None # Check that the correct endpoint was called based on version - if user_utils.mas_version >= '9.1': + if user_utils.mas_version >= "9.1": assert get_core.call_count == 0 assert get_manage.call_count == 1 else: @@ -432,7 +485,7 @@ def test_get_user_error(user_utils, requests_mock, mock_manage_api_key): user_utils.get_user(user_id) # Check that the correct endpoint was called based on version - if user_utils.mas_version >= '9.1': + if user_utils.mas_version >= "9.1": assert get_core.call_count == 0 assert get_manage.call_count == 1 else: @@ -450,7 +503,7 @@ def test_get_or_create_user_exists(user_utils, requests_mock, mock_manage_api_ke request_headers={"x-access-token": TOKEN}, json={"id": user_id}, status_code=201, - additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}), ) # Mock Manage API endpoint for version >= 9.1 @@ -459,11 +512,11 @@ def test_get_or_create_user_exists(user_utils, requests_mock, mock_manage_api_ke request_headers={"apikey": mock_manage_api_key["apikey"]}, json={"id": user_id}, status_code=201, - additional_matcher=lambda req: additional_matcher(req, json={"personid": user_id}, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, json={"personid": user_id}, cert=PEM_PATH), ) # Use correct payload structure based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): payload = {"personid": user_id} else: payload = {"id": user_id} @@ -471,20 +524,20 @@ def test_get_or_create_user_exists(user_utils, requests_mock, mock_manage_api_ke resource_id, user_data = user_utils.get_or_create_user(payload) # For version >= 9.1, Manage API uses "personid" and "displayname" # For version < 9.1, Core API uses "id" and "displayName" - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert user_data["personid"] == user_id assert user_data["displayname"] == user_id else: assert user_data["id"] == user_id assert user_data["displayName"] == user_id # For version >= 9.1, resource_id should be extracted; for < 9.1, it should be None - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert resource_id is not None assert resource_id == f"{user_id}_resource_id" else: assert resource_id is None # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count == 1 else: @@ -504,7 +557,7 @@ def test_get_or_create_user_notfound(user_utils, requests_mock, mock_manage_api_ request_headers={"x-access-token": TOKEN}, json={"id": user_id, "displayName": user_id}, status_code=201, - additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}), ) # Mock Manage API endpoint for version >= 9.1 @@ -513,11 +566,11 @@ def test_get_or_create_user_notfound(user_utils, requests_mock, mock_manage_api_ request_headers={"apikey": mock_manage_api_key["apikey"]}, json={"id": user_id, "displayName": user_id}, status_code=201, - additional_matcher=lambda req: additional_matcher(req, json={"personid": user_id}, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, json={"personid": user_id}, cert=PEM_PATH), ) # Use correct payload structure based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): payload = {"personid": user_id} else: payload = {"id": user_id} @@ -525,10 +578,10 @@ def test_get_or_create_user_notfound(user_utils, requests_mock, mock_manage_api_ resource_id, user_data = user_utils.get_or_create_user(payload) assert user_data == {"id": user_id, "displayName": user_id} # For version >= 9.1, resource_id might be None if not in response; for < 9.1, it should be None - if Version(user_utils.mas_version) < Version('9.1'): + if Version(user_utils.mas_version) < Version("9.1"): assert resource_id is None # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count == 1 assert post_core.call_count == 0 @@ -550,7 +603,7 @@ def test_get_or_create_user_error(user_utils, requests_mock, mock_manage_api_key request_headers={"x-access-token": TOKEN}, json={"error": "unknown"}, status_code=500, - additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}), ) # Mock Manage API endpoint for version >= 9.1 @@ -559,11 +612,11 @@ def test_get_or_create_user_error(user_utils, requests_mock, mock_manage_api_key request_headers={"apikey": mock_manage_api_key["apikey"]}, json={"error": "unknown"}, status_code=500, - additional_matcher=lambda req: additional_matcher(req, json={"personid": user_id}, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, json={"personid": user_id}, cert=PEM_PATH), ) # Use correct payload structure based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): payload = {"personid": user_id} else: payload = {"id": user_id} @@ -571,7 +624,7 @@ def test_get_or_create_user_error(user_utils, requests_mock, mock_manage_api_key with pytest.raises(Exception): user_utils.get_or_create_user(payload) # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count == 1 assert post_core.call_count == 0 @@ -590,7 +643,7 @@ def test_update_user(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"id": user_id}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}), ) user_utils.update_user({"id": user_id}) assert put.call_count == 1 @@ -603,7 +656,7 @@ def test_update_user_error(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"error": "nofound"}, status_code=404, - additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"id": user_id}), ) with pytest.raises(Exception): user_utils.update_user({"id": user_id}) @@ -617,7 +670,7 @@ def test_update_user_display_name(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"id": user_id}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"displayName": "display_name"}) + additional_matcher=lambda req: additional_matcher(req, json={"displayName": "display_name"}), ) user_utils.update_user_display_name(user_id, "display_name") assert patche.call_count == 1 @@ -630,7 +683,7 @@ def test_update_user_display_name_error(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"error": "notfound"}, status_code=404, - additional_matcher=lambda req: additional_matcher(req, json={"displayName": "display_name"}) + additional_matcher=lambda req: additional_matcher(req, json={"displayName": "display_name"}), ) with pytest.raises(Exception): user_utils.update_user_display_name(user_id, "display_name") @@ -649,7 +702,7 @@ def test_link_user_to_local_idp(user_utils, requests_mock, mock_manage_api_key): request_headers={"x-access-token": TOKEN}, json={"id": user_id}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"idpUserId": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"idpUserId": user_id}), ) # Mock Manage API PATCH request for version >= 9.1 @@ -659,7 +712,7 @@ def test_link_user_to_local_idp(user_utils, requests_mock, mock_manage_api_key): "Content-Type": "application/json", "apikey": mock_manage_api_key["apikey"], "x-method-override": "PATCH", - "patchtype": "MERGE" + "patchtype": "MERGE", }, json={"id": user_id}, status_code=200, @@ -668,28 +721,35 @@ def test_link_user_to_local_idp(user_utils, requests_mock, mock_manage_api_key): json={ "maxuser": { "userid": user_id, - "masuseridp": [{ - "emailpassword": True, - "idpid": "local", - "logintype": "0", - "idploginid": user_id, - "idptype": "local", - "enabled": True - }] + "masuseridp": [ + { + "emailpassword": True, + "idpid": "local", + "logintype": "0", + "idploginid": user_id, + "idptype": "local", + "enabled": True, + } + ], } }, - cert=PEM_PATH - ) + cert=PEM_PATH, + ), ) # Call the function with appropriate parameters based on version - if Version(user_utils.mas_version) >= Version('9.1'): - user_utils.link_user_to_local_idp(user_id, email_password=email_password, manage_api_key=mock_manage_api_key, resource_id=resource_id) + if Version(user_utils.mas_version) >= Version("9.1"): + user_utils.link_user_to_local_idp( + user_id, + email_password=email_password, + manage_api_key=mock_manage_api_key, + resource_id=resource_id, + ) else: user_utils.link_user_to_local_idp(user_id, email_password=email_password) # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count == 1 assert put.call_count == 0 @@ -708,22 +768,22 @@ def test_link_user_to_local_idp_usernotfound(user_utils, requests_mock, mock_man put = requests_mock.put( f"{MAS_API_URL}/v3/users/{user_id}/idps/local", - additional_matcher=lambda req: additional_matcher(req, json={"idpUserId": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"idpUserId": user_id}), ) patch = requests_mock.post( f"{MANAGE_API_URL}/maximo/api/os/masperuser/{resource_id}?lean=1&ccm=1", - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) with pytest.raises(Exception): - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): user_utils.link_user_to_local_idp(user_id, manage_api_key=mock_manage_api_key, resource_id=resource_id) else: user_utils.link_user_to_local_idp(user_id) # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count == 1 else: @@ -744,13 +804,15 @@ def test_link_user_to_local_idp_already_linked(user_utils, requests_mock, mock_m 200, mock_manage_api_key, json_manage={ - "member": [{ - "href": f"api/os/masperuser/{resource_id}", - "personid": user_id, - "displayname": user_id, - "identities": {"_local": {}} - }] - } + "member": [ + { + "href": f"api/os/masperuser/{resource_id}", + "personid": user_id, + "displayname": user_id, + "identities": {"_local": {}}, + } + ] + }, ) put = requests_mock.put( @@ -758,22 +820,27 @@ def test_link_user_to_local_idp_already_linked(user_utils, requests_mock, mock_m request_headers={"x-access-token": TOKEN}, json={"identities": {}}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"idpUserId": user_id}) + additional_matcher=lambda req: additional_matcher(req, json={"idpUserId": user_id}), ) patch = requests_mock.post( f"{MANAGE_API_URL}/maximo/api/os/masperuser/{resource_id}?lean=1&ccm=1", - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) # Call the function with appropriate parameters based on version - if Version(user_utils.mas_version) >= Version('9.1'): - user_utils.link_user_to_local_idp(user_id, email_password=email_password, manage_api_key=mock_manage_api_key, resource_id=resource_id) + if Version(user_utils.mas_version) >= Version("9.1"): + user_utils.link_user_to_local_idp( + user_id, + email_password=email_password, + manage_api_key=mock_manage_api_key, + resource_id=resource_id, + ) else: user_utils.link_user_to_local_idp(user_id, email_password=email_password) # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count == 1 else: @@ -790,7 +857,7 @@ def test_get_user_workspaces(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json=[{"id": "masdev"}], status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) workspaces = user_utils.get_user_workspaces(user_id) assert workspaces == [{"id": "masdev"}] @@ -804,7 +871,7 @@ def test_get_user_workspaces_usernotfound(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={}, status_code=404, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) with pytest.raises(Exception): user_utils.get_user_workspaces(user_id) @@ -818,7 +885,7 @@ def test_get_user_workspaces_error(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"error": "internal"}, status_code=500, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) with pytest.raises(Exception): user_utils.get_user_workspaces(user_id) @@ -832,14 +899,14 @@ def test_add_user_to_workspace_already_a_member(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json=[{"id": "someotherworkspace"}, {"id": MAS_WORKSPACE_ID}], status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) put = requests_mock.put( f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/users/{user_id}", request_headers={"x-access-token": TOKEN}, json=[{"id": "masdev"}], status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"permissions": {"workspaceAdmin": True}}) + additional_matcher=lambda req: additional_matcher(req, json={"permissions": {"workspaceAdmin": True}}), ) user_utils.add_user_to_workspace(user_id, is_workspace_admin=True) assert get.call_count == 1 @@ -852,14 +919,14 @@ def test_add_user_to_workspace(user_utils, requests_mock): f"{MAS_API_URL}/v3/users/{user_id}/workspaces", request_headers={"x-access-token": TOKEN}, json=[{"id": "someotherworkspace"}], - status_code=200 + status_code=200, ) put = requests_mock.put( f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/users/{user_id}", request_headers={"x-access-token": TOKEN}, json={}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"permissions": {"workspaceAdmin": True}}) + additional_matcher=lambda req: additional_matcher(req, json={"permissions": {"workspaceAdmin": True}}), ) user_utils.add_user_to_workspace(user_id, is_workspace_admin=True) assert get.call_count == 1 @@ -872,14 +939,14 @@ def test_add_user_to_workspace_error(user_utils, requests_mock): f"{MAS_API_URL}/v3/users/{user_id}/workspaces", request_headers={"x-access-token": TOKEN}, json=[{"id": "someotherworkspace"}], - status_code=200 + status_code=200, ) put = requests_mock.put( f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/users/{user_id}", request_headers={"x-access-token": TOKEN}, json={"error": "internal"}, status_code=500, - additional_matcher=lambda req: additional_matcher(req, json={"permissions": {"workspaceAdmin": True}}) + additional_matcher=lambda req: additional_matcher(req, json={"permissions": {"workspaceAdmin": True}}), ) with pytest.raises(Exception): user_utils.add_user_to_workspace(user_id, is_workspace_admin=True) @@ -895,14 +962,14 @@ def test_get_user_application_permissions(user_utils, requests_mock): "userId": user_id, "workspaceId": MAS_WORKSPACE_ID, "userUrl": "https://api.yourmasdomain.com/users/joebloggs", - "workspaceUrl": "https://api.yourmasdomain.com/workspaces/myworkspace1" + "workspaceUrl": "https://api.yourmasdomain.com/workspaces/myworkspace1", } get = requests_mock.get( f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications/{application_id}/users/{user_id}", request_headers={"x-access-token": TOKEN}, json=response_json, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) assert user_utils.get_user_application_permissions(user_id, application_id) == response_json assert get.call_count == 1 @@ -916,7 +983,7 @@ def test_get_user_application_permissions_notfound(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"error": "notfound"}, status_code=404, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) assert user_utils.get_user_application_permissions(user_id, application_id) is None assert get.call_count == 1 @@ -930,7 +997,7 @@ def test_get_user_application_permissions_error(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"error": "internal"}, status_code=500, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) with pytest.raises(Exception): user_utils.get_user_application_permissions(user_id, application_id) @@ -945,14 +1012,14 @@ def test_set_user_application_permissions(user_utils, requests_mock): request_headers={"x-access-token": TOKEN}, json={"error": "notfound"}, status_code=404, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) put = requests_mock.put( f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications/{application_id}/users/{user_id}", request_headers={"x-access-token": TOKEN}, json={}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"role": "USER"}) + additional_matcher=lambda req: additional_matcher(req, json={"role": "USER"}), ) user_utils.set_user_application_permission(user_id, application_id, "USER") assert get.call_count == 1 @@ -967,21 +1034,21 @@ def test_set_user_application_permissions_alreadyset(user_utils, requests_mock): "userId": user_id, "workspaceId": MAS_WORKSPACE_ID, "userUrl": "https://api.yourmasdomain.com/users/joebloggs", - "workspaceUrl": "https://api.yourmasdomain.com/workspaces/myworkspace1" + "workspaceUrl": "https://api.yourmasdomain.com/workspaces/myworkspace1", } get = requests_mock.get( f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications/{application_id}/users/{user_id}", request_headers={"x-access-token": TOKEN}, json=get_response_json, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) put = requests_mock.put( f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications/{application_id}/users/{user_id}", request_headers={"x-access-token": TOKEN}, json={}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, json={"role": "USER"}) + additional_matcher=lambda req: additional_matcher(req, json={"role": "USER"}), ) user_utils.set_user_application_permission(user_id, application_id, "USER") assert get.call_count == 1 @@ -1006,14 +1073,14 @@ def test_resync_users(user_utils, requests_mock, mock_manage_api_key): json={"id": user_id}, status_code=200, # uid=user_id captures the current value of user_id during each loop iteration, ensuring that the lambda uses the correct value when it is eventually called. - additional_matcher=lambda req, uid=user_id: additional_matcher(req, json={"displayName": uid}) + additional_matcher=lambda req, uid=user_id: additional_matcher(req, json={"displayName": uid}), ) ) user_utils.resync_users(user_ids) # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): for get_core in gets_core: assert get_core.call_count == 0 for get_manage in gets_manage: @@ -1030,7 +1097,7 @@ def test_resync_users(user_utils, requests_mock, mock_manage_api_key): def test_check_user_sync(user_utils, requests_mock, mock_manage_api_key): # Skip for version >= 9.1 as Manage API doesn't return applications field - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): pytest.skip("check_user_sync not applicable for version >= 9.1 (Manage API doesn't return applications field)") user_id = "user1" @@ -1050,17 +1117,9 @@ def json_callback_core(request, context): return { "id": user_id, "applications": { - "other": { - "sync": { - "state": "ERROR" - } - }, - application_id: { - "sync": { - "state": state - } - } - } + "other": {"sync": {"state": "ERROR"}}, + application_id: {"sync": {"state": state}}, + }, } def json_callback_manage(request, context): @@ -1070,11 +1129,13 @@ def json_callback_manage(request, context): resource_id = f"{user_id}_resource_id" # Manage API doesn't return applications field for version >= 9.1 return { - "member": [{ - "href": f"api/os/masperuser/{resource_id}", - "personid": user_id, - "displayname": user_id - }] + "member": [ + { + "href": f"api/os/masperuser/{resource_id}", + "personid": user_id, + "displayname": user_id, + } + ] } get_core, get_manage, get_manage_personid = mock_get_user( @@ -1083,7 +1144,7 @@ def json_callback_manage(request, context): json_callback_core, 200, mock_manage_api_key, - json_manage=json_callback_manage + json_manage=json_callback_manage, ) user_utils.check_user_sync(user_id, application_id, timeout_secs=8, retry_interval_secs=0) @@ -1091,7 +1152,7 @@ def json_callback_manage(request, context): # Check that the correct endpoint was called based on version # Note: For version >= 9.1, get_user makes 2 requests (query + resource_id GET) # but we only track the first query request in get_manage mock - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 # Each get_user call makes 2 requests, but we only count the query request assert get_manage.call_count == 3 @@ -1102,7 +1163,7 @@ def json_callback_manage(request, context): def test_check_user_sync_timeout(user_utils, requests_mock, mock_manage_api_key): # Skip for version >= 9.1 as Manage API doesn't return applications field - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): pytest.skip("check_user_sync not applicable for version >= 9.1 (Manage API doesn't return applications field)") user_id = "user1" @@ -1115,34 +1176,28 @@ def test_check_user_sync_timeout(user_utils, requests_mock, mock_manage_api_key) { "id": user_id, "applications": { - "other": { - "sync": { - "state": "ERROR" - } - }, - application_id: { - "sync": { - "state": "PENDING" - } - } - } + "other": {"sync": {"state": "ERROR"}}, + application_id: {"sync": {"state": "PENDING"}}, + }, }, 200, mock_manage_api_key, json_manage={ - "member": [{ - "href": f"api/os/masperuser/{resource_id}", - "personid": user_id, - "displayname": user_id - }] - } + "member": [ + { + "href": f"api/os/masperuser/{resource_id}", + "personid": user_id, + "displayname": user_id, + } + ] + }, ) with pytest.raises(Exception) as excinfo: user_utils.check_user_sync(user_id, application_id, timeout_secs=0.3, retry_interval_secs=0.05) assert str(excinfo.value) == f"User {user_id} sync failed to complete for app within {0.3} seconds" # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count > 1 else: @@ -1152,7 +1207,7 @@ def test_check_user_sync_timeout(user_utils, requests_mock, mock_manage_api_key) def test_check_user_sync_appstate_notfound(user_utils, requests_mock, mock_manage_api_key): # Skip for version >= 9.1 as Manage API doesn't return applications field - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): pytest.skip("check_user_sync not applicable for version >= 9.1 (Manage API doesn't return applications field)") user_id = "user1" @@ -1170,29 +1225,17 @@ def json_callback_core(request, context): "id": user_id, "displayName": user_id, "applications": { - "other": { - "sync": { - "state": "ERROR" - } - }, - application_id: { - "sync": { - "state": "SUCCESS" - } - } - } + "other": {"sync": {"state": "ERROR"}}, + application_id: {"sync": {"state": "SUCCESS"}}, + }, } else: ret = { "id": user_id, "displayName": user_id, "applications": { - "other": { - "sync": { - "state": "ERROR" - } - }, - } + "other": {"sync": {"state": "ERROR"}}, + }, } attempts = attempts + 1 return ret @@ -1202,11 +1245,13 @@ def json_callback_manage(request, context): resource_id = f"{user_id}_resource_id" # Manage API doesn't return applications field for version >= 9.1 ret = { - "member": [{ - "href": f"api/os/masperuser/{resource_id}", - "personid": user_id, - "displayname": user_id - }] + "member": [ + { + "href": f"api/os/masperuser/{resource_id}", + "personid": user_id, + "displayname": user_id, + } + ] } attempts = attempts + 1 return ret @@ -1215,7 +1260,7 @@ def json_callback_manage(request, context): f"{MAS_API_URL}/v3/users/{user_id}", request_headers={"x-access-token": TOKEN}, json={"id": user_id}, - status_code=200 + status_code=200, ) get_core, get_manage, get_manage_personid = mock_get_user( @@ -1224,13 +1269,13 @@ def json_callback_manage(request, context): json_callback_core, 200, mock_manage_api_key, - json_manage=json_callback_manage + json_manage=json_callback_manage, ) user_utils.check_user_sync(user_id, application_id, timeout_secs=8, retry_interval_secs=0) # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 # For version >= 9.1, each get_user call makes 2 requests assert get_manage.call_count == 3 @@ -1245,7 +1290,7 @@ def json_callback_manage(request, context): def test_check_user_sync_appstate_transient_error(user_utils, requests_mock, mock_manage_api_key): # Skip for version >= 9.1 as Manage API doesn't return applications field - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): pytest.skip("check_user_sync not applicable for version >= 9.1 (Manage API doesn't return applications field)") user_id = "user1" @@ -1262,25 +1307,13 @@ def json_callback_core(request, context): ret = { "id": user_id, "displayName": user_id, - "applications": { - application_id: { - "sync": { - "state": "SUCCESS" - } - } - } + "applications": {application_id: {"sync": {"state": "SUCCESS"}}}, } else: ret = { "id": user_id, "displayName": user_id, - "applications": { - application_id: { - "sync": { - "state": "ERROR" - } - } - } + "applications": {application_id: {"sync": {"state": "ERROR"}}}, } attempts = attempts + 1 return ret @@ -1290,11 +1323,13 @@ def json_callback_manage(request, context): resource_id = f"{user_id}_resource_id" # Manage API doesn't return applications field for version >= 9.1 ret = { - "member": [{ - "href": f"api/os/masperuser/{resource_id}", - "personid": user_id, - "displayname": user_id - }] + "member": [ + { + "href": f"api/os/masperuser/{resource_id}", + "personid": user_id, + "displayname": user_id, + } + ] } attempts = attempts + 1 return ret @@ -1303,7 +1338,7 @@ def json_callback_manage(request, context): f"{MAS_API_URL}/v3/users/{user_id}", request_headers={"x-access-token": TOKEN}, json={"id": user_id}, - status_code=200 + status_code=200, ) get_core, get_manage, get_manage_personid = mock_get_user( @@ -1312,13 +1347,13 @@ def json_callback_manage(request, context): json_callback_core, 200, mock_manage_api_key, - json_manage=json_callback_manage + json_manage=json_callback_manage, ) user_utils.check_user_sync(user_id, application_id, timeout_secs=8, retry_interval_secs=0) # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 # For version >= 9.1, each get_user call makes 2 requests assert get_manage.call_count == 3 @@ -1339,7 +1374,7 @@ def test_check_user_sync_appstate_persistent_error(user_utils, requests_mock, mo f"{MAS_API_URL}/v3/users/{user_id}", request_headers={"x-access-token": TOKEN}, json={"id": user_id}, - status_code=200 + status_code=200, ) resource_id = f"{user_id}_resource_id" @@ -1349,23 +1384,19 @@ def test_check_user_sync_appstate_persistent_error(user_utils, requests_mock, mo { "id": user_id, "displayName": user_id, - "applications": { - application_id: { - "sync": { - "state": "ERROR" - } - } - } + "applications": {application_id: {"sync": {"state": "ERROR"}}}, }, 200, mock_manage_api_key, json_manage={ - "member": [{ - "href": f"api/os/masperuser/{resource_id}", - "personid": user_id, - "displayname": user_id - }] - } + "member": [ + { + "href": f"api/os/masperuser/{resource_id}", + "personid": user_id, + "displayname": user_id, + } + ] + }, ) with pytest.raises(Exception) as excinfo: @@ -1373,7 +1404,7 @@ def test_check_user_sync_appstate_persistent_error(user_utils, requests_mock, mo assert str(excinfo.value) == f"User {user_id} sync failed to complete for app within {0.3} seconds" # Check that the correct endpoint was called based on version - if Version(user_utils.mas_version) >= Version('9.1'): + if Version(user_utils.mas_version) >= Version("9.1"): assert get_core.call_count == 0 assert get_manage.call_count > 1 # an "update_user_display_name" should have been triggered for every 2 get calls (1 call by check_user_sync, 1 by resync) @@ -1387,14 +1418,17 @@ def test_check_user_sync_appstate_persistent_error(user_utils, requests_mock, mo def test_get_manage_api_key_for_user_exists(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} + apikey = { + "userid": user_id, + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json"}, json={"member": [apikey]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) assert user_utils.get_manage_api_key_for_user(user_id) == apikey @@ -1405,11 +1439,11 @@ def test_get_manage_api_key_for_user_notfound(user_utils, requests_mock): user_id = "user1" get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json"}, json={"member": []}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) assert user_utils.get_manage_api_key_for_user(user_id) is None @@ -1420,11 +1454,11 @@ def test_get_manage_api_key_for_user_error(user_utils, requests_mock): user_id = "user1" get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json"}, text="boom", status_code=500, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) with pytest.raises(Exception) as excinfo: @@ -1436,22 +1470,25 @@ def test_get_manage_api_key_for_user_error(user_utils, requests_mock): @pytest.mark.parametrize("temporary", [(True), (False)]) def test_create_or_get_manage_api_key_for_user_new_api_key(temporary, user_utils, requests_mock, mock_atexit): user_id = "user1" - apikey = {"userid": user_id, "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} + apikey = { + "userid": user_id, + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } post = requests_mock.post( f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1", request_headers={"content-type": "application/json"}, json={"id": user_id}, status_code=201, - additional_matcher=lambda req: additional_matcher(req, json={"expiration": -1, "userid": user_id}, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, json={"expiration": -1, "userid": user_id}, cert=PEM_PATH), ) get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json"}, json={"member": [apikey]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) assert user_utils.create_or_get_manage_api_key_for_user(user_id, temporary=temporary) == apikey @@ -1460,30 +1497,37 @@ def test_create_or_get_manage_api_key_for_user_new_api_key(temporary, user_utils # if temporary, check we registered the exit hook to delete the temporary Manage API Key if temporary: - assert call(user_utils.delete_manage_api_key, apikey) in mock_atexit.mock_calls, "delete_manage_api_key exit hook not registered for temporary api key that we created" + assert ( + call(user_utils.delete_manage_api_key, apikey) in mock_atexit.mock_calls + ), "delete_manage_api_key exit hook not registered for temporary api key that we created" else: - assert call(user_utils.delete_manage_api_key, apikey) not in mock_atexit.mock_calls, "delete_manage_api_key exit hook registered unexpectedly for non-temporary api key that we created" + assert ( + call(user_utils.delete_manage_api_key, apikey) not in mock_atexit.mock_calls + ), "delete_manage_api_key exit hook registered unexpectedly for non-temporary api key that we created" @pytest.mark.parametrize("temporary", [(True), (False)]) def test_create_or_get_manage_api_key_for_user_existing_api_key(temporary, user_utils, requests_mock, mock_atexit): user_id = "user1" - apikey = {"userid": user_id, "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} + apikey = { + "userid": user_id, + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } post = requests_mock.post( f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1", request_headers={"content-type": "application/json"}, json={"Error": {"reasonCode": "BMXAA10051E"}}, status_code=400, - additional_matcher=lambda req: additional_matcher(req, json={"expiration": -1, "userid": user_id}, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, json={"expiration": -1, "userid": user_id}, cert=PEM_PATH), ) get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json"}, json={"member": [apikey]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) assert user_utils.create_or_get_manage_api_key_for_user(user_id, temporary=temporary) == apikey @@ -1491,27 +1535,32 @@ def test_create_or_get_manage_api_key_for_user_existing_api_key(temporary, user_ assert get.call_count == 1 # even if temporary is set, because we did not create the api key, we should not registered a hook to delete it - assert call(user_utils.delete_manage_api_key, apikey) not in mock_atexit.mock_calls, "delete_manage_api_key exit hook registered unexpectedly for existing API Key that we did not create" + assert ( + call(user_utils.delete_manage_api_key, apikey) not in mock_atexit.mock_calls + ), "delete_manage_api_key exit hook registered unexpectedly for existing API Key that we did not create" def test_create_or_get_manage_api_key_for_user_error(user_utils, requests_mock, mock_atexit): user_id = "user1" - apikey = {"userid": user_id, "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} + apikey = { + "userid": user_id, + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } post = requests_mock.post( f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1", request_headers={"content-type": "application/json"}, text="boom", status_code=400, - additional_matcher=lambda req: additional_matcher(req, json={"expiration": -1, "userid": user_id}, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, json={"expiration": -1, "userid": user_id}, cert=PEM_PATH), ) get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapiapikey?ccm=1&lean=1&oslc.select=*&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json"}, json={"member": [apikey]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) with pytest.raises(Exception) as excinfo: @@ -1519,20 +1568,25 @@ def test_create_or_get_manage_api_key_for_user_error(user_utils, requests_mock, assert str(excinfo.value) == "400 boom" assert post.call_count == 1 assert get.call_count == 0 - assert call(user_utils.delete_manage_api_key, apikey) not in mock_atexit.mock_calls, "delete_manage_api_key exit hook not registered even though we failed to create the api key" + assert ( + call(user_utils.delete_manage_api_key, apikey) not in mock_atexit.mock_calls + ), "delete_manage_api_key exit hook not registered even though we failed to create the api key" def test_delete_manage_api_key(user_utils, requests_mock): user_id = "user1" apikey_id = "theapikeyid" - apikey = {"userid": user_id, "href": f"https://{MANAGE_API_URL}:{MANAGE_API_PORT}/maximo/api/os/mxapiapikey/{apikey_id}"} + apikey = { + "userid": user_id, + "href": f"https://{MANAGE_API_URL}:{MANAGE_API_PORT}/maximo/api/os/mxapiapikey/{apikey_id}", + } delete = requests_mock.delete( f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey/{apikey_id}?ccm=1&lean=1", request_headers={"accept": "application/json"}, text="notused", status_code=204, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) user_utils.delete_manage_api_key(apikey) @@ -1542,14 +1596,17 @@ def test_delete_manage_api_key(user_utils, requests_mock): def test_delete_manage_api_key_notfound(user_utils, requests_mock): user_id = "user1" apikey_id = "theapikeyid" - apikey = {"userid": user_id, "href": f"https://{MANAGE_API_URL}:{MANAGE_API_PORT}/maximo/api/os/mxapiapikey/{apikey_id}"} + apikey = { + "userid": user_id, + "href": f"https://{MANAGE_API_URL}:{MANAGE_API_PORT}/maximo/api/os/mxapiapikey/{apikey_id}", + } delete = requests_mock.delete( f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey/{apikey_id}?ccm=1&lean=1", request_headers={"accept": "application/json"}, text="notused", status_code=404, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) user_utils.delete_manage_api_key(apikey) @@ -1566,7 +1623,7 @@ def test_delete_manage_api_key_bad_href(user_utils, requests_mock): request_headers={"accept": "application/json"}, text="notused", status_code=204, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) with pytest.raises(Exception) as excinfo: @@ -1578,14 +1635,17 @@ def test_delete_manage_api_key_bad_href(user_utils, requests_mock): def test_delete_manage_api_key_error(user_utils, requests_mock): user_id = "user1" apikey_id = "theapikeyid" - apikey = {"userid": user_id, "href": f"https://{MANAGE_API_URL}:{MANAGE_API_PORT}/maximo/api/os/mxapiapikey/{apikey_id}"} + apikey = { + "userid": user_id, + "href": f"https://{MANAGE_API_URL}:{MANAGE_API_PORT}/maximo/api/os/mxapiapikey/{apikey_id}", + } delete = requests_mock.delete( f"{MANAGE_API_URL}/maximo/api/os/mxapiapikey/{apikey_id}?ccm=1&lean=1", request_headers={"accept": "application/json"}, text="boom", status_code=500, - additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH) + additional_matcher=lambda req: additional_matcher(req, cert=PEM_PATH), ) with pytest.raises(Exception) as excinfo: @@ -1596,16 +1656,20 @@ def test_delete_manage_api_key_error(user_utils, requests_mock): def test_get_manage_group_id(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, json={"member": [{"maxgroupid": group_id}]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) assert user_utils.get_manage_group_id(group_name, apikey) == group_id @@ -1614,15 +1678,19 @@ def test_get_manage_group_id(user_utils, requests_mock): def test_get_manage_group_id_error(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, text="boom", status_code=500, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) with pytest.raises(Exception) as excinfo: user_utils.get_manage_group_id(group_name, apikey) @@ -1632,15 +1700,19 @@ def test_get_manage_group_id_error(user_utils, requests_mock): def test_get_manage_group_id_notfound(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" get = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, json={"member": [{}]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) assert user_utils.get_manage_group_id(group_name, apikey) is None assert get.call_count == 1 @@ -1648,24 +1720,28 @@ def test_get_manage_group_id_notfound(user_utils, requests_mock): def test_is_user_in_manage_group_yes(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get_group_id = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json"}, json={"member": [{"maxgroupid": group_id}], "apikey": apikey["apikey"]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) get_group_user = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, json={"member": [{}]}, # <--- member length non-empty indicates that the user is a member of the group status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) assert user_utils.is_user_in_manage_group(group_name, user_id, apikey) @@ -1675,24 +1751,28 @@ def test_is_user_in_manage_group_yes(user_utils, requests_mock): def test_is_user_in_manage_group_no(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get_group_id = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json"}, json={"member": [{"maxgroupid": group_id}], "apikey": apikey["apikey"]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) get_group_user = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, json={"member": []}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) assert not user_utils.is_user_in_manage_group(group_name, user_id, apikey) @@ -1702,24 +1782,28 @@ def test_is_user_in_manage_group_no(user_utils, requests_mock): def test_is_user_in_manage_group_error(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get_group_id = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json"}, json={"member": [{"maxgroupid": group_id}], "apikey": apikey["apikey"]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) get_group_user = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, text="boom", status_code=500, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) with pytest.raises(Exception) as excinfo: @@ -1731,24 +1815,28 @@ def test_is_user_in_manage_group_error(user_utils, requests_mock): def test_is_user_in_manage_group_no_group_found(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get_group_id = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json"}, json={"member": [], "apikey": apikey["apikey"]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) get_group_user = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid=\"{user_id}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup/{group_id}/groupuser?lean=1&oslc.where=userid="{user_id}"', request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, text="boom", status_code=500, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) with pytest.raises(Exception) as excinfo: @@ -1760,16 +1848,20 @@ def test_is_user_in_manage_group_no_group_found(user_utils, requests_mock): def test_add_user_to_manage_group(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get_group_id = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json"}, json={"member": [{"maxgroupid": group_id}], "apikey": apikey["apikey"]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) get_group_user = requests_mock.get( @@ -1777,7 +1869,7 @@ def test_add_user_to_manage_group(user_utils, requests_mock): request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, json={"member": []}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) add_group_user = requests_mock.post( @@ -1787,11 +1879,11 @@ def test_add_user_to_manage_group(user_utils, requests_mock): "content-type": "application/json", "x-method-override": "PATCH", "patchtype": "MERGE", - "apikey": apikey["apikey"] + "apikey": apikey["apikey"], }, json={}, status_code=204, - additional_matcher=lambda req: additional_matcher(req, json={"groupuser": [{"userid": user_id}]}) + additional_matcher=lambda req: additional_matcher(req, json={"groupuser": [{"userid": user_id}]}), ) assert user_utils.add_user_to_manage_group(user_id, group_name, apikey) is None @@ -1802,16 +1894,20 @@ def test_add_user_to_manage_group(user_utils, requests_mock): def test_add_user_to_manage_group_already_member(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get_group_id = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json"}, json={"member": [{"maxgroupid": group_id}], "apikey": apikey["apikey"]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) get_group_user = requests_mock.get( @@ -1819,7 +1915,7 @@ def test_add_user_to_manage_group_already_member(user_utils, requests_mock): request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, json={"member": [{}]}, # <--- member length non-empty indicates that the user is a member of the group status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) add_group_user = requests_mock.post( @@ -1829,11 +1925,11 @@ def test_add_user_to_manage_group_already_member(user_utils, requests_mock): "content-type": "application/json", "x-method-override": "PATCH", "patchtype": "MERGE", - "apikey": apikey["apikey"] + "apikey": apikey["apikey"], }, json={}, status_code=204, - additional_matcher=lambda req: additional_matcher(req, json={"groupuser": [{"userid": user_id}]}) + additional_matcher=lambda req: additional_matcher(req, json={"groupuser": [{"userid": user_id}]}), ) assert user_utils.add_user_to_manage_group(user_id, group_name, apikey) is None @@ -1844,16 +1940,20 @@ def test_add_user_to_manage_group_already_member(user_utils, requests_mock): def test_add_user_to_manage_group_error(user_utils, requests_mock): user_id = "user1" - apikey = {"userid": user_id, "apikey": "342fwasdasd", "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid"} # pragma: allowlist secret + apikey = { + "userid": user_id, + "apikey": "342fwasdasd", + "href": f"https://{MANAGE_API_URL}/maximo/api/os/mxapikey/theapikeyid", + } # pragma: allowlist secret group_name = "thegroup" group_id = "39231234" get_group_id = requests_mock.get( - f"{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname=\"{group_name}\"", + f'{MANAGE_API_URL}/maximo/api/os/mxapigroup?ccm=1&lean=1&oslc.select=maxgroupid&oslc.where=groupname="{group_name}"', request_headers={"accept": "application/json"}, json={"member": [{"maxgroupid": group_id}], "apikey": apikey["apikey"]}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) get_group_user = requests_mock.get( @@ -1861,7 +1961,7 @@ def test_add_user_to_manage_group_error(user_utils, requests_mock): request_headers={"accept": "application/json", "apikey": apikey["apikey"]}, json={"member": []}, status_code=200, - additional_matcher=lambda req: additional_matcher(req) + additional_matcher=lambda req: additional_matcher(req), ) add_group_user = requests_mock.post( @@ -1871,11 +1971,11 @@ def test_add_user_to_manage_group_error(user_utils, requests_mock): "content-type": "application/json", "x-method-override": "PATCH", "patchtype": "MERGE", - "apikey": apikey["apikey"] + "apikey": apikey["apikey"], }, text="boom", status_code=500, - additional_matcher=lambda req: additional_matcher(req, json={"groupuser": [{"userid": user_id}]}) + additional_matcher=lambda req: additional_matcher(req, json={"groupuser": [{"userid": user_id}]}), ) with pytest.raises(Exception) as excinfo: user_utils.add_user_to_manage_group(user_id, group_name, apikey) @@ -1890,7 +1990,7 @@ def test_get_mas_applications_in_workspace(user_utils, requests_mock): f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications", request_headers={"x-access-token": TOKEN}, json=[{"id": "manage"}], - status_code=200 + status_code=200, ) assert user_utils.get_mas_applications_in_workspace() == [{"id": "manage"}] assert get.call_count == 1 @@ -1901,7 +2001,7 @@ def test_get_mas_applications_in_workspace_error(user_utils, requests_mock): f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications", request_headers={"x-access-token": TOKEN}, json={"error": "internal"}, - status_code=500 + status_code=500, ) with pytest.raises(Exception) as excinfo: user_utils.get_mas_applications_in_workspace() @@ -1915,7 +2015,7 @@ def test_get_mas_application_availability(user_utils, requests_mock): f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications/{application_id}", request_headers={"x-access-token": TOKEN}, json={"id": "manage"}, - status_code=200 + status_code=200, ) assert user_utils.get_mas_application_availability(application_id) == {"id": "manage"} assert get.call_count == 1 @@ -1927,7 +2027,7 @@ def test_get_mas_application_availability_error(user_utils, requests_mock): f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications/{application_id}", request_headers={"x-access-token": TOKEN}, json={"error": "internal"}, - status_code=500 + status_code=500, ) with pytest.raises(Exception) as excinfo: user_utils.get_mas_application_availability(application_id) @@ -1985,7 +2085,7 @@ def json_callback(request, context): f"{MAS_API_URL}/workspaces/{MAS_WORKSPACE_ID}/applications/{application_id}", request_headers={"x-access-token": TOKEN}, json=json_callback, - status_code=200 + status_code=200, ) user_utils.await_mas_application_availability(application_id, timeout_secs=5, retry_interval_secs=0) @@ -2002,7 +2102,7 @@ def test_await_mas_application_availability_timeout(user_utils, requests_mock): "available": False, "ready": False, }, - status_code=200 + status_code=200, ) with pytest.raises(Exception) as excinfo: @@ -2018,7 +2118,7 @@ def test_parse_initial_users_from_aws_secret_json(user_utils): "user1@example.com": "primary,joe,bloggs", "user2@example.com": " primary , ben , bob ", "user3@example.com": "secondary ,bill, bibb", - "user4@example.com": "primary ,bab, bub,user4" + "user4@example.com": "primary ,bab, bub,user4", } ) @@ -2042,7 +2142,7 @@ def test_parse_initial_users_from_aws_secret_json(user_utils): "given_name": "bab", "family_name": "bub", "id": "user4", - } + }, ], "secondary": [ { @@ -2051,22 +2151,18 @@ def test_parse_initial_users_from_aws_secret_json(user_utils): "family_name": "bibb", "id": "user3@example.com", } - ] + ], } } assert actual_initial_users == expected_initial_users with pytest.raises(Exception) as excinfo: - user_utils.parse_initial_users_from_aws_secret_json({ - "user1@example.com": "primary" - }) + user_utils.parse_initial_users_from_aws_secret_json({"user1@example.com": "primary"}) assert "Wrong number of CSV values for user1@example.com (expected 3 or 4 but got 1)" == str(excinfo.value) with pytest.raises(Exception) as excinfo: - user_utils.parse_initial_users_from_aws_secret_json({ - "user1@example.com": "unknown,x,y" - }) + user_utils.parse_initial_users_from_aws_secret_json({"user1@example.com": "unknown,x,y"}) assert "Unknown user type for user1@example.com: unknown" == str(excinfo.value) @@ -2090,72 +2186,104 @@ def test_create_initial_user_for_saas_no_family_name(user_utils): def test_create_initial_user_for_saas_unsupported_type(user_utils): with pytest.raises(Exception) as excinfo: - user_utils.create_initial_user_for_saas({"given_name": "asdasd", "family_name": "sdfzsd", "email": "asdasd"}, "whoknows") + user_utils.create_initial_user_for_saas( + {"given_name": "asdasd", "family_name": "sdfzsd", "email": "asdasd"}, + "whoknows", + ) assert str(excinfo.value) == "Unsupported user_type: whoknows" + # Assisted by watsonx Code Assistant -@pytest.mark.parametrize("user_type, user_id, user_email, is_workspace_admin, application_role, manage_role, facilities_role, manage_security_groups_90, manage_security_groups_91", [ - ( - "PRIMARY", - None, - "bill.bob@acme.com", - True, - "ADMIN", - "MANAGEUSER", - "PREMIUM", - ["MAXADMIN"], - ["USERMANAGEMENT"] - ), - ( - "PRIMARY", - "billbob", - "bill.bob@acme.com", - True, - "ADMIN", - "MANAGEUSER", - "PREMIUM", - ["MAXADMIN"], - ["USERMANAGEMENT"] - ), - ( - "SECONDARY", - None, - "bab.bon@acme.com", - False, - "USER", - "MANAGEUSER", - "BASE", - [], - [] - ), - ( - "SECONDARY", - "babbon", - "bab.bon@acme.com", - False, - "USER", - "MANAGEUSER", - "BASE", - [], - [] - ) -]) +@pytest.mark.parametrize( + "user_type, user_id, user_email, is_workspace_admin, application_role, manage_role, facilities_role, manage_security_groups_90, manage_security_groups_91", + [ + ( + "PRIMARY", + None, + "bill.bob@acme.com", + True, + "ADMIN", + "MANAGEUSER", + "PREMIUM", + ["MAXADMIN"], + ["USERMANAGEMENT"], + ), + ( + "PRIMARY", + "billbob", + "bill.bob@acme.com", + True, + "ADMIN", + "MANAGEUSER", + "PREMIUM", + ["MAXADMIN"], + ["USERMANAGEMENT"], + ), + ( + "SECONDARY", + None, + "bab.bon@acme.com", + False, + "USER", + "MANAGEUSER", + "BASE", + [], + [], + ), + ( + "SECONDARY", + "babbon", + "bab.bon@acme.com", + False, + "USER", + "MANAGEUSER", + "BASE", + [], + [], + ), + ], +) def test_create_initial_user_for_saas( - user_type, user_id, user_email, is_workspace_admin, application_role, manage_role, facilities_role, manage_security_groups_90, manage_security_groups_91, - user_utils, requests_mock + user_type, + user_id, + user_email, + is_workspace_admin, + application_role, + manage_role, + facilities_role, + manage_security_groups_90, + manage_security_groups_91, + user_utils, + requests_mock, ): # Determine expected values based on MAS version mas_version = user_utils.mas_version - if mas_version == '9.0': + if mas_version == "9.0": manage_security_groups = manage_security_groups_90 if user_type == "PRIMARY": - permissions = {"systemAdmin": False, "userAdmin": True, "apikeyAdmin": False} - entitlement = {"application": "PREMIUM", "admin": "ADMIN_BASE", "alwaysReserveLicense": True} + permissions = { + "systemAdmin": False, + "userAdmin": True, + "apikeyAdmin": False, + } + entitlement = { + "application": "PREMIUM", + "admin": "ADMIN_BASE", + "alwaysReserveLicense": True, + } else: # SECONDARY - permissions = {"systemAdmin": False, "userAdmin": False, "apikeyAdmin": False} - entitlement = {"application": "BASE", "admin": "NONE", "alwaysReserveLicense": True} + permissions = { + "systemAdmin": False, + "userAdmin": False, + "apikeyAdmin": False, + } + entitlement = { + "application": "BASE", + "admin": "NONE", + "alwaysReserveLicense": True, + } else: # 9.1 manage_security_groups = manage_security_groups_91 permissions = None # Not used in 9.1 @@ -2163,16 +2291,18 @@ def test_create_initial_user_for_saas( # Mock get_or_create_user to return appropriate response based on version # Note: user_id might be None at this point, it gets set to user_email later actual_user_id = user_id if user_id is not None else user_email - if Version(mas_version) >= Version('9.1'): + if Version(mas_version) >= Version("9.1"): # For 9.1, return tuple (resource_id, user_data) with member array containing href resource_id = f"_{actual_user_id.replace('@', '_').replace('.', '_')}_resource_id" - user_utils.get_or_create_user = MagicMock(return_value=( - resource_id, - { - "member": [{"href": f"api/os/masperuser/{resource_id}"}], - "id": actual_user_id - } - )) + user_utils.get_or_create_user = MagicMock( + return_value=( + resource_id, + { + "member": [{"href": f"api/os/masperuser/{resource_id}"}], + "id": actual_user_id, + }, + ) + ) else: # For version < 9.1, return tuple (None, user_data) user_utils.get_or_create_user = MagicMock(return_value=(None, {"id": actual_user_id})) @@ -2195,7 +2325,7 @@ def test_create_initial_user_for_saas( initial_users = { "email": user_email, "given_name": user_given_name, - "family_name": user_family_name + "family_name": user_family_name, } if user_id is None: @@ -2206,26 +2336,20 @@ def test_create_initial_user_for_saas( username = user_id # For version 9.1 PRIMARY users, pass groupreassign parameter - if Version(mas_version) >= Version('9.1') and user_type == "PRIMARY": + if Version(mas_version) >= Version("9.1") and user_type == "PRIMARY": groupreassign = [{"groupname": "USERMANAGEMENT"}] user_utils.create_initial_user_for_saas(initial_users, user_type, groupreassign) else: user_utils.create_initial_user_for_saas(initial_users, user_type) # Build expected user_def based on version - if mas_version == '9.0': + if mas_version == "9.0": expected_user_def = { "id": user_id, "status": {"active": True}, "username": username, "owner": "local", - "emails": [ - { - "value": user_email, - "type": "Work", - "primary": True - } - ], + "emails": [{"value": user_email, "type": "Work", "primary": True}], "phoneNumbers": [], "addresses": [], "displayName": display_name, @@ -2233,7 +2357,7 @@ def test_create_initial_user_for_saas( "permissions": permissions, "entitlement": entitlement, "givenName": user_given_name, - "familyName": user_family_name + "familyName": user_family_name, } else: # >=9.1 if user_type == "PRIMARY": @@ -2247,11 +2371,7 @@ def test_create_initial_user_for_saas( "isauthorized": 1, "idpadmin": True, "status": "ACTIVE", - "groupuser": [ - { - "groupname": "USERMANAGEMENT" - } - ] + "groupuser": [{"groupname": "USERMANAGEMENT"}], } else: # SECONDARY maxuser_def = { @@ -2263,7 +2383,7 @@ def test_create_initial_user_for_saas( "apikeyadmin": False, "isauthorized": 0, "idpadmin": False, - "status": "ACTIVE" + "status": "ACTIVE", } expected_user_def = { @@ -2273,46 +2393,49 @@ def test_create_initial_user_for_saas( "primaryphone": "", "addressline1": "", "displayName": display_name, - "maxuser": maxuser_def + "maxuser": maxuser_def, } user_utils.get_or_create_user.assert_called_once_with(expected_user_def) # Check link_user_to_local_idp call based on version - if Version(mas_version) >= Version('9.1'): + if Version(mas_version) >= Version("9.1"): resource_id = f"_{actual_user_id.replace('@', '_').replace('.', '_')}_resource_id" - user_utils.link_user_to_local_idp.assert_called_once_with(user_id, email_password=True, manage_api_key=manage_api_key, resource_id=resource_id) + user_utils.link_user_to_local_idp.assert_called_once_with( + user_id, + email_password=True, + manage_api_key=manage_api_key, + resource_id=resource_id, + ) else: user_utils.link_user_to_local_idp.assert_called_once_with(user_id, email_password=True) user_utils.add_user_to_workspace.assert_called_once_with(user_id, is_workspace_admin=is_workspace_admin) # For version < 9.1, await_mas_application_availability and set_user_application_permission are called # For version >= 9.1, they are NOT called - if mas_version == '9.0': + if mas_version == "9.0": user_utils.await_mas_application_availability.assert_has_calls([call("manage"), call("iot")]) - user_utils.set_user_application_permission.assert_has_calls([ - call(user_id, "manage", manage_role), - call(user_id, "iot", application_role), - call(user_id, "facilities", facilities_role), - ]) + user_utils.set_user_application_permission.assert_has_calls( + [ + call(user_id, "manage", manage_role), + call(user_id, "iot", application_role), + call(user_id, "facilities", facilities_role), + ] + ) else: # >=9.1 user_utils.await_mas_application_availability.assert_not_called() user_utils.set_user_application_permission.assert_not_called() # check_user_sync is only called for version < 9.1 # For version >= 9.1, Manage API doesn't return applications field, so sync check is not performed - if mas_version == '9.0': - user_utils.check_user_sync.assert_has_calls([ - call(user_id, "manage"), - call(user_id, "iot"), - call(user_id, "facilities") - ]) + if mas_version == "9.0": + user_utils.check_user_sync.assert_has_calls([call(user_id, "manage"), call(user_id, "iot"), call(user_id, "facilities")]) else: # 9.1 user_utils.check_user_sync.assert_not_called() # For version >= 9.1, API key is always created (needed for link_user_to_local_idp) # For version < 9.1, API key is only created if there are manage_security_groups - if Version(mas_version) >= Version('9.1') or len(manage_security_groups) > 0: + if Version(mas_version) >= Version("9.1") or len(manage_security_groups) > 0: user_utils.create_or_get_manage_api_key_for_user.assert_called_once_with("MXINTADM", temporary=True) else: user_utils.create_or_get_manage_api_key_for_user.assert_not_called() @@ -2320,9 +2443,14 @@ def test_create_initial_user_for_saas( if len(manage_security_groups) > 0: # For version < 9.1, add_user_to_manage_group is called # For version >= 9.1, set_user_group_reassignment_auth is called for PRIMARY users - if mas_version == '9.0': + if mas_version == "9.0": user_utils.add_user_to_manage_group.assert_has_calls( - list(map(lambda sg: call(user_id, sg, manage_api_key), manage_security_groups)) + list( + map( + lambda sg: call(user_id, sg, manage_api_key), + manage_security_groups, + ) + ) ) user_utils.set_user_group_reassignment_auth.assert_not_called() else: # >=9.1 @@ -2331,7 +2459,12 @@ def test_create_initial_user_for_saas( # For versions >= 9.1, both user_id and resource_id are passed actual_user_id = user_id if user_id is not None else user_email resource_id = f"_{actual_user_id.replace('@', '_').replace('.', '_')}_resource_id" - user_utils.set_user_group_reassignment_auth.assert_called_once_with(actual_user_id, resource_id, [{"groupname": "USERMANAGEMENT"}], manage_api_key) + user_utils.set_user_group_reassignment_auth.assert_called_once_with( + actual_user_id, + resource_id, + [{"groupname": "USERMANAGEMENT"}], + manage_api_key, + ) else: user_utils.set_user_group_reassignment_auth.assert_not_called() @@ -2373,20 +2506,13 @@ def test_create_initial_users_for_saas(user_utils): def fail_for_users_b_and_e(user, user_type, groupreassign=None): if user["email"] in ["b", "e"]: raise Exception(f"{user['email']} should fail") + user_utils.create_initial_user_for_saas.side_effect = fail_for_users_b_and_e initial_users = { "users": { - "primary": [ - {"email": "a"}, - {"email": "b"}, - {"email": "c"} - ], - "secondary": [ - {"email": "d"}, - {"email": "e"}, - {"email": "f"} - ] + "primary": [{"email": "a"}, {"email": "b"}, {"email": "c"}], + "secondary": [{"email": "d"}, {"email": "e"}, {"email": "f"}], } } @@ -2400,7 +2526,7 @@ def fail_for_users_b_and_e(user, user_type, groupreassign=None): "failed": [ {"email": "b"}, {"email": "e"}, - ] + ], } user_utils.await_mas_application_availability.assert_has_calls([call("manage"), call("iot")]) diff --git a/test/src/test_utils.py b/test/src/test_utils.py index d4ca5bdc..ab31499d 100644 --- a/test/src/test_utils.py +++ b/test/src/test_utils.py @@ -12,14 +12,14 @@ def test_version_before(): - assert utils.isVersionBefore('9.1.0', '9.1.x-feature') is False - assert utils.isVersionBefore('9.1.0', '9.0.0') is True - assert utils.isVersionBefore('8.11.1', '9.1.0') is False - assert utils.isVersionBefore('9.1.0', '9.1.x-stable') is False + assert utils.isVersionBefore("9.1.0", "9.1.x-feature") is False + assert utils.isVersionBefore("9.1.0", "9.0.0") is True + assert utils.isVersionBefore("8.11.1", "9.1.0") is False + assert utils.isVersionBefore("9.1.0", "9.1.x-stable") is False def test_version_equal_of_after(): - assert utils.isVersionEqualOrAfter('9.1.0', '9.2.x-feature') is True - assert utils.isVersionEqualOrAfter('9.1.0', '9.0.0') is False - assert utils.isVersionEqualOrAfter('8.11.1', '9.1.0') is True - assert utils.isVersionEqualOrAfter('9.2.0', '9.1.x-stable') is False + assert utils.isVersionEqualOrAfter("9.1.0", "9.2.x-feature") is True + assert utils.isVersionEqualOrAfter("9.1.0", "9.0.0") is False + assert utils.isVersionEqualOrAfter("8.11.1", "9.1.0") is True + assert utils.isVersionEqualOrAfter("9.2.0", "9.1.x-stable") is False