From 62464afb83ff28b9da1bcf3829940d6e53dd5df8 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 16:04:42 +0200 Subject: [PATCH] devops_terraform_build now might work --- .../application/terraform_service.py | 160 +++++++++++++++++- .../ddadevops/devops_terraform_build.py | 151 ++--------------- src/main/python/ddadevops/domain/__init__.py | 2 +- .../python/ddadevops/domain/devops_factory.py | 4 +- src/main/python/ddadevops/domain/terraform.py | 2 +- .../infrastructure/infrastructure.py | 9 + src/test/python/domain/test_terraform.py | 12 +- 7 files changed, 189 insertions(+), 151 deletions(-) diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index f3ba585..a0986ab 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -1,22 +1,26 @@ from pathlib import Path -from ..domain import Devops, BuildType -from ..infrastructure import FileApi, ResourceApi, ImageApi +from dda_python_terraform import Terraform, IsFlagged +from packaging import version +from ..domain import Devops, BuildType, TerraformDomain +from ..infrastructure import FileApi, ResourceApi, TerraformApi + +# TODO: mv more fkt to Terraform_api ? class TerraformService: def __init__( - self, file_api: FileApi, resource_api: ResourceApi, image_api: ImageApi + self, file_api: FileApi, resource_api: ResourceApi, terraform_api: TerraformApi ): self.file_api = file_api self.resource_api = resource_api - self.image_api = image_api + self.terraform_api = terraform_api @classmethod def prod(cls): return cls( FileApi(), ResourceApi(), - ImageApi(), + TerraformApi(), ) def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): @@ -39,6 +43,12 @@ class TerraformService: f"{terraform.build_commons_path()}/*", devops.build_path() ) + def __print_terraform_command__(self, terraform: Terraform, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if terraform_domain.tf_debug_print_terraform_command: + output = f"cd {devops.build_path()} && {terraform.latest_cmd()}" + print(output) + def copy_local_state(self, devops: Devops): # TODO: orignal was unchecked ... self.file_api.cp("terraform.tfstate", devops.build_path()) @@ -60,3 +70,143 @@ class TerraformService: self.file_api.cp("*.tfvars", devops.build_path()) self.file_api.cp_recursive("scripts", devops.build_path()) + def init_client(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + terraform = Terraform( + working_dir=devops.build_path(), + terraform_semantic_version=terraform_domain.tf_terraform_semantic_version, + ) + terraform.init() + self.__print_terraform_command__(terraform, devops) + if terraform_domain.tf_use_workspace: + try: + terraform.workspace("select", self.stage) + self.__print_terraform_command__(terraform, devops) + except: + terraform.workspace("new", self.stage) + self.__print_terraform_command__(terraform, devops) + return terraform + + def write_output(self, terraform, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + result = terraform.output(json=IsFlagged) + self.__print_terraform_command__(terraform, devops) + self.file_api.write_json_to_file( + Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}"), result + ) + + def read_output(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + return self.file_api.read_json_fro_file( + Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}") + ) + + def plan(self, devops: Devops, fail_on_diff=False): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if fail_on_diff: + detailed_exitcode = IsFlagged + else: + detailed_exitcode = None + terraform = self.init_client(devops) + return_code, _, stderr = terraform.plan( + detailed_exitcode=detailed_exitcode, + capture_output=False, + raise_on_error=False, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform) + if return_code not in (0, 2): + raise RuntimeError(return_code, "terraform error:", stderr) + if return_code == 2: + raise RuntimeError(return_code, "diff in config found:", stderr) + + def apply(self, devops: Devops, auto_approve=False): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if auto_approve: + auto_approve_flag = IsFlagged + else: + auto_approve_flag = None + terraform = self.init_client(devops) + if version.parse( + terraform_domain.tf_terraform_semantic_version + ) >= version.parse("1.0.0"): + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + else: + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + skip_plan=auto_approve, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) + self.write_output(terraform, devops) + + def refresh(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + terraform = self.init_client(devops) + return_code, _, stderr = terraform.refresh( + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) + self.write_output(terraform, devops) + + def destroy(self, devops: Devops, auto_approve=False): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if auto_approve: + auto_approve_flag = IsFlagged + else: + auto_approve_flag = None + terraform = self.init_client(devops) + if version.parse( + terraform_domain.tf_terraform_semantic_version + ) >= version.parse("1.0.0"): + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + else: + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + force=auto_approve_flag, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) + + def tf_import( + self, + devops: Devops, + tf_import_name, + tf_import_resource, + ): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + return_code, _, stderr = terraform.import_cmd( + tf_import_name, + tf_import_resource, + capture_output=False, + raise_on_error=True, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.print_terraform_command(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 769823f..830c12f 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -1,11 +1,3 @@ -import sys -from os import chmod -from json import load, dumps -from subprocess import run -from packaging import version -from pkg_resources import resource_string -from dda_python_terraform import Terraform, IsFlagged -from .python_util import filter_none from .devops_build import DevopsBuild, create_devops_build_config @@ -67,153 +59,40 @@ class DevopsTerraformBuild(DevopsBuild): devops = self.devops_repo.get_devops(self.project) self.teraform_service.rescue_local_state(devops) - def init_client(self): - terraform = Terraform( - working_dir=self.build_path(), - terraform_semantic_version=self.terraform_semantic_version, - ) - terraform.init() - self.print_terraform_command(terraform) - if self.use_workspace: - try: - terraform.workspace("select", self.stage) - self.print_terraform_command(terraform) - except: - terraform.workspace("new", self.stage) - self.print_terraform_command(terraform) - return terraform - - def write_output(self, terraform): - result = terraform.output(json=IsFlagged) - self.print_terraform_command(terraform) - with open( - self.build_path() + self.output_json_name, "w", encoding="utf-8" - ) as output_file: - output_file.write(dumps(result)) - chmod(self.build_path() + self.output_json_name, 0o600) - - def read_output_json(self): - with open( - self.build_path() + self.output_json_name, "r", encoding="utf-8" - ) as file: - return load(file) + def read_output_json(self) -> map: + devops = self.devops_repo.get_devops(self.project) + return self.teraform_service.read_output(devops) def plan(self): - terraform = self.init_client() - return_code, _, stderr = terraform.plan( - detailed_exitcode=None, - capture_output=False, - raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.plan(devops) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def plan_fail_on_diff(self): - terraform = self.init_client() - return_code, _, stderr = terraform.plan( - detailed_exitcode=IsFlagged, - capture_output=False, - raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.plan(devops, fail_on_diff=True) self.post_build() - self.print_terraform_command(terraform) - if return_code not in (0, 2): - raise RuntimeError(return_code, "terraform error:", stderr) - if return_code == 2: - raise RuntimeError(return_code, "diff in config found:", stderr) def apply(self, auto_approve=False): - terraform = self.init_client() - if auto_approve: - auto_approve_flag = IsFlagged - else: - auto_approve_flag = None - if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.apply( - capture_output=False, - raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) - else: - return_code, _, stderr = terraform.apply( - capture_output=False, - raise_on_error=True, - skip_plan=auto_approve, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) - self.write_output(terraform) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.apply(devops, auto_approve=auto_approve) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def refresh(self): - terraform = self.init_client() - return_code, _, stderr = terraform.refresh( - var=self.project_vars(), var_file=self.additional_tfvar_files - ) - self.write_output(terraform) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.refresh(devops) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def destroy(self, auto_approve=False): - terraform = self.init_client() - if auto_approve: - auto_approve_flag = IsFlagged - else: - auto_approve_flag = None - if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.destroy( - capture_output=False, - raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) - else: - return_code, _, stderr = terraform.destroy( - capture_output=False, - raise_on_error=True, - force=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.refresh(devops) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def tf_import( self, tf_import_name, tf_import_resource, ): - terraform = self.init_client() - return_code, _, stderr = terraform.import_cmd( - tf_import_name, - tf_import_resource, - capture_output=False, - raise_on_error=True, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.tf_import(devops, tf_import_name, tf_import_resource) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) - - def print_terraform_command(self, terraform): - if self.debug_print_terraform_command: - output = "cd " + self.build_path() + " && " + terraform.latest_cmd() - print(output) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 967c0f2..c45278d 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -2,7 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k -from .terraform import Terraform +from .terraform import TerraformDomain from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index d045b50..f43e69e 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -3,7 +3,7 @@ from .common import Validateable, Devops, BuildType, MixinType from .image import Image from .c4k import C4k from .provs_k3s import K3s -from .terraform import Terraform +from .terraform import TerraformDomain from .release import Release from .version import Version @@ -24,7 +24,7 @@ class DevopsFactory: if BuildType.K3S in build_types: specialized_builds[BuildType.K3S] = K3s(inp) if BuildType.K3S in build_types: - specialized_builds[BuildType.TERRAFORM] = Terraform(inp) + specialized_builds[BuildType.TERRAFORM] = TerraformDomain(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index acd1292..e0a6e38 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -8,7 +8,7 @@ from .common import ( ) -class Terraform(Validateable): +class TerraformDomain(Validateable): def __init__(self, inp: dict): self.module = inp.get("module") self.stage = inp.get("stage") diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 6cef62f..82ba879 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -3,6 +3,7 @@ from pathlib import Path from sys import stdout from os import chmod, environ from pkg_resources import resource_string +from json import load, dumps import yaml @@ -37,6 +38,14 @@ class FileApi: yaml.dump(data, output_file) chmod(path, 0o600) + def write_json_to_file(self, path: Path, data: map): + with open(path, "w", encoding="utf-8") as output_file: + output_file.write(dumps(data)) + chmod(path, 0o600) + + def read_json_fro_file(self, path: Path) -> map: + with open(path, "r", encoding="utf-8") as input_file: + return load(input_file) class ImageApi: def image(self, name: str, path: Path): diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index b042bf0..720e9ff 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -1,6 +1,6 @@ import pytest from pathlib import Path -from src.main.python.ddadevops.domain import DnsRecord, BuildType, Terraform +from src.main.python.ddadevops.domain import DnsRecord, BuildType, TerraformDomain from .helper import build_devops, devops_config @@ -12,12 +12,12 @@ def test_creation(): def test_should_calculate_output_json_name(): config = devops_config({}) - sut = Terraform(config) + sut = TerraformDomain(config) assert "the_out.json" == sut.output_json_name() config = devops_config({}) del config["tf_output_json_name"] - sut = Terraform(config) + sut = TerraformDomain(config) assert "out_module.json" == sut.output_json_name() @@ -25,14 +25,14 @@ def test_should_calculate_terraform_build_commons_path(): config = devops_config({}) del config["tf_build_commons_path"] del config["tf_build_commons_dir_name"] - sut = Terraform(config) + sut = TerraformDomain(config) assert Path("terraform") == sut.terraform_build_commons_path() config = devops_config({}) - sut = Terraform(config) + sut = TerraformDomain(config) assert Path("build_commons_path/terraform") == sut.terraform_build_commons_path() def test_should_calculate_project_vars(): config = devops_config({}) - sut = Terraform(config) + sut = TerraformDomain(config) assert {'module': 'module', 'stage': 'test'} == sut.project_vars()