From 15335ae73fb4add3d5c60043f571cb0bf0d4c646 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 08:34:43 +0200 Subject: [PATCH] introduce terraform --- doc/architecture/Domain.md | 13 + .../ddadevops/devops_terraform_build.py | 226 ++++++++++-------- src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 1 + .../python/ddadevops/domain/devops_factory.py | 3 + src/main/python/ddadevops/domain/terraform.py | 36 +++ src/test/python/domain/helper.py | 12 +- src/test/python/domain/test_terraform.py | 19 ++ 8 files changed, 211 insertions(+), 100 deletions(-) create mode 100644 src/main/python/ddadevops/domain/terraform.py create mode 100644 src/test/python/domain/test_terraform.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index fa18bc7..8b36d39 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -37,6 +37,18 @@ classDiagram k3s_app_filename_to_provision } + class Terraform { + tf_additional_vars + tf_output_json_name + tf_use_workspace + tf_use_package_common_files + tf_build_commons_path + tf_commons_dir_name + tf_debug_print_terraform_command + tf_additional_tfvar_files + tf_terraform_semantic_version + } + class DnsRecord { fqdn ipv4 @@ -82,6 +94,7 @@ classDiagram Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" ProvsK3s: spcialized_builds + Devops *-- "0..1" Terraform: spcialized_builds Devops *-- "0..1" Release: mixins Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 37cbc0d..a8c024c 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -9,94 +9,87 @@ from .python_util import filter_none from .devops_build import DevopsBuild, create_devops_build_config - - -def create_devops_terraform_build_config(stage, - project_root_path, - module, - additional_vars, - build_dir_name='target', - output_json_name=None, - use_workspace=True, - use_package_common_files=True, - build_commons_path=None, - terraform_build_commons_dir_name='terraform', - debug_print_terraform_command=False, - additional_tfvar_files=None, - terraform_semantic_version="1.0.8"): +def create_devops_terraform_build_config( + stage, + project_root_path, + module, + additional_vars, + build_dir_name="target", + output_json_name=None, + use_workspace=True, + use_package_common_files=True, + build_commons_path=None, + terraform_build_commons_dir_name="terraform", + debug_print_terraform_command=False, + additional_tfvar_files=None, + terraform_semantic_version="1.0.8", +): if not output_json_name: - output_json_name = 'out_' + module + '.json' + output_json_name = "out_" + module + ".json" if not additional_tfvar_files: additional_tfvar_files = [] - ret = create_devops_build_config( - stage, project_root_path, module, build_dir_name) - ret.update({'additional_vars': additional_vars, - 'output_json_name': output_json_name, - 'use_workspace': use_workspace, - 'use_package_common_files': use_package_common_files, - 'build_commons_path': build_commons_path, - 'terraform_build_commons_dir_name': terraform_build_commons_dir_name, - 'debug_print_terraform_command': debug_print_terraform_command, - 'additional_tfvar_files': additional_tfvar_files, - 'terraform_semantic_version': terraform_semantic_version}) + ret = create_devops_build_config(stage, project_root_path, module, build_dir_name) + ret.update( + { + "additional_vars": additional_vars, + "output_json_name": output_json_name, + "use_workspace": use_workspace, + "use_package_common_files": use_package_common_files, + "build_commons_path": build_commons_path, + "terraform_build_commons_dir_name": terraform_build_commons_dir_name, + "debug_print_terraform_command": debug_print_terraform_command, + "additional_tfvar_files": additional_tfvar_files, + "terraform_semantic_version": terraform_semantic_version, + } + ) return ret -class DevopsTerraformBuild(DevopsBuild): +class DevopsTerraformBuild(DevopsBuild): def __init__(self, project, config): inp = config.copy() - inp["name"]=project.name - inp["module"]=config.get("module") - inp["stage"]=config.get("stage") - inp["project_root_path"]=config.get("project_root_path") - inp["build_types"]=config.get("build_types", []) - inp["mixin_types"]=config.get("mixin_types", []) + inp["name"] = project.name + inp["module"] = config.get("module") + inp["stage"] = config.get("stage") + inp["project_root_path"] = config.get("project_root_path") + inp["build_types"] = config.get("build_types", []) + inp["mixin_types"] = config.get("mixin_types", []) super().__init__(project, inp) - project.build_depends_on('dda-python-terraform') - self.additional_vars = config['additional_vars'] - self.output_json_name = config['output_json_name'] - self.use_workspace = config['use_workspace'] - self.use_package_common_files = config['use_package_common_files'] - self.build_commons_path = config['build_commons_path'] - self.terraform_build_commons_dir_name = config['terraform_build_commons_dir_name'] - self.debug_print_terraform_command = config['debug_print_terraform_command'] - self.additional_tfvar_files = config['additional_tfvar_files'] - self.terraform_semantic_version = config['terraform_semantic_version'] - self.stage = config["stage"] - self.module = config["module"] + project.build_depends_on("dda-python-terraform") def terraform_build_commons_path(self): - mylist = [self.build_commons_path, - self.terraform_build_commons_dir_name] - return '/'.join(filter_none(mylist)) + '/' + mylist = [self.build_commons_path, self.terraform_build_commons_dir_name] + return "/".join(filter_none(mylist)) + "/" def project_vars(self): - ret = {'stage': self.stage} + ret = {"stage": self.stage} if self.module: - ret['module'] = self.module + ret["module"] = self.module if self.additional_vars: ret.update(self.additional_vars) return ret def copy_build_resource_file_from_package(self, name): - my_data = resource_string( - __name__, "src/main/resources/terraform/" + name) - with open(self.build_path() + '/' + name, "w", encoding="utf-8") as output_file: + my_data = resource_string(__name__, "src/main/resources/terraform/" + name) + with open(self.build_path() + "/" + name, "w", encoding="utf-8") as output_file: output_file.write(my_data.decode(sys.stdout.encoding)) def copy_build_resources_from_package(self): - self.copy_build_resource_file_from_package('versions.tf') - self.copy_build_resource_file_from_package('terraform_build_vars.tf') + self.copy_build_resource_file_from_package("versions.tf") + self.copy_build_resource_file_from_package("terraform_build_vars.tf") def copy_build_resources_from_dir(self): - run('cp -f ' + self.terraform_build_commons_path() + - '* ' + self.build_path(), shell=True, check=False) + run( + "cp -f " + self.terraform_build_commons_path() + "* " + self.build_path(), + shell=True, + check=False, + ) def copy_local_state(self): - run('cp terraform.tfstate ' + self.build_path(), shell=True, check=False) + run("cp terraform.tfstate " + self.build_path(), shell=True, check=False) def rescue_local_state(self): - run('cp ' + self.build_path() + '/terraform.tfstate .', shell=True, check=False) + run("cp " + self.build_path() + "/terraform.tfstate .", shell=True, check=False) def initialize_build_dir(self): super().initialize_build_dir() @@ -105,43 +98,54 @@ class DevopsTerraformBuild(DevopsBuild): else: self.copy_build_resources_from_dir() self.copy_local_state() - run('cp *.tf ' + self.build_path(), shell=True, check=False) - run('cp *.properties ' + self.build_path(), shell=True, check=False) - run('cp *.tfvars ' + self.build_path(), shell=True, check=False) - run('cp -r scripts ' + self.build_path(), shell=True, check=False) + run("cp *.tf " + self.build_path(), shell=True, check=False) + run("cp *.properties " + self.build_path(), shell=True, check=False) + run("cp *.tfvars " + self.build_path(), shell=True, check=False) + run("cp -r scripts " + self.build_path(), shell=True, check=False) def post_build(self): self.rescue_local_state() def init_client(self): - terraform = Terraform(working_dir=self.build_path(), terraform_semantic_version=self.terraform_semantic_version) + terraform = Terraform( + working_dir=self.build_path(), + terraform_semantic_version=self.terraform_semantic_version, + ) terraform.init() self.print_terraform_command(terraform) if self.use_workspace: try: - terraform.workspace('select', self.stage) + terraform.workspace("select", self.stage) self.print_terraform_command(terraform) except: - terraform.workspace('new', self.stage) + terraform.workspace("new", self.stage) self.print_terraform_command(terraform) return terraform def write_output(self, terraform): result = terraform.output(json=IsFlagged) self.print_terraform_command(terraform) - with open(self.build_path() + self.output_json_name, "w", encoding="utf-8") as output_file: + with open( + self.build_path() + self.output_json_name, "w", encoding="utf-8" + ) as output_file: output_file.write(dumps(result)) chmod(self.build_path() + self.output_json_name, 0o600) def read_output_json(self): - with open(self.build_path() + self.output_json_name, 'r', encoding="utf-8") as file: + with open( + self.build_path() + self.output_json_name, "r", encoding="utf-8" + ) as file: return load(file) def plan(self): terraform = self.init_client() - return_code, _, stderr = terraform.plan(detailed_exitcode=None, capture_output=False, raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.plan( + detailed_exitcode=None, + capture_output=False, + raise_on_error=False, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code > 0: @@ -149,9 +153,13 @@ class DevopsTerraformBuild(DevopsBuild): def plan_fail_on_diff(self): terraform = self.init_client() - return_code, _, stderr = terraform.plan(detailed_exitcode=IsFlagged, capture_output=False, raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.plan( + detailed_exitcode=IsFlagged, + capture_output=False, + raise_on_error=False, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code not in (0, 2): @@ -166,15 +174,21 @@ class DevopsTerraformBuild(DevopsBuild): else: auto_approve_flag = None if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.apply(capture_output=False, raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) else: - return_code, _, stderr = terraform.apply(capture_output=False, raise_on_error=True, - skip_plan=auto_approve, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + skip_plan=auto_approve, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.write_output(terraform) self.post_build() self.print_terraform_command(terraform) @@ -184,8 +198,8 @@ class DevopsTerraformBuild(DevopsBuild): def refresh(self): terraform = self.init_client() return_code, _, stderr = terraform.refresh( - var=self.project_vars(), - var_file=self.additional_tfvar_files) + var=self.project_vars(), var_file=self.additional_tfvar_files + ) self.write_output(terraform) self.post_build() self.print_terraform_command(terraform) @@ -199,26 +213,40 @@ class DevopsTerraformBuild(DevopsBuild): else: auto_approve_flag = None if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.destroy(capture_output=False, raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) else: - return_code, _, stderr = terraform.destroy(capture_output=False, raise_on_error=True, - force=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + force=auto_approve_flag, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code > 0: raise RuntimeError(return_code, "terraform error:", stderr) - def tf_import(self, tf_import_name, tf_import_resource,): + def tf_import( + self, + tf_import_name, + tf_import_resource, + ): terraform = self.init_client() - return_code, _, stderr = terraform.import_cmd(tf_import_name, tf_import_resource, - capture_output=False, raise_on_error=True, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.import_cmd( + tf_import_name, + tf_import_resource, + capture_output=False, + raise_on_error=True, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code > 0: @@ -226,5 +254,5 @@ class DevopsTerraformBuild(DevopsBuild): def print_terraform_command(self, terraform): if self.debug_print_terraform_command: - output = 'cd ' + self.build_path() + ' && ' + terraform.latest_cmd() + output = "cd " + self.build_path() + " && " + terraform.latest_cmd() print(output) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 022e38f..967c0f2 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -2,6 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k +from .terraform import Terraform from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 946c1c3..c270a49 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -10,6 +10,7 @@ class BuildType(Enum): IMAGE = 0 C4K = 1 K3S = 2 + TERRAFORM = 3 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 556348c..d045b50 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -3,6 +3,7 @@ from .common import Validateable, Devops, BuildType, MixinType from .image import Image from .c4k import C4k from .provs_k3s import K3s +from .terraform import Terraform from .release import Release from .version import Version @@ -22,6 +23,8 @@ class DevopsFactory: specialized_builds[BuildType.C4K] = C4k(inp) if BuildType.K3S in build_types: specialized_builds[BuildType.K3S] = K3s(inp) + if BuildType.K3S in build_types: + specialized_builds[BuildType.TERRAFORM] = Terraform(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py new file mode 100644 index 0000000..28eaaa8 --- /dev/null +++ b/src/main/python/ddadevops/domain/terraform.py @@ -0,0 +1,36 @@ +from typing import List, Optional +from .common import ( + Validateable, + DnsRecord, + Devops, +) + + +class Terraform(Validateable): + def __init__(self, inp: dict): + self.module = inp.get("module") + self.stage = inp.get("stage") + self.tf_additional_vars = inp.get("tf_additional_vars") + self.tf_output_json_name = inp.get("tf_output_json_name") + self.tf_build_commons_path = inp.get("tf_build_commons_path") + self.tf_additional_tfvar_files = inp.get("tf_additional_tfvar_files", []) + self.tf_use_workspace = inp.get("tf_use_workspace", True) + self.tf_debug_print_terraform_command = inp.get( + "tf_debug_print_terraform_command", False + ) + self.tf_commons_dir_name = inp.get("tf_commons_dir_name", "terraform") + self.tf_terraform_semantic_version = inp.get( + "tf_terraform_semantic_version", "1.0.8" + ) + self.tf_use_package_common_files = inp.get("tf_use_package_common_files", True) + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("module") + return result + + def output_json_name(self) -> str: + if self.tf_output_json_name: + return self.tf_output_json_name + else: + return f"out_{self.module}.json" diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 9386531..2b09198 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -9,7 +9,7 @@ def devops_config(overrides: dict) -> dict: "stage": "test", "project_root_path": "root_path", "build_dir_name": "target", - "build_types": ["IMAGE", "C4K", "K3S"], + "build_types": ["IMAGE", "C4K", "K3S", "TERRAFORM"], "mixin_types": ["RELEASE"], "image_dockerhub_user": "dockerhub_user", "image_dockerhub_password": "dockerhub_password", @@ -24,6 +24,16 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", + "tf_additional_vars": None, + "tf_output_json_name": "the_out.json", + "tf_use_workspace": None, + "tf_use_package_common_files": None, + "tf_build_commons_path": None, + "tf_commons_dir_name": None, + "tf_debug_print_terraform_command": None, + "tf_additional_tfvar_files": None, + "tf_terraform_semantic_version": None, + "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py new file mode 100644 index 0000000..013bc94 --- /dev/null +++ b/src/test/python/domain/test_terraform.py @@ -0,0 +1,19 @@ +import pytest +from pathlib import Path +from src.main.python.ddadevops.domain import DnsRecord, BuildType, Terraform +from .helper import build_devops + + +def test_creation(): + sut = build_devops({}) + assert BuildType.TERRAFORM in sut.specialized_builds + assert sut.specialized_builds[BuildType.TERRAFORM] + +def test_should_calculate_output_json_name(): + devops = build_devops({}) + sut = devops.specialized_builds[BuildType.TERRAFORM] + assert 'the_out.json' == sut.output_json_name() + + devops = build_devops({"tf_output_json_name": None,}) + sut = devops.specialized_builds[BuildType.TERRAFORM] + assert 'out_module.json' == sut.output_json_name()