From 0b577597e8e47f92b6a3a9050c5d69cbd13b08c7 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 16:56:02 +0200 Subject: [PATCH] introduce aws_provider --- build.py | 2 +- doc/architecture/Domain.md | 5 ++ src/main/python/ddadevops/__init__.py | 4 +- .../ddadevops/aws_backend_properties_mixin.py | 52 ------------ src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 1 + .../python/ddadevops/domain/provider_aws.py | 80 +++++++++++++++++++ .../ddadevops/domain/provider_digitalocean.py | 26 +++--- src/main/python/ddadevops/domain/terraform.py | 3 + .../terraform/aws_backend_properties_vars.tf | 4 +- .../terraform/do_backend_properties_vars.tf | 4 +- src/test/python/domain/helper.py | 6 +- src/test/python/domain/test_provider_aws.py | 77 ++++++++++++++++++ src/test/python/domain/test_terraform.py | 19 ++++- 14 files changed, 209 insertions(+), 75 deletions(-) delete mode 100644 src/main/python/ddadevops/aws_backend_properties_mixin.py create mode 100644 src/main/python/ddadevops/domain/provider_aws.py create mode 100644 src/test/python/domain/test_provider_aws.py diff --git a/build.py b/build.py index ebeaf85..3de1f42 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev59" +version = "4.0.0-dev61" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index d557f6c..778f1dd 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -65,6 +65,10 @@ classDiagram class Hetzner { hetzner_api_key } + + class Aws { + aws_account_name + } class DnsRecord { fqdn @@ -116,6 +120,7 @@ classDiagram Devops *-- "0..1" Release: mixins TerraformDomain *-- "0..1" Digitalocean: providers TerraformDomain *-- "0..1" Hetzner: providers + TerraformDomain *-- "0..1" Aws: providers Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index f348694..c58913a 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -4,10 +4,8 @@ terraform, dda-pallet, aws & hetzner-cloud. """ -from .python_util import execute from .provs_k3s_build import ProvsK3sBuild -from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config -from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config +#from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .c4k_build import C4kBuild from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config from .devops_image_build import DevopsImageBuild diff --git a/src/main/python/ddadevops/aws_backend_properties_mixin.py b/src/main/python/ddadevops/aws_backend_properties_mixin.py deleted file mode 100644 index ef6cebe..0000000 --- a/src/main/python/ddadevops/aws_backend_properties_mixin.py +++ /dev/null @@ -1,52 +0,0 @@ -from dda_python_terraform import Terraform -from .devops_terraform_build import DevopsTerraformBuild - - -def add_aws_backend_properties_mixin_config(config, account_name): - config.update({"AwsBackendPropertiesMixin": {"account_name": account_name}}) - return config - - -class AwsBackendPropertiesMixin(DevopsTerraformBuild): - def __init__(self, project, config): - super().__init__(project, config) - aws_mixin_config = config["AwsBackendPropertiesMixin"] - self.account_name = aws_mixin_config["account_name"] - self.backend_config = ( - "backend." + self.account_name + "." + self.stage + ".properties" - ) - self.additional_tfvar_files.append(self.backend_config) - - def project_vars(self): - ret = super().project_vars() - ret.update({"account_name": self.account_name}) - return ret - - def copy_build_resources_from_package(self): - super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package("provider_registry.tf") - self.copy_build_resource_file_from_package("aws_provider.tf") - self.copy_build_resource_file_from_package("aws_backend_properties_vars.tf") - self.copy_build_resource_file_from_package("aws_backend_with_properties.tf") - - def copy_local_state(self): - pass - - def rescue_local_state(self): - pass - - def init_client(self): - terraform = Terraform( - working_dir=self.build_path(), - terraform_semantic_version=self.terraform_semantic_version, - ) - terraform.init(backend_config=self.backend_config) - self.print_terraform_command(terraform) - if self.use_workspace: - try: - terraform.workspace("select", self.stage) - self.print_terraform_command(terraform) - except: - terraform.workspace("new", self.stage) - self.print_terraform_command(terraform) - return terraform diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index b281a61..fdb1784 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -5,6 +5,7 @@ from .c4k import C4k from .terraform import TerraformDomain from .provider_digitalocean import Digitalocean from .provider_hetzner import Hetzner +from .provider_aws import Aws from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 01dc29d..fc32311 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -16,6 +16,7 @@ class BuildType(Enum): class ProviderType(Enum): DIGITALOCEAN = 0 HETZNER = 1 + AWS = 2 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/provider_aws.py b/src/main/python/ddadevops/domain/provider_aws.py new file mode 100644 index 0000000..e0ef793 --- /dev/null +++ b/src/main/python/ddadevops/domain/provider_aws.py @@ -0,0 +1,80 @@ +from typing import List, Dict, Set, Any +from .common import Validateable, CredentialMappingDefault + + +class Aws(Validateable, CredentialMappingDefault): + def __init__( + self, + inp: dict, + ): + self.stage = inp.get("stage") + self.module = inp.get("module") + self.aws_bucket = inp.get("aws_bucket") + self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id") + self.aws_account_name = inp.get("aws_account_name", self.stage) + self.aws_bucket_key = inp.get("aws_bucket_key", self.module) + self.aws_as_backend = inp.get("aws_as_backend", False) + self.aws_region = inp.get("aws_region", "eu-central-1") + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("stage") + result += self.__validate_is_not_empty__("module") + result += self.__validate_is_not_empty__("aws_account_name") + result += self.__validate_is_not_empty__("aws_as_backend") + if self.aws_as_backend: + result += self.__validate_is_not_empty__("aws_bucket") + result += self.__validate_is_not_empty__("aws_bucket_key") + result += self.__validate_is_not_empty__("aws_bucket_kms_key_id") + result += self.__validate_is_not_empty__("aws_region") + return result + + def backend_config(self) -> Dict[str, Any]: + result = {} + if self.aws_as_backend: + result = { + "bucket": self.aws_bucket, + "key": self.__bucket_key__(), + "region": self.aws_region, + } + if self.aws_bucket_kms_key_id: + result["kms_key_id"] = self.aws_bucket_kms_key_id + return result + + def resources_from_package(self) -> Set[str]: + result = {"provider_registry.tf", "aws_provider.tf"} + if self.aws_as_backend: + result.update( + {"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"} + ) + return result + + def project_vars(self): + result = {} + if self.aws_as_backend: + result.update( + { + "account_name": self.aws_account_name, + "bucket": self.aws_bucket, + "key": self.__bucket_key__(), + "kms_key_id": self.aws_bucket_kms_key_id, + "region": self.aws_region, + } + ) + return result + + def is_local_state(self): + return not self.aws_as_backend + + def __bucket_key__(self): + result = "" + if self.aws_as_backend: + if self.aws_account_name and self.aws_bucket_key: + result = f"{self.aws_account_name}/{self.aws_bucket_key}" + else: + result = f"{self.stage}/{self.module}" + return result + + @classmethod + def get_mapping_default(cls) -> List[Dict[str, str]]: + return [] diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index 491d93b..e00ce4f 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -35,6 +35,19 @@ class Digitalocean(Validateable, CredentialMappingDefault): result += self.__validate_is_not_empty__("do_region") return result + def backend_config(self) -> Dict[str, Any]: + result = {} + if self.do_as_backend: + result = { + "access_key": self.do_spaces_access_id, + "secret_key": self.do_spaces_secret_key, + "endpoint": self.do_endpoint, + "bucket": self.do_bucket, + "key": self.__bucket_key__(), + "region": self.do_region, + } + return result + def resources_from_package(self) -> Set[str]: result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} if self.do_as_backend: @@ -61,19 +74,6 @@ class Digitalocean(Validateable, CredentialMappingDefault): ) return result - def backend_config(self) -> Dict[str, Any]: - result = {} - if self.do_as_backend: - result = { - "access_key": self.do_spaces_access_id, - "secret_key": self.do_spaces_secret_key, - "endpoint": self.do_endpoint, - "bucket": self.do_bucket, - "key": self.__bucket_key__(), - "region": self.do_region, - } - return result - def is_local_state(self): return not self.do_as_backend diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 8df506f..c50f8d4 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -7,6 +7,7 @@ from .common import ( ) from .provider_digitalocean import Digitalocean from .provider_hetzner import Hetzner +from .provider_aws import Aws class TerraformDomain(Validateable): @@ -39,6 +40,8 @@ class TerraformDomain(Validateable): self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp) if ProviderType.HETZNER in provider_types: self.providers[ProviderType.HETZNER] = Hetzner(inp) + if ProviderType.AWS in provider_types: + self.providers[ProviderType.AWS] = Aws(inp) def validate(self) -> List[str]: result = [] diff --git a/src/main/resources/terraform/aws_backend_properties_vars.tf b/src/main/resources/terraform/aws_backend_properties_vars.tf index d261cf7..08abc4b 100644 --- a/src/main/resources/terraform/aws_backend_properties_vars.tf +++ b/src/main/resources/terraform/aws_backend_properties_vars.tf @@ -1,5 +1,5 @@ +variable "account_name" {} variable "bucket" {} variable "key" {} variable "kms_key_id" {} -variable "region" {} -variable "account_name" {} \ No newline at end of file +variable "region" {} \ No newline at end of file diff --git a/src/main/resources/terraform/do_backend_properties_vars.tf b/src/main/resources/terraform/do_backend_properties_vars.tf index bb24162..8b722b2 100644 --- a/src/main/resources/terraform/do_backend_properties_vars.tf +++ b/src/main/resources/terraform/do_backend_properties_vars.tf @@ -1,5 +1,5 @@ +variable "account_name" {} variable "endpoint" {} variable "bucket" {} variable "key" {} -variable "region" {} -variable "account_name" {} \ No newline at end of file +variable "region" {} \ No newline at end of file diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index df1a4a1..53b48b5 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -24,7 +24,7 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", - "tf_provider_types": ["DIGITALOCEAN", "HETZNER"], + "tf_provider_types": ["DIGITALOCEAN", "HETZNER", "AWS"], "tf_additional_vars": [], "tf_output_json_name": "the_out.json", "tf_use_workspace": None, @@ -45,6 +45,10 @@ def devops_config(overrides: dict) -> dict: "do_bucket": "bucket", "do_region": "region", "hetzner_api_key": "hetzner_api_key", + "aws_as_backend": True, + "aws_bucket": "bucket", + "aws_region": "region", + "aws_bucket_kms_key_id": "aws_bucket_kms_key_id", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_provider_aws.py b/src/test/python/domain/test_provider_aws.py new file mode 100644 index 0000000..7334039 --- /dev/null +++ b/src/test/python/domain/test_provider_aws.py @@ -0,0 +1,77 @@ +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain import ( + BuildType, + Aws, +) +from .helper import devops_config + + +def test_aws_creation(): + sut = Aws( + { + "module": "module", + "stage": "test", + "aws_account_name": "aws_account_name", + } + ) + assert sut is not None + assert sut.is_valid() + + sut = Aws( + { + "module": "module", + "stage": "test", + "aws_as_backend": True, + "aws_bucket": "bucket", + "aws_bucket_kms_key_id": "aws_bucket_kms_key_id", + } + ) + assert sut is not None + assert sut.is_valid() + + +def test_should_calculate_backend_config(): + sut = Aws( + devops_config( + { + "module": "dns_aws", + "stage": "prod", + "aws_bucket": "meissa-configuration", + "aws_bucket_kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration", + "aws_region": "eu-central-1", + } + ) + ) + assert { + "bucket": "meissa-configuration", + "key": "prod/dns_aws", + "kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration", + "region": "eu-central-1", + } == sut.backend_config() + + +def test_should_calculate_project_vars(): + sut = Aws( + devops_config( + { + "aws_as_backend": False, + } + ) + ) + assert {} == sut.project_vars() + + sut = Aws( + devops_config( + { + "aws_as_backend": True, + } + ) + ) + assert { + "account_name": "test", + "bucket": "bucket", + "key": "test/module", + "kms_key_id": "aws_bucket_kms_key_id", + "region": "region", + } == sut.project_vars() diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 5633cd1..30ab62a 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -16,6 +16,7 @@ def test_creation(): assert sut assert sut.providers[ProviderType.DIGITALOCEAN] assert sut.providers[ProviderType.HETZNER] + assert sut.providers[ProviderType.AWS] def test_should_calculate_output_json_name(): @@ -42,6 +43,14 @@ def test_should_validate(): sut = TerraformDomain(config) assert not sut.is_valid() + config = devops_config( + { + "aws_account_name": "", + } + ) + sut = TerraformDomain(config) + assert not sut.is_valid() + def test_should_calculate_terraform_build_commons_path(): config = devops_config({}) @@ -64,7 +73,12 @@ def test_should_calculate_project_vars(): sut = TerraformDomain(config) assert {"module": "module", "stage": "test"} == sut.project_vars() - config = devops_config({"do_as_backend": False,}) + config = devops_config( + { + "do_as_backend": False, + "aws_as_backend": False, + } + ) sut = TerraformDomain(config) assert { "module": "module", @@ -144,6 +158,9 @@ def test_should_calculate_resources_from_package(): "provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf", + "aws_backend_with_properties.tf", + "aws_provider.tf", + "aws_backend_properties_vars.tf", "my.file", } == sut.resources_from_package()