introduce aws_provider

build.py

@@ -28,7 +28,7 @@ use_plugin("python.distutils")
 default_task = "publish"
 
 name = "ddadevops"
-version = "4.0.0-dev59"
+version = "4.0.0-dev61"
 summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
 description = __doc__
 authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]

doc/architecture/Domain.md

@@ -66,6 +66,10 @@ classDiagram
         hetzner_api_key
     }
 
+    class Aws {
+        aws_account_name
+    }
+    
     class DnsRecord {
         fqdn
         ipv4
@@ -116,6 +120,7 @@ classDiagram
     Devops *-- "0..1" Release: mixins
     TerraformDomain *-- "0..1" Digitalocean: providers
     TerraformDomain *-- "0..1" Hetzner: providers
+    TerraformDomain *-- "0..1" Aws: providers
     Release o-- "0..1" BuildFile: primary_build_file
     Release o-- "0..n" BuildFile: secondary_build_files
     BuildFile *-- "1" Version

src/main/python/ddadevops/__init__.py

@@ -4,10 +4,8 @@ terraform, dda-pallet, aws & hetzner-cloud.
 
 """
 
-from .python_util import execute
 from .provs_k3s_build import ProvsK3sBuild
-from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config
+#from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config
-from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config
 from .c4k_build import C4kBuild
 from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config
 from .devops_image_build import DevopsImageBuild

src/main/python/ddadevops/aws_backend_properties_mixin.py

@@ -1,52 +0,0 @@
-from dda_python_terraform import Terraform
-from .devops_terraform_build import DevopsTerraformBuild
-
-
-def add_aws_backend_properties_mixin_config(config, account_name):
-    config.update({"AwsBackendPropertiesMixin": {"account_name": account_name}})
-    return config
-
-
-class AwsBackendPropertiesMixin(DevopsTerraformBuild):
-    def __init__(self, project, config):
-        super().__init__(project, config)
-        aws_mixin_config = config["AwsBackendPropertiesMixin"]
-        self.account_name = aws_mixin_config["account_name"]
-        self.backend_config = (
-            "backend." + self.account_name + "." + self.stage + ".properties"
-        )
-        self.additional_tfvar_files.append(self.backend_config)
-
-    def project_vars(self):
-        ret = super().project_vars()
-        ret.update({"account_name": self.account_name})
-        return ret
-
-    def copy_build_resources_from_package(self):
-        super().copy_build_resources_from_package()
-        self.copy_build_resource_file_from_package("provider_registry.tf")
-        self.copy_build_resource_file_from_package("aws_provider.tf")
-        self.copy_build_resource_file_from_package("aws_backend_properties_vars.tf")
-        self.copy_build_resource_file_from_package("aws_backend_with_properties.tf")
-
-    def copy_local_state(self):
-        pass
-
-    def rescue_local_state(self):
-        pass
-
-    def init_client(self):
-        terraform = Terraform(
-            working_dir=self.build_path(),
-            terraform_semantic_version=self.terraform_semantic_version,
-        )
-        terraform.init(backend_config=self.backend_config)
-        self.print_terraform_command(terraform)
-        if self.use_workspace:
-            try:
-                terraform.workspace("select", self.stage)
-                self.print_terraform_command(terraform)
-            except:
-                terraform.workspace("new", self.stage)
-                self.print_terraform_command(terraform)
-        return terraform

src/main/python/ddadevops/domain/__init__.py

@@ -5,6 +5,7 @@ from .c4k import C4k
 from .terraform import TerraformDomain
 from .provider_digitalocean import Digitalocean
 from .provider_hetzner import Hetzner
+from .provider_aws import Aws
 from .provs_k3s import K3s
 from .release import Release
 from .credentials import Credentials, CredentialMapping, GopassType

src/main/python/ddadevops/domain/common.py

@@ -16,6 +16,7 @@ class BuildType(Enum):
 class ProviderType(Enum):
     DIGITALOCEAN = 0
     HETZNER = 1
+    AWS = 2
 
 
 class MixinType(Enum):

src/main/python/ddadevops/domain/provider_aws.py

@@ -0,0 +1,80 @@
+from typing import List, Dict, Set, Any
+from .common import Validateable, CredentialMappingDefault
+
+
+class Aws(Validateable, CredentialMappingDefault):
+    def __init__(
+        self,
+        inp: dict,
+    ):
+        self.stage = inp.get("stage")
+        self.module = inp.get("module")
+        self.aws_bucket = inp.get("aws_bucket")
+        self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id")
+        self.aws_account_name = inp.get("aws_account_name", self.stage)
+        self.aws_bucket_key = inp.get("aws_bucket_key", self.module)
+        self.aws_as_backend = inp.get("aws_as_backend", False)
+        self.aws_region = inp.get("aws_region", "eu-central-1")
+
+    def validate(self) -> List[str]:
+        result = []
+        result += self.__validate_is_not_empty__("stage")
+        result += self.__validate_is_not_empty__("module")
+        result += self.__validate_is_not_empty__("aws_account_name")
+        result += self.__validate_is_not_empty__("aws_as_backend")
+        if self.aws_as_backend:
+            result += self.__validate_is_not_empty__("aws_bucket")
+            result += self.__validate_is_not_empty__("aws_bucket_key")
+            result += self.__validate_is_not_empty__("aws_bucket_kms_key_id")
+            result += self.__validate_is_not_empty__("aws_region")
+        return result
+
+    def backend_config(self) -> Dict[str, Any]:
+        result = {}
+        if self.aws_as_backend:
+            result = {
+                "bucket": self.aws_bucket,
+                "key": self.__bucket_key__(),
+                "region": self.aws_region,
+            }
+            if self.aws_bucket_kms_key_id:
+                result["kms_key_id"] = self.aws_bucket_kms_key_id
+        return result
+
+    def resources_from_package(self) -> Set[str]:
+        result = {"provider_registry.tf", "aws_provider.tf"}
+        if self.aws_as_backend:
+            result.update(
+                {"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"}
+            )
+        return result
+
+    def project_vars(self):
+        result = {}
+        if self.aws_as_backend:
+            result.update(
+                {
+                    "account_name": self.aws_account_name,
+                    "bucket": self.aws_bucket,
+                    "key": self.__bucket_key__(),
+                    "kms_key_id": self.aws_bucket_kms_key_id,
+                    "region": self.aws_region,
+                }
+            )
+        return result
+
+    def is_local_state(self):
+        return not self.aws_as_backend
+
+    def __bucket_key__(self):
+        result = ""
+        if self.aws_as_backend:
+            if self.aws_account_name and self.aws_bucket_key:
+                result = f"{self.aws_account_name}/{self.aws_bucket_key}"
+            else:
+                result = f"{self.stage}/{self.module}"
+        return result
+
+    @classmethod
+    def get_mapping_default(cls) -> List[Dict[str, str]]:
+        return []

src/main/python/ddadevops/domain/provider_digitalocean.py

@@ -35,6 +35,19 @@ class Digitalocean(Validateable, CredentialMappingDefault):
             result += self.__validate_is_not_empty__("do_region")
         return result
 
+    def backend_config(self) -> Dict[str, Any]:
+        result = {}
+        if self.do_as_backend:
+            result = {
+                "access_key": self.do_spaces_access_id,
+                "secret_key": self.do_spaces_secret_key,
+                "endpoint": self.do_endpoint,
+                "bucket": self.do_bucket,
+                "key": self.__bucket_key__(),
+                "region": self.do_region,
+            }
+        return result
+
     def resources_from_package(self) -> Set[str]:
         result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"}
         if self.do_as_backend:
@@ -61,19 +74,6 @@ class Digitalocean(Validateable, CredentialMappingDefault):
             )
         return result
 
-    def backend_config(self) -> Dict[str, Any]:
-        result = {}
-        if self.do_as_backend:
-            result = {
-                "access_key": self.do_spaces_access_id,
-                "secret_key": self.do_spaces_secret_key,
-                "endpoint": self.do_endpoint,
-                "bucket": self.do_bucket,
-                "key": self.__bucket_key__(),
-                "region": self.do_region,
-            }
-        return result
-
     def is_local_state(self):
         return not self.do_as_backend
 

src/main/python/ddadevops/domain/terraform.py

@@ -7,6 +7,7 @@ from .common import (
 )
 from .provider_digitalocean import Digitalocean
 from .provider_hetzner import Hetzner
+from .provider_aws import Aws
 
 
 class TerraformDomain(Validateable):
@@ -39,6 +40,8 @@ class TerraformDomain(Validateable):
             self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp)
         if ProviderType.HETZNER in provider_types:
             self.providers[ProviderType.HETZNER] = Hetzner(inp)
+        if ProviderType.AWS in provider_types:
+            self.providers[ProviderType.AWS] = Aws(inp)
 
     def validate(self) -> List[str]:
         result = []

src/main/resources/terraform/aws_backend_properties_vars.tf

@@ -1,5 +1,5 @@
+variable "account_name" {}
 variable "bucket" {}
 variable "key" {}
 variable "kms_key_id" {}
 variable "region" {}
-variable "account_name" {}

src/main/resources/terraform/do_backend_properties_vars.tf

@@ -1,5 +1,5 @@
+variable "account_name" {}
 variable "endpoint" {}
 variable "bucket" {}
 variable "key" {}
 variable "region" {}
-variable "account_name" {}

src/test/python/domain/helper.py

@@ -24,7 +24,7 @@ def devops_config(overrides: dict) -> dict:
         "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint",
         "k3s_enable_echo": "false",
         "k3s_app_filename_to_provision": "k3s_app.yaml",
-        "tf_provider_types": ["DIGITALOCEAN", "HETZNER"],
+        "tf_provider_types": ["DIGITALOCEAN", "HETZNER", "AWS"],
         "tf_additional_vars": [],
         "tf_output_json_name": "the_out.json",
         "tf_use_workspace": None,
@@ -45,6 +45,10 @@ def devops_config(overrides: dict) -> dict:
         "do_bucket": "bucket",
         "do_region": "region",
         "hetzner_api_key": "hetzner_api_key",
+        "aws_as_backend": True,
+        "aws_bucket": "bucket",
+        "aws_region": "region",
+        "aws_bucket_kms_key_id": "aws_bucket_kms_key_id",
         "release_type": "NONE",
         "release_main_branch": "main",
         "release_current_branch": "my_feature",

src/test/python/domain/test_provider_aws.py

@@ -0,0 +1,77 @@
+from pybuilder.core import Project
+from pathlib import Path
+from src.main.python.ddadevops.domain import (
+    BuildType,
+    Aws,
+)
+from .helper import devops_config
+
+
+def test_aws_creation():
+    sut = Aws(
+        {
+            "module": "module",
+            "stage": "test",
+            "aws_account_name": "aws_account_name",
+        }
+    )
+    assert sut is not None
+    assert sut.is_valid()
+
+    sut = Aws(
+        {
+            "module": "module",
+            "stage": "test",
+            "aws_as_backend": True,
+            "aws_bucket": "bucket",
+            "aws_bucket_kms_key_id": "aws_bucket_kms_key_id",
+        }
+    )
+    assert sut is not None
+    assert sut.is_valid()
+
+
+def test_should_calculate_backend_config():
+    sut = Aws(
+        devops_config(
+            {
+                "module": "dns_aws",
+                "stage": "prod",
+                "aws_bucket": "meissa-configuration",
+                "aws_bucket_kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration",
+                "aws_region": "eu-central-1",
+            }
+        )
+    )
+    assert {
+        "bucket": "meissa-configuration",
+        "key": "prod/dns_aws",
+        "kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration",
+        "region": "eu-central-1",
+    } == sut.backend_config()
+
+
+def test_should_calculate_project_vars():
+    sut = Aws(
+        devops_config(
+            {
+                "aws_as_backend": False,
+            }
+        )
+    )
+    assert {} == sut.project_vars()
+
+    sut = Aws(
+        devops_config(
+            {
+                "aws_as_backend": True,
+            }
+        )
+    )
+    assert {
+        "account_name": "test",
+        "bucket": "bucket",
+        "key": "test/module",
+        "kms_key_id": "aws_bucket_kms_key_id",
+        "region": "region",
+    } == sut.project_vars()

src/test/python/domain/test_terraform.py

@@ -16,6 +16,7 @@ def test_creation():
     assert sut
     assert sut.providers[ProviderType.DIGITALOCEAN]
     assert sut.providers[ProviderType.HETZNER]
+    assert sut.providers[ProviderType.AWS]
 
 
 def test_should_calculate_output_json_name():
@@ -42,6 +43,14 @@ def test_should_validate():
     sut = TerraformDomain(config)
     assert not sut.is_valid()
 
+    config = devops_config(
+        {
+            "aws_account_name": "",
+        }
+    )
+    sut = TerraformDomain(config)
+    assert not sut.is_valid()
+
 
 def test_should_calculate_terraform_build_commons_path():
     config = devops_config({})
@@ -64,7 +73,12 @@ def test_should_calculate_project_vars():
     sut = TerraformDomain(config)
     assert {"module": "module", "stage": "test"} == sut.project_vars()
 
-    config = devops_config({"do_as_backend": False,})
+    config = devops_config(
+        {
+            "do_as_backend": False,
+            "aws_as_backend": False,
+        }
+    )
     sut = TerraformDomain(config)
     assert {
         "module": "module",
@@ -144,6 +158,9 @@ def test_should_calculate_resources_from_package():
         "provider_registry.tf",
         "hetzner_provider.tf",
         "hetzner_mixin_vars.tf",
+        "aws_backend_with_properties.tf",
+        "aws_provider.tf",
+        "aws_backend_properties_vars.tf",
         "my.file",
     } == sut.resources_from_package()