Merge branch 'add-aws-backend' into 'main'

Add aws backend

See merge request domaindrivenarchitecture/dda-devops-build!14

build.py

@@ -33,7 +33,7 @@ default_task = "dev"
 name = "ddadevops"
 MODULE = "not-used"
 PROJECT_ROOT_PATH = "."
-version = "4.0.0-dev74"
+version = "4.0.0-dev79"
 summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
 description = __doc__
 authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]
@@ -142,7 +142,6 @@ def tag_bump_and_push_release(project):
 
 def build(project, release_type):
     build = get_devops_build(project)
-    # TODO: release_type is here a string!
     build.update_release_type(release_type)
     test(project)
     lint(project)

doc/CredentialsMappin.md

@@ -0,0 +1,111 @@
+# CredentialsMapping
+
+Credentials used for builds can be received from various places during build object initialization.
+Precedence is
+1. if there is a ENV, that is used
+2. if there is a gopass_path only is given, the gopass password from path is used
+3. if there is a gopass_path and gopass_field is given, the field from path is used.
+
+
+
+```mermaid
+classDiagram
+    class Credentials {
+        <<AggregateRoot>>
+    }
+    class CredentialMapping {
+        name
+        gopass_path
+        gopass_field
+        gopass_type()
+        name_for_input() - name used in context of build (according to python conventions converted to lower case snake case)
+        name_for_environment () - name used in context of ENV (according to ENV conventions converted to upper case snake case)
+    }
+
+    Credentials *-- "0..n" CredentialMapping: mappings[name]
+    Credentials *-- "0..n" CredentialMapping: default_mappings
+```
+
+## Input
+
+| name         | description                                              | default |
+| ------------ | -------------------------------------------------------- | ------- |
+| gopass_path  | path of secret in gopass                                 | -       |
+| gopass_field | optional: if given, the field is read, the password else | -       |
+| name         | name in context of build & ENV                           | -       |  |
+
+## Example Usage
+### build.py
+
+```python
+from os import environ
+from pybuilder.core import task, init
+from ddadevops import *
+
+name = 'my-project'
+MODULE = 'my-module'
+PROJECT_ROOT_PATH = '..'
+
+class MyBuild(DevopsTerraformBuild, C4kBuild, ProvsK3sBuild):
+    pass
+
+@init
+def initialize(project):
+    project.build_depends_on("ddadevops>=4.0.0")
+    stage = environ["STAGE"]
+    
+    if stage == "test":
+        tmp_letsencrypt_endpoint = "staging"
+        tmp_jitsi_secrets_path = "server/jitsi-test"
+    elif stage == "prod":
+        tmp_letsencrypt_endpoint = "prod"
+        tmp_jitsi_secrets_path = "server/jitsi-prod"
+
+    c4k_config = {"issuer": tmp_letsencrypt_endpoint}
+    c4k_auth = {
+        "jvb-auth-password": gopass_field_from_path(
+            tmp_jitsi_secrets_path, "jvb-auth-password"
+        ),
+        "jicofo-auth-password": gopass_field_from_path(
+            tmp_jitsi_secrets_path, "jvb-auth-password"
+        ),
+        "jicofo-component-secret": gopass_field_from_path(
+            tmp_jitsi_secrets_path, "jicofo-component-secret"
+        ),
+    }
+
+    config = {
+        "credentials_mapping": [
+            {
+                "gopass_path": environ.get("DIGITALOCEAN_TOKEN_KEY_PATH", None),
+                "name": "do_api_key",
+            },
+            {
+                "gopass_path": environ.get("HETZNER_API_KEY_PATH", None),
+                "name": "hetzner_api_key",
+            },
+        ],
+        "name": name,
+        "module": MODULE,
+        "stage": stage,
+        "project_root_path": PROJECT_ROOT_PATH,
+        "build_types": ["TERRAFORM", "C4K", "K3S"],
+        "mixin_types": [],
+        "tf_provider_types": ["DIGITALOCEAN", "HETZNER"],
+        "tf_use_workspace": False,
+        "tf_terraform_semantic_version": "1.4.2",
+        "do_as_backend": True,
+        "do_bucket": "my-configuration",
+        "k3s_app_filename_to_provision": "out_jitsi.yaml",
+        "k3s_letsencrypt_endpoint": tmp_letsencrypt_endpoint,
+        "k3s_letsencrypt_email": "admin@your.domain",
+        "c4k_config": c4k_config,
+        "c4k_auth": c4k_auth,
+    }
+
+    build = MyBuild(project, config)
+    build.initialize_build_dir()
+
+
+
+```

doc/DevopsTerraformBuildWithAwsProvider.md

@@ -4,6 +4,8 @@
 
 | name                  | description                                                                                                                    | default        |
 | --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------- |
+| aws_access_key        | your aws access-key                                                                                                            |                |
+| aws_secret_key        | your aws secret-key                                                                                                            |                |
 | aws_as_backend        | you can use aws s3 as backend state storage                                                                                    | False          |
 | aws_region            | in case of backend usage                                                                                                       | "eu-central-1" |
 | aws_bucket            | in case of backend usage, the bucket your state is stored in. the url is S3://{aws_bucket}/{aws_bucket_key}/{aws_account_name} |                |

doc/architecture/Domain.md

@@ -50,7 +50,7 @@ classDiagram
         tf_terraform_semantic_version
     }
 
-    class Digitalocean {
+    class ProviderDigitalocean {
         do_api_key
         do_spaces_access_key
         do_spaces_secret_key
@@ -62,12 +62,19 @@ classDiagram
         do_region
     }
 
-    class Hetzner {
+    class ProviderHetzner {
         hetzner_api_key
     }
 
-    class Aws {
+    class ProviderAws {
+        aws_access_key
+        aws_secret_key
+        aws_bucket
+        aws_bucket_kms_key_id
         aws_account_name
+        aws_bucket_key
+        aws_as_backend
+        aws_region
     }
     
     class DnsRecord {
@@ -118,9 +125,9 @@ classDiagram
     Devops *-- "0..1" ProvsK3s: specialized_builds
     Devops *-- "0..1" TerraformDomain: specialized_builds
     Devops *-- "0..1" Release: mixins
-    TerraformDomain *-- "0..1" Digitalocean: providers
+    TerraformDomain *-- "0..1" ProviderDigitalocean: providers
-    TerraformDomain *-- "0..1" Hetzner: providers
+    TerraformDomain *-- "0..1" ProviderHetzner: providers
-    TerraformDomain *-- "0..1" Aws: providers
+    TerraformDomain *-- "0..1" ProviderAws: providers
     Release o-- "0..1" BuildFile: primary_build_file
     Release o-- "0..n" BuildFile: secondary_build_files
     BuildFile *-- "1" Version

src/main/python/ddadevops/domain/provider_aws.py

@@ -9,6 +9,8 @@ class Aws(Validateable, CredentialMappingDefault):
     ):
         self.stage = inp.get("stage")
         self.module = inp.get("module")
+        self.aws_access_key = inp.get("aws_access_key")
+        self.aws_secret_key = inp.get("aws_secret_key")
         self.aws_bucket = inp.get("aws_bucket")
         self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id")
         self.aws_account_name = inp.get("aws_account_name", self.stage)
@@ -20,12 +22,13 @@ class Aws(Validateable, CredentialMappingDefault):
         result = []
         result += self.__validate_is_not_empty__("stage")
         result += self.__validate_is_not_empty__("module")
+        result += self.__validate_is_not_empty__("aws_access_key")
+        result += self.__validate_is_not_empty__("aws_secret_key")
         result += self.__validate_is_not_empty__("aws_account_name")
         result += self.__validate_is_not_empty__("aws_as_backend")
         if self.aws_as_backend:
             result += self.__validate_is_not_empty__("aws_bucket")
             result += self.__validate_is_not_empty__("aws_bucket_key")
-            result += self.__validate_is_not_empty__("aws_bucket_kms_key_id")
             result += self.__validate_is_not_empty__("aws_region")
         return result
 
@@ -42,25 +45,44 @@ class Aws(Validateable, CredentialMappingDefault):
         return result
 
     def resources_from_package(self) -> Set[str]:
-        result = {"provider_registry.tf", "aws_provider.tf"}
+        result = {"provider_registry.tf", "aws_provider.tf", "aws_provider_vars.tf"}
         if self.aws_as_backend:
-            result.update(
+            if self.aws_bucket_kms_key_id:
-                {"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"}
+                result.update(
-            )
+                    {
+                        "aws_backend_wkms_vars.tf",
+                        "aws_backend.tf",
+                    }
+                )
+            else:
+                result.update(
+                    {
+                        "aws_backend_wokms_vars.tf",
+                        "aws_backend.tf",
+                    }
+                )
         return result
 
     def project_vars(self):
-        result = {}
+        result = {
+            "aws_access_key": self.aws_access_key,
+            "aws_secret_key": self.aws_secret_key,
+            "aws_region": self.aws_region,
+        }
         if self.aws_as_backend:
             result.update(
                 {
                     "account_name": self.aws_account_name,
                     "bucket": self.aws_bucket,
                     "key": self.__bucket_key__(),
-                    "kms_key_id": self.aws_bucket_kms_key_id,
-                    "region": self.aws_region,
                 }
             )
+            if self.aws_bucket_kms_key_id:
+                result.update(
+                    {
+                        "kms_key_id": self.aws_bucket_kms_key_id,
+                    }
+                )
         return result
 
     def is_local_state(self):

src/main/python/ddadevops/domain/provider_digitalocean.py

@@ -49,10 +49,10 @@ class Digitalocean(Validateable, CredentialMappingDefault):
         return result
 
     def resources_from_package(self) -> Set[str]:
-        result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"}
+        result = {"provider_registry.tf", "do_provider.tf", "do_provider_vars.tf"}
         if self.do_as_backend:
             result.update(
-                {"do_backend_properties_vars.tf", "do_backend_with_properties.tf"}
+                {"do_backend_vars.tf", "do_backend.tf"}
             )
         return result
 

src/main/python/ddadevops/domain/provider_hetzner.py

@@ -18,7 +18,11 @@ class Hetzner(Validateable, CredentialMappingDefault):
         return {}
 
     def resources_from_package(self) -> Set[str]:
-        return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"}
+        return {
+            "provider_registry.tf",
+            "hetzner_provider.tf",
+            "hetzner_provider_vars.tf",
+        }
 
     def project_vars(self):
         return {"hetzner_api_key": self.hetzner_api_key}

src/main/resources/terraform/aws_backend_wkms_vars.tf

@@ -1,5 +1,4 @@
 variable "account_name" {}
 variable "bucket" {}
 variable "key" {}
-variable "kms_key_id" {}
+variable "kms_key_id" {}
-variable "region" {}

src/main/resources/terraform/aws_backend_wokms_vars.tf

@@ -0,0 +1,3 @@
+variable "account_name" {}
+variable "bucket" {}
+variable "key" {}

src/main/resources/terraform/aws_provider.tf

@@ -1,3 +1,5 @@
 provider "aws" {
-  region = var.region
+  region = var.aws_region
+  access_key = var.aws_access_key
+  secret_key = var.aws_secret_key
 }

src/main/resources/terraform/aws_provider_vars.tf

@@ -0,0 +1,3 @@
+variable "aws_access_key" {}
+variable "aws_secret_key" {}
+variable "aws_region" {}

src/main/resources/terraform/provider_registry.tf

@@ -1,29 +1,24 @@
 terraform {
   required_providers {
-
-    exoscale = {
-      source  = "exoscale/exoscale"
-      version = ">= 0.29.0"
-    }
      
     hcloud = {
       source = "hetznercloud/hcloud"
-      version = ">= 1.23.0"
+      version = ">= 1.41"
     }
 
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 3.0"
+      version = "~> 5.6"
     }
 
     hetznerdns = {
       source = "timohirt/hetznerdns"
-      version = ">= 1.1.0"
+      version = ">= 2.2"
     }
 
     digitalocean = {
       source  = "digitalocean/digitalocean"
-      version = "~> 2.0"
+      version = "~> 2.28"
     }
   }
 }

src/test/python/domain/helper.py

@@ -42,6 +42,8 @@ def devops_config(overrides: dict) -> dict:
         "do_bucket": "bucket",
         "do_region": "region",
         "hetzner_api_key": "hetzner_api_key",
+        "aws_access_key": "aws_access_key",
+        "aws_secret_key": "aws_secret_key",
         "aws_as_backend": True,
         "aws_bucket": "bucket",
         "aws_region": "region",

src/test/python/domain/test_provider_aws.py

@@ -12,6 +12,8 @@ def test_aws_creation():
         {
             "module": "module",
             "stage": "test",
+            "aws_access_key": "aws_access_key",
+            "aws_secret_key": "aws_secret_key",
             "aws_account_name": "aws_account_name",
         }
     )
@@ -22,6 +24,8 @@ def test_aws_creation():
         {
             "module": "module",
             "stage": "test",
+            "aws_access_key": "aws_access_key",
+            "aws_secret_key": "aws_secret_key",
             "aws_as_backend": True,
             "aws_bucket": "bucket",
             "aws_bucket_kms_key_id": "aws_bucket_kms_key_id",
@@ -59,7 +63,11 @@ def test_should_calculate_project_vars():
             }
         )
     )
-    assert {} == sut.project_vars()
+    assert {
+        "aws_access_key": "aws_access_key",
+        "aws_secret_key": "aws_secret_key",
+        "aws_region": "region",
+    } == sut.project_vars()
 
     sut = Aws(
         devops_config(
@@ -69,9 +77,11 @@ def test_should_calculate_project_vars():
         )
     )
     assert {
+        "aws_access_key": "aws_access_key",
+        "aws_secret_key": "aws_secret_key",
+        "aws_region": "region",
         "account_name": "test",
         "bucket": "bucket",
         "key": "test/module",
         "kms_key_id": "aws_bucket_kms_key_id",
-        "region": "region",
     } == sut.project_vars()

src/test/python/domain/test_provider_digitalocean.py

@@ -76,9 +76,9 @@ def test_should_calculate_project_vars():
         "do_api_key": "api_key",
         "do_spaces_access_id": "spaces_id",
         "do_spaces_secret_key": "spaces_secret",
+        "region": "region",
         "account_name": "test",
         "endpoint": "endpoint",
         "bucket": "bucket",
         "key": "test/module",
-        "region": "region",
     } == sut.project_vars()

src/test/python/domain/test_terraform.py

@@ -87,6 +87,9 @@ def test_should_calculate_project_vars():
         "do_spaces_access_id": "spaces_id",
         "do_spaces_secret_key": "spaces_secret",
         "hetzner_api_key": "hetzner_api_key",
+        "aws_access_key": "aws_access_key",
+        "aws_secret_key": "aws_secret_key",
+        "aws_region": "region",
     } == sut.project_vars()
 
 
@@ -111,7 +114,7 @@ def test_should_calculate_resources_from_package():
         "terraform_build_vars.tf",
         "provider_registry.tf",
         "do_provider.tf",
-        "do_mixin_vars.tf",
+        "do_provider_vars.tf",
     } == sut.resources_from_package()
 
     sut = TerraformDomain(
@@ -127,9 +130,9 @@ def test_should_calculate_resources_from_package():
         "terraform_build_vars.tf",
         "provider_registry.tf",
         "do_provider.tf",
-        "do_mixin_vars.tf",
+        "do_provider_vars.tf",
-        "do_backend_properties_vars.tf",
+        "do_backend_vars.tf",
-        "do_backend_with_properties.tf",
+        "do_backend.tf",
     } == sut.resources_from_package()
 
     config = devops_config({"tf_provider_types": ["HETZNER"]})
@@ -139,7 +142,7 @@ def test_should_calculate_resources_from_package():
         "terraform_build_vars.tf",
         "provider_registry.tf",
         "hetzner_provider.tf",
-        "hetzner_mixin_vars.tf",
+        "hetzner_provider_vars.tf",
     } == sut.resources_from_package()
 
     config = devops_config(
@@ -154,13 +157,14 @@ def test_should_calculate_resources_from_package():
         "terraform_build_vars.tf",
         "provider_registry.tf",
         "do_provider.tf",
-        "do_mixin_vars.tf",
+        "do_provider_vars.tf",
         "provider_registry.tf",
         "hetzner_provider.tf",
-        "hetzner_mixin_vars.tf",
+        "hetzner_provider_vars.tf",
-        "aws_backend_with_properties.tf",
+        "aws_backend.tf",
         "aws_provider.tf",
-        "aws_backend_properties_vars.tf",
+        "aws_provider_vars.tf",
+        "aws_backend_wkms_vars.tf",
         "my.file",
     } == sut.resources_from_package()