diff --git a/build.py b/build.py index c59de6e..ea5c1b6 100644 --- a/build.py +++ b/build.py @@ -33,7 +33,7 @@ default_task = "dev" name = "ddadevops" MODULE = "not-used" PROJECT_ROOT_PATH = "." -version = "4.0.0-dev74" +version = "4.0.0-dev79" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] @@ -142,7 +142,6 @@ def tag_bump_and_push_release(project): def build(project, release_type): build = get_devops_build(project) - # TODO: release_type is here a string! build.update_release_type(release_type) test(project) lint(project) diff --git a/doc/CredentialsMappin.md b/doc/CredentialsMappin.md new file mode 100644 index 0000000..5c2fe92 --- /dev/null +++ b/doc/CredentialsMappin.md @@ -0,0 +1,111 @@ +# CredentialsMapping + +Credentials used for builds can be received from various places during build object initialization. +Precedence is +1. if there is a ENV, that is used +2. if there is a gopass_path only is given, the gopass password from path is used +3. if there is a gopass_path and gopass_field is given, the field from path is used. + + + +```mermaid +classDiagram + class Credentials { + <> + } + class CredentialMapping { + name + gopass_path + gopass_field + gopass_type() + name_for_input() - name used in context of build (according to python conventions converted to lower case snake case) + name_for_environment () - name used in context of ENV (according to ENV conventions converted to upper case snake case) + } + + Credentials *-- "0..n" CredentialMapping: mappings[name] + Credentials *-- "0..n" CredentialMapping: default_mappings +``` + +## Input + +| name | description | default | +| ------------ | -------------------------------------------------------- | ------- | +| gopass_path | path of secret in gopass | - | +| gopass_field | optional: if given, the field is read, the password else | - | +| name | name in context of build & ENV | - | | + +## Example Usage +### build.py + +```python +from os import environ +from pybuilder.core import task, init +from ddadevops import * + +name = 'my-project' +MODULE = 'my-module' +PROJECT_ROOT_PATH = '..' + +class MyBuild(DevopsTerraformBuild, C4kBuild, ProvsK3sBuild): + pass + +@init +def initialize(project): + project.build_depends_on("ddadevops>=4.0.0") + stage = environ["STAGE"] + + if stage == "test": + tmp_letsencrypt_endpoint = "staging" + tmp_jitsi_secrets_path = "server/jitsi-test" + elif stage == "prod": + tmp_letsencrypt_endpoint = "prod" + tmp_jitsi_secrets_path = "server/jitsi-prod" + + c4k_config = {"issuer": tmp_letsencrypt_endpoint} + c4k_auth = { + "jvb-auth-password": gopass_field_from_path( + tmp_jitsi_secrets_path, "jvb-auth-password" + ), + "jicofo-auth-password": gopass_field_from_path( + tmp_jitsi_secrets_path, "jvb-auth-password" + ), + "jicofo-component-secret": gopass_field_from_path( + tmp_jitsi_secrets_path, "jicofo-component-secret" + ), + } + + config = { + "credentials_mapping": [ + { + "gopass_path": environ.get("DIGITALOCEAN_TOKEN_KEY_PATH", None), + "name": "do_api_key", + }, + { + "gopass_path": environ.get("HETZNER_API_KEY_PATH", None), + "name": "hetzner_api_key", + }, + ], + "name": name, + "module": MODULE, + "stage": stage, + "project_root_path": PROJECT_ROOT_PATH, + "build_types": ["TERRAFORM", "C4K", "K3S"], + "mixin_types": [], + "tf_provider_types": ["DIGITALOCEAN", "HETZNER"], + "tf_use_workspace": False, + "tf_terraform_semantic_version": "1.4.2", + "do_as_backend": True, + "do_bucket": "my-configuration", + "k3s_app_filename_to_provision": "out_jitsi.yaml", + "k3s_letsencrypt_endpoint": tmp_letsencrypt_endpoint, + "k3s_letsencrypt_email": "admin@your.domain", + "c4k_config": c4k_config, + "c4k_auth": c4k_auth, + } + + build = MyBuild(project, config) + build.initialize_build_dir() + + + +``` diff --git a/doc/DevopsTerraformBuildWithAwsProvider.md b/doc/DevopsTerraformBuildWithAwsProvider.md index 1dca96b..b752a8a 100644 --- a/doc/DevopsTerraformBuildWithAwsProvider.md +++ b/doc/DevopsTerraformBuildWithAwsProvider.md @@ -4,6 +4,8 @@ | name | description | default | | --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------- | +| aws_access_key | your aws access-key | | +| aws_secret_key | your aws secret-key | | | aws_as_backend | you can use aws s3 as backend state storage | False | | aws_region | in case of backend usage | "eu-central-1" | | aws_bucket | in case of backend usage, the bucket your state is stored in. the url is S3://{aws_bucket}/{aws_bucket_key}/{aws_account_name} | | diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 8148756..e35af13 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -50,7 +50,7 @@ classDiagram tf_terraform_semantic_version } - class Digitalocean { + class ProviderDigitalocean { do_api_key do_spaces_access_key do_spaces_secret_key @@ -62,12 +62,19 @@ classDiagram do_region } - class Hetzner { + class ProviderHetzner { hetzner_api_key } - class Aws { + class ProviderAws { + aws_access_key + aws_secret_key + aws_bucket + aws_bucket_kms_key_id aws_account_name + aws_bucket_key + aws_as_backend + aws_region } class DnsRecord { @@ -118,9 +125,9 @@ classDiagram Devops *-- "0..1" ProvsK3s: specialized_builds Devops *-- "0..1" TerraformDomain: specialized_builds Devops *-- "0..1" Release: mixins - TerraformDomain *-- "0..1" Digitalocean: providers - TerraformDomain *-- "0..1" Hetzner: providers - TerraformDomain *-- "0..1" Aws: providers + TerraformDomain *-- "0..1" ProviderDigitalocean: providers + TerraformDomain *-- "0..1" ProviderHetzner: providers + TerraformDomain *-- "0..1" ProviderAws: providers Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version diff --git a/src/main/python/ddadevops/domain/provider_aws.py b/src/main/python/ddadevops/domain/provider_aws.py index e0ef793..870cd79 100644 --- a/src/main/python/ddadevops/domain/provider_aws.py +++ b/src/main/python/ddadevops/domain/provider_aws.py @@ -9,6 +9,8 @@ class Aws(Validateable, CredentialMappingDefault): ): self.stage = inp.get("stage") self.module = inp.get("module") + self.aws_access_key = inp.get("aws_access_key") + self.aws_secret_key = inp.get("aws_secret_key") self.aws_bucket = inp.get("aws_bucket") self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id") self.aws_account_name = inp.get("aws_account_name", self.stage) @@ -20,12 +22,13 @@ class Aws(Validateable, CredentialMappingDefault): result = [] result += self.__validate_is_not_empty__("stage") result += self.__validate_is_not_empty__("module") + result += self.__validate_is_not_empty__("aws_access_key") + result += self.__validate_is_not_empty__("aws_secret_key") result += self.__validate_is_not_empty__("aws_account_name") result += self.__validate_is_not_empty__("aws_as_backend") if self.aws_as_backend: result += self.__validate_is_not_empty__("aws_bucket") result += self.__validate_is_not_empty__("aws_bucket_key") - result += self.__validate_is_not_empty__("aws_bucket_kms_key_id") result += self.__validate_is_not_empty__("aws_region") return result @@ -42,25 +45,44 @@ class Aws(Validateable, CredentialMappingDefault): return result def resources_from_package(self) -> Set[str]: - result = {"provider_registry.tf", "aws_provider.tf"} + result = {"provider_registry.tf", "aws_provider.tf", "aws_provider_vars.tf"} if self.aws_as_backend: - result.update( - {"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"} - ) + if self.aws_bucket_kms_key_id: + result.update( + { + "aws_backend_wkms_vars.tf", + "aws_backend.tf", + } + ) + else: + result.update( + { + "aws_backend_wokms_vars.tf", + "aws_backend.tf", + } + ) return result def project_vars(self): - result = {} + result = { + "aws_access_key": self.aws_access_key, + "aws_secret_key": self.aws_secret_key, + "aws_region": self.aws_region, + } if self.aws_as_backend: result.update( { "account_name": self.aws_account_name, "bucket": self.aws_bucket, "key": self.__bucket_key__(), - "kms_key_id": self.aws_bucket_kms_key_id, - "region": self.aws_region, } ) + if self.aws_bucket_kms_key_id: + result.update( + { + "kms_key_id": self.aws_bucket_kms_key_id, + } + ) return result def is_local_state(self): diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index f21fa83..2bc2c2e 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -49,10 +49,10 @@ class Digitalocean(Validateable, CredentialMappingDefault): return result def resources_from_package(self) -> Set[str]: - result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} + result = {"provider_registry.tf", "do_provider.tf", "do_provider_vars.tf"} if self.do_as_backend: result.update( - {"do_backend_properties_vars.tf", "do_backend_with_properties.tf"} + {"do_backend_vars.tf", "do_backend.tf"} ) return result diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py index bb0b786..3f5e752 100644 --- a/src/main/python/ddadevops/domain/provider_hetzner.py +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -18,7 +18,11 @@ class Hetzner(Validateable, CredentialMappingDefault): return {} def resources_from_package(self) -> Set[str]: - return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"} + return { + "provider_registry.tf", + "hetzner_provider.tf", + "hetzner_provider_vars.tf", + } def project_vars(self): return {"hetzner_api_key": self.hetzner_api_key} diff --git a/src/main/resources/terraform/aws_backend_with_properties.tf b/src/main/resources/terraform/aws_backend.tf similarity index 100% rename from src/main/resources/terraform/aws_backend_with_properties.tf rename to src/main/resources/terraform/aws_backend.tf diff --git a/src/main/resources/terraform/aws_backend_properties_vars.tf b/src/main/resources/terraform/aws_backend_wkms_vars.tf similarity index 59% rename from src/main/resources/terraform/aws_backend_properties_vars.tf rename to src/main/resources/terraform/aws_backend_wkms_vars.tf index 08abc4b..3349eb0 100644 --- a/src/main/resources/terraform/aws_backend_properties_vars.tf +++ b/src/main/resources/terraform/aws_backend_wkms_vars.tf @@ -1,5 +1,4 @@ variable "account_name" {} variable "bucket" {} variable "key" {} -variable "kms_key_id" {} -variable "region" {} \ No newline at end of file +variable "kms_key_id" {} \ No newline at end of file diff --git a/src/main/resources/terraform/aws_backend_wokms_vars.tf b/src/main/resources/terraform/aws_backend_wokms_vars.tf new file mode 100644 index 0000000..3fc147f --- /dev/null +++ b/src/main/resources/terraform/aws_backend_wokms_vars.tf @@ -0,0 +1,3 @@ +variable "account_name" {} +variable "bucket" {} +variable "key" {} \ No newline at end of file diff --git a/src/main/resources/terraform/aws_provider.tf b/src/main/resources/terraform/aws_provider.tf index dc58d9a..e62a63c 100644 --- a/src/main/resources/terraform/aws_provider.tf +++ b/src/main/resources/terraform/aws_provider.tf @@ -1,3 +1,5 @@ provider "aws" { - region = var.region + region = var.aws_region + access_key = var.aws_access_key + secret_key = var.aws_secret_key } diff --git a/src/main/resources/terraform/aws_provider_vars.tf b/src/main/resources/terraform/aws_provider_vars.tf new file mode 100644 index 0000000..52c2559 --- /dev/null +++ b/src/main/resources/terraform/aws_provider_vars.tf @@ -0,0 +1,3 @@ +variable "aws_access_key" {} +variable "aws_secret_key" {} +variable "aws_region" {} diff --git a/src/main/resources/terraform/do_backend_with_properties.tf b/src/main/resources/terraform/do_backend.tf similarity index 100% rename from src/main/resources/terraform/do_backend_with_properties.tf rename to src/main/resources/terraform/do_backend.tf diff --git a/src/main/resources/terraform/do_backend_properties_vars.tf b/src/main/resources/terraform/do_backend_vars.tf similarity index 100% rename from src/main/resources/terraform/do_backend_properties_vars.tf rename to src/main/resources/terraform/do_backend_vars.tf diff --git a/src/main/resources/terraform/do_mixin_vars.tf b/src/main/resources/terraform/do_provider_vars.tf similarity index 100% rename from src/main/resources/terraform/do_mixin_vars.tf rename to src/main/resources/terraform/do_provider_vars.tf diff --git a/src/main/resources/terraform/hetzner_mixin_vars.tf b/src/main/resources/terraform/hetzner_provider_vars.tf similarity index 100% rename from src/main/resources/terraform/hetzner_mixin_vars.tf rename to src/main/resources/terraform/hetzner_provider_vars.tf diff --git a/src/main/resources/terraform/provider_registry.tf b/src/main/resources/terraform/provider_registry.tf index eb53c21..6336aab 100644 --- a/src/main/resources/terraform/provider_registry.tf +++ b/src/main/resources/terraform/provider_registry.tf @@ -1,29 +1,24 @@ terraform { required_providers { - - exoscale = { - source = "exoscale/exoscale" - version = ">= 0.29.0" - } hcloud = { source = "hetznercloud/hcloud" - version = ">= 1.23.0" + version = ">= 1.41" } aws = { source = "hashicorp/aws" - version = "~> 3.0" + version = "~> 5.6" } hetznerdns = { source = "timohirt/hetznerdns" - version = ">= 1.1.0" + version = ">= 2.2" } digitalocean = { source = "digitalocean/digitalocean" - version = "~> 2.0" + version = "~> 2.28" } } } \ No newline at end of file diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 48183fe..cf58a0f 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -42,6 +42,8 @@ def devops_config(overrides: dict) -> dict: "do_bucket": "bucket", "do_region": "region", "hetzner_api_key": "hetzner_api_key", + "aws_access_key": "aws_access_key", + "aws_secret_key": "aws_secret_key", "aws_as_backend": True, "aws_bucket": "bucket", "aws_region": "region", diff --git a/src/test/python/domain/test_provider_aws.py b/src/test/python/domain/test_provider_aws.py index 7334039..8287ef4 100644 --- a/src/test/python/domain/test_provider_aws.py +++ b/src/test/python/domain/test_provider_aws.py @@ -12,6 +12,8 @@ def test_aws_creation(): { "module": "module", "stage": "test", + "aws_access_key": "aws_access_key", + "aws_secret_key": "aws_secret_key", "aws_account_name": "aws_account_name", } ) @@ -22,6 +24,8 @@ def test_aws_creation(): { "module": "module", "stage": "test", + "aws_access_key": "aws_access_key", + "aws_secret_key": "aws_secret_key", "aws_as_backend": True, "aws_bucket": "bucket", "aws_bucket_kms_key_id": "aws_bucket_kms_key_id", @@ -59,7 +63,11 @@ def test_should_calculate_project_vars(): } ) ) - assert {} == sut.project_vars() + assert { + "aws_access_key": "aws_access_key", + "aws_secret_key": "aws_secret_key", + "aws_region": "region", + } == sut.project_vars() sut = Aws( devops_config( @@ -69,9 +77,11 @@ def test_should_calculate_project_vars(): ) ) assert { + "aws_access_key": "aws_access_key", + "aws_secret_key": "aws_secret_key", + "aws_region": "region", "account_name": "test", "bucket": "bucket", "key": "test/module", "kms_key_id": "aws_bucket_kms_key_id", - "region": "region", } == sut.project_vars() diff --git a/src/test/python/domain/test_provider_digitalocean.py b/src/test/python/domain/test_provider_digitalocean.py index c4d8782..0425d3c 100644 --- a/src/test/python/domain/test_provider_digitalocean.py +++ b/src/test/python/domain/test_provider_digitalocean.py @@ -76,9 +76,9 @@ def test_should_calculate_project_vars(): "do_api_key": "api_key", "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", + "region": "region", "account_name": "test", "endpoint": "endpoint", "bucket": "bucket", "key": "test/module", - "region": "region", } == sut.project_vars() diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 30ab62a..d93773d 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -87,6 +87,9 @@ def test_should_calculate_project_vars(): "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", "hetzner_api_key": "hetzner_api_key", + "aws_access_key": "aws_access_key", + "aws_secret_key": "aws_secret_key", + "aws_region": "region", } == sut.project_vars() @@ -111,7 +114,7 @@ def test_should_calculate_resources_from_package(): "terraform_build_vars.tf", "provider_registry.tf", "do_provider.tf", - "do_mixin_vars.tf", + "do_provider_vars.tf", } == sut.resources_from_package() sut = TerraformDomain( @@ -127,9 +130,9 @@ def test_should_calculate_resources_from_package(): "terraform_build_vars.tf", "provider_registry.tf", "do_provider.tf", - "do_mixin_vars.tf", - "do_backend_properties_vars.tf", - "do_backend_with_properties.tf", + "do_provider_vars.tf", + "do_backend_vars.tf", + "do_backend.tf", } == sut.resources_from_package() config = devops_config({"tf_provider_types": ["HETZNER"]}) @@ -139,7 +142,7 @@ def test_should_calculate_resources_from_package(): "terraform_build_vars.tf", "provider_registry.tf", "hetzner_provider.tf", - "hetzner_mixin_vars.tf", + "hetzner_provider_vars.tf", } == sut.resources_from_package() config = devops_config( @@ -154,13 +157,14 @@ def test_should_calculate_resources_from_package(): "terraform_build_vars.tf", "provider_registry.tf", "do_provider.tf", - "do_mixin_vars.tf", + "do_provider_vars.tf", "provider_registry.tf", "hetzner_provider.tf", - "hetzner_mixin_vars.tf", - "aws_backend_with_properties.tf", + "hetzner_provider_vars.tf", + "aws_backend.tf", "aws_provider.tf", - "aws_backend_properties_vars.tf", + "aws_provider_vars.tf", + "aws_backend_wkms_vars.tf", "my.file", } == sut.resources_from_package()