Merge branch 'add-aws-backend' into 'main'

Add aws backend

See merge request domaindrivenarchitecture/dda-devops-build!14
This commit is contained in:
Michael Jerger 2023-07-06 07:29:47 +00:00
commit b11defa5ce
21 changed files with 206 additions and 43 deletions

View file

@ -33,7 +33,7 @@ default_task = "dev"
name = "ddadevops" name = "ddadevops"
MODULE = "not-used" MODULE = "not-used"
PROJECT_ROOT_PATH = "." PROJECT_ROOT_PATH = "."
version = "4.0.0-dev74" version = "4.0.0-dev79"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__ description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]
@ -142,7 +142,6 @@ def tag_bump_and_push_release(project):
def build(project, release_type): def build(project, release_type):
build = get_devops_build(project) build = get_devops_build(project)
# TODO: release_type is here a string!
build.update_release_type(release_type) build.update_release_type(release_type)
test(project) test(project)
lint(project) lint(project)

111
doc/CredentialsMappin.md Normal file
View file

@ -0,0 +1,111 @@
# CredentialsMapping
Credentials used for builds can be received from various places during build object initialization.
Precedence is
1. if there is a ENV, that is used
2. if there is a gopass_path only is given, the gopass password from path is used
3. if there is a gopass_path and gopass_field is given, the field from path is used.
```mermaid
classDiagram
class Credentials {
<<AggregateRoot>>
}
class CredentialMapping {
name
gopass_path
gopass_field
gopass_type()
name_for_input() - name used in context of build (according to python conventions converted to lower case snake case)
name_for_environment () - name used in context of ENV (according to ENV conventions converted to upper case snake case)
}
Credentials *-- "0..n" CredentialMapping: mappings[name]
Credentials *-- "0..n" CredentialMapping: default_mappings
```
## Input
| name | description | default |
| ------------ | -------------------------------------------------------- | ------- |
| gopass_path | path of secret in gopass | - |
| gopass_field | optional: if given, the field is read, the password else | - |
| name | name in context of build & ENV | - | |
## Example Usage
### build.py
```python
from os import environ
from pybuilder.core import task, init
from ddadevops import *
name = 'my-project'
MODULE = 'my-module'
PROJECT_ROOT_PATH = '..'
class MyBuild(DevopsTerraformBuild, C4kBuild, ProvsK3sBuild):
pass
@init
def initialize(project):
project.build_depends_on("ddadevops>=4.0.0")
stage = environ["STAGE"]
if stage == "test":
tmp_letsencrypt_endpoint = "staging"
tmp_jitsi_secrets_path = "server/jitsi-test"
elif stage == "prod":
tmp_letsencrypt_endpoint = "prod"
tmp_jitsi_secrets_path = "server/jitsi-prod"
c4k_config = {"issuer": tmp_letsencrypt_endpoint}
c4k_auth = {
"jvb-auth-password": gopass_field_from_path(
tmp_jitsi_secrets_path, "jvb-auth-password"
),
"jicofo-auth-password": gopass_field_from_path(
tmp_jitsi_secrets_path, "jvb-auth-password"
),
"jicofo-component-secret": gopass_field_from_path(
tmp_jitsi_secrets_path, "jicofo-component-secret"
),
}
config = {
"credentials_mapping": [
{
"gopass_path": environ.get("DIGITALOCEAN_TOKEN_KEY_PATH", None),
"name": "do_api_key",
},
{
"gopass_path": environ.get("HETZNER_API_KEY_PATH", None),
"name": "hetzner_api_key",
},
],
"name": name,
"module": MODULE,
"stage": stage,
"project_root_path": PROJECT_ROOT_PATH,
"build_types": ["TERRAFORM", "C4K", "K3S"],
"mixin_types": [],
"tf_provider_types": ["DIGITALOCEAN", "HETZNER"],
"tf_use_workspace": False,
"tf_terraform_semantic_version": "1.4.2",
"do_as_backend": True,
"do_bucket": "my-configuration",
"k3s_app_filename_to_provision": "out_jitsi.yaml",
"k3s_letsencrypt_endpoint": tmp_letsencrypt_endpoint,
"k3s_letsencrypt_email": "admin@your.domain",
"c4k_config": c4k_config,
"c4k_auth": c4k_auth,
}
build = MyBuild(project, config)
build.initialize_build_dir()
```

View file

@ -4,6 +4,8 @@
| name | description | default | | name | description | default |
| --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------- | | --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------- |
| aws_access_key | your aws access-key | |
| aws_secret_key | your aws secret-key | |
| aws_as_backend | you can use aws s3 as backend state storage | False | | aws_as_backend | you can use aws s3 as backend state storage | False |
| aws_region | in case of backend usage | "eu-central-1" | | aws_region | in case of backend usage | "eu-central-1" |
| aws_bucket | in case of backend usage, the bucket your state is stored in. the url is S3://{aws_bucket}/{aws_bucket_key}/{aws_account_name} | | | aws_bucket | in case of backend usage, the bucket your state is stored in. the url is S3://{aws_bucket}/{aws_bucket_key}/{aws_account_name} | |

View file

@ -50,7 +50,7 @@ classDiagram
tf_terraform_semantic_version tf_terraform_semantic_version
} }
class Digitalocean { class ProviderDigitalocean {
do_api_key do_api_key
do_spaces_access_key do_spaces_access_key
do_spaces_secret_key do_spaces_secret_key
@ -62,12 +62,19 @@ classDiagram
do_region do_region
} }
class Hetzner { class ProviderHetzner {
hetzner_api_key hetzner_api_key
} }
class Aws { class ProviderAws {
aws_access_key
aws_secret_key
aws_bucket
aws_bucket_kms_key_id
aws_account_name aws_account_name
aws_bucket_key
aws_as_backend
aws_region
} }
class DnsRecord { class DnsRecord {
@ -118,9 +125,9 @@ classDiagram
Devops *-- "0..1" ProvsK3s: specialized_builds Devops *-- "0..1" ProvsK3s: specialized_builds
Devops *-- "0..1" TerraformDomain: specialized_builds Devops *-- "0..1" TerraformDomain: specialized_builds
Devops *-- "0..1" Release: mixins Devops *-- "0..1" Release: mixins
TerraformDomain *-- "0..1" Digitalocean: providers TerraformDomain *-- "0..1" ProviderDigitalocean: providers
TerraformDomain *-- "0..1" Hetzner: providers TerraformDomain *-- "0..1" ProviderHetzner: providers
TerraformDomain *-- "0..1" Aws: providers TerraformDomain *-- "0..1" ProviderAws: providers
Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..1" BuildFile: primary_build_file
Release o-- "0..n" BuildFile: secondary_build_files Release o-- "0..n" BuildFile: secondary_build_files
BuildFile *-- "1" Version BuildFile *-- "1" Version

View file

@ -9,6 +9,8 @@ class Aws(Validateable, CredentialMappingDefault):
): ):
self.stage = inp.get("stage") self.stage = inp.get("stage")
self.module = inp.get("module") self.module = inp.get("module")
self.aws_access_key = inp.get("aws_access_key")
self.aws_secret_key = inp.get("aws_secret_key")
self.aws_bucket = inp.get("aws_bucket") self.aws_bucket = inp.get("aws_bucket")
self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id") self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id")
self.aws_account_name = inp.get("aws_account_name", self.stage) self.aws_account_name = inp.get("aws_account_name", self.stage)
@ -20,12 +22,13 @@ class Aws(Validateable, CredentialMappingDefault):
result = [] result = []
result += self.__validate_is_not_empty__("stage") result += self.__validate_is_not_empty__("stage")
result += self.__validate_is_not_empty__("module") result += self.__validate_is_not_empty__("module")
result += self.__validate_is_not_empty__("aws_access_key")
result += self.__validate_is_not_empty__("aws_secret_key")
result += self.__validate_is_not_empty__("aws_account_name") result += self.__validate_is_not_empty__("aws_account_name")
result += self.__validate_is_not_empty__("aws_as_backend") result += self.__validate_is_not_empty__("aws_as_backend")
if self.aws_as_backend: if self.aws_as_backend:
result += self.__validate_is_not_empty__("aws_bucket") result += self.__validate_is_not_empty__("aws_bucket")
result += self.__validate_is_not_empty__("aws_bucket_key") result += self.__validate_is_not_empty__("aws_bucket_key")
result += self.__validate_is_not_empty__("aws_bucket_kms_key_id")
result += self.__validate_is_not_empty__("aws_region") result += self.__validate_is_not_empty__("aws_region")
return result return result
@ -42,25 +45,44 @@ class Aws(Validateable, CredentialMappingDefault):
return result return result
def resources_from_package(self) -> Set[str]: def resources_from_package(self) -> Set[str]:
result = {"provider_registry.tf", "aws_provider.tf"} result = {"provider_registry.tf", "aws_provider.tf", "aws_provider_vars.tf"}
if self.aws_as_backend: if self.aws_as_backend:
result.update( if self.aws_bucket_kms_key_id:
{"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"} result.update(
) {
"aws_backend_wkms_vars.tf",
"aws_backend.tf",
}
)
else:
result.update(
{
"aws_backend_wokms_vars.tf",
"aws_backend.tf",
}
)
return result return result
def project_vars(self): def project_vars(self):
result = {} result = {
"aws_access_key": self.aws_access_key,
"aws_secret_key": self.aws_secret_key,
"aws_region": self.aws_region,
}
if self.aws_as_backend: if self.aws_as_backend:
result.update( result.update(
{ {
"account_name": self.aws_account_name, "account_name": self.aws_account_name,
"bucket": self.aws_bucket, "bucket": self.aws_bucket,
"key": self.__bucket_key__(), "key": self.__bucket_key__(),
"kms_key_id": self.aws_bucket_kms_key_id,
"region": self.aws_region,
} }
) )
if self.aws_bucket_kms_key_id:
result.update(
{
"kms_key_id": self.aws_bucket_kms_key_id,
}
)
return result return result
def is_local_state(self): def is_local_state(self):

View file

@ -49,10 +49,10 @@ class Digitalocean(Validateable, CredentialMappingDefault):
return result return result
def resources_from_package(self) -> Set[str]: def resources_from_package(self) -> Set[str]:
result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} result = {"provider_registry.tf", "do_provider.tf", "do_provider_vars.tf"}
if self.do_as_backend: if self.do_as_backend:
result.update( result.update(
{"do_backend_properties_vars.tf", "do_backend_with_properties.tf"} {"do_backend_vars.tf", "do_backend.tf"}
) )
return result return result

View file

@ -18,7 +18,11 @@ class Hetzner(Validateable, CredentialMappingDefault):
return {} return {}
def resources_from_package(self) -> Set[str]: def resources_from_package(self) -> Set[str]:
return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"} return {
"provider_registry.tf",
"hetzner_provider.tf",
"hetzner_provider_vars.tf",
}
def project_vars(self): def project_vars(self):
return {"hetzner_api_key": self.hetzner_api_key} return {"hetzner_api_key": self.hetzner_api_key}

View file

@ -1,5 +1,4 @@
variable "account_name" {} variable "account_name" {}
variable "bucket" {} variable "bucket" {}
variable "key" {} variable "key" {}
variable "kms_key_id" {} variable "kms_key_id" {}
variable "region" {}

View file

@ -0,0 +1,3 @@
variable "account_name" {}
variable "bucket" {}
variable "key" {}

View file

@ -1,3 +1,5 @@
provider "aws" { provider "aws" {
region = var.region region = var.aws_region
access_key = var.aws_access_key
secret_key = var.aws_secret_key
} }

View file

@ -0,0 +1,3 @@
variable "aws_access_key" {}
variable "aws_secret_key" {}
variable "aws_region" {}

View file

@ -1,29 +1,24 @@
terraform { terraform {
required_providers { required_providers {
exoscale = {
source = "exoscale/exoscale"
version = ">= 0.29.0"
}
hcloud = { hcloud = {
source = "hetznercloud/hcloud" source = "hetznercloud/hcloud"
version = ">= 1.23.0" version = ">= 1.41"
} }
aws = { aws = {
source = "hashicorp/aws" source = "hashicorp/aws"
version = "~> 3.0" version = "~> 5.6"
} }
hetznerdns = { hetznerdns = {
source = "timohirt/hetznerdns" source = "timohirt/hetznerdns"
version = ">= 1.1.0" version = ">= 2.2"
} }
digitalocean = { digitalocean = {
source = "digitalocean/digitalocean" source = "digitalocean/digitalocean"
version = "~> 2.0" version = "~> 2.28"
} }
} }
} }

View file

@ -42,6 +42,8 @@ def devops_config(overrides: dict) -> dict:
"do_bucket": "bucket", "do_bucket": "bucket",
"do_region": "region", "do_region": "region",
"hetzner_api_key": "hetzner_api_key", "hetzner_api_key": "hetzner_api_key",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_as_backend": True, "aws_as_backend": True,
"aws_bucket": "bucket", "aws_bucket": "bucket",
"aws_region": "region", "aws_region": "region",

View file

@ -12,6 +12,8 @@ def test_aws_creation():
{ {
"module": "module", "module": "module",
"stage": "test", "stage": "test",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_account_name": "aws_account_name", "aws_account_name": "aws_account_name",
} }
) )
@ -22,6 +24,8 @@ def test_aws_creation():
{ {
"module": "module", "module": "module",
"stage": "test", "stage": "test",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_as_backend": True, "aws_as_backend": True,
"aws_bucket": "bucket", "aws_bucket": "bucket",
"aws_bucket_kms_key_id": "aws_bucket_kms_key_id", "aws_bucket_kms_key_id": "aws_bucket_kms_key_id",
@ -59,7 +63,11 @@ def test_should_calculate_project_vars():
} }
) )
) )
assert {} == sut.project_vars() assert {
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_region": "region",
} == sut.project_vars()
sut = Aws( sut = Aws(
devops_config( devops_config(
@ -69,9 +77,11 @@ def test_should_calculate_project_vars():
) )
) )
assert { assert {
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_region": "region",
"account_name": "test", "account_name": "test",
"bucket": "bucket", "bucket": "bucket",
"key": "test/module", "key": "test/module",
"kms_key_id": "aws_bucket_kms_key_id", "kms_key_id": "aws_bucket_kms_key_id",
"region": "region",
} == sut.project_vars() } == sut.project_vars()

View file

@ -76,9 +76,9 @@ def test_should_calculate_project_vars():
"do_api_key": "api_key", "do_api_key": "api_key",
"do_spaces_access_id": "spaces_id", "do_spaces_access_id": "spaces_id",
"do_spaces_secret_key": "spaces_secret", "do_spaces_secret_key": "spaces_secret",
"region": "region",
"account_name": "test", "account_name": "test",
"endpoint": "endpoint", "endpoint": "endpoint",
"bucket": "bucket", "bucket": "bucket",
"key": "test/module", "key": "test/module",
"region": "region",
} == sut.project_vars() } == sut.project_vars()

View file

@ -87,6 +87,9 @@ def test_should_calculate_project_vars():
"do_spaces_access_id": "spaces_id", "do_spaces_access_id": "spaces_id",
"do_spaces_secret_key": "spaces_secret", "do_spaces_secret_key": "spaces_secret",
"hetzner_api_key": "hetzner_api_key", "hetzner_api_key": "hetzner_api_key",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_region": "region",
} == sut.project_vars() } == sut.project_vars()
@ -111,7 +114,7 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf", "terraform_build_vars.tf",
"provider_registry.tf", "provider_registry.tf",
"do_provider.tf", "do_provider.tf",
"do_mixin_vars.tf", "do_provider_vars.tf",
} == sut.resources_from_package() } == sut.resources_from_package()
sut = TerraformDomain( sut = TerraformDomain(
@ -127,9 +130,9 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf", "terraform_build_vars.tf",
"provider_registry.tf", "provider_registry.tf",
"do_provider.tf", "do_provider.tf",
"do_mixin_vars.tf", "do_provider_vars.tf",
"do_backend_properties_vars.tf", "do_backend_vars.tf",
"do_backend_with_properties.tf", "do_backend.tf",
} == sut.resources_from_package() } == sut.resources_from_package()
config = devops_config({"tf_provider_types": ["HETZNER"]}) config = devops_config({"tf_provider_types": ["HETZNER"]})
@ -139,7 +142,7 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf", "terraform_build_vars.tf",
"provider_registry.tf", "provider_registry.tf",
"hetzner_provider.tf", "hetzner_provider.tf",
"hetzner_mixin_vars.tf", "hetzner_provider_vars.tf",
} == sut.resources_from_package() } == sut.resources_from_package()
config = devops_config( config = devops_config(
@ -154,13 +157,14 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf", "terraform_build_vars.tf",
"provider_registry.tf", "provider_registry.tf",
"do_provider.tf", "do_provider.tf",
"do_mixin_vars.tf", "do_provider_vars.tf",
"provider_registry.tf", "provider_registry.tf",
"hetzner_provider.tf", "hetzner_provider.tf",
"hetzner_mixin_vars.tf", "hetzner_provider_vars.tf",
"aws_backend_with_properties.tf", "aws_backend.tf",
"aws_provider.tf", "aws_provider.tf",
"aws_backend_properties_vars.tf", "aws_provider_vars.tf",
"aws_backend_wkms_vars.tf",
"my.file", "my.file",
} == sut.resources_from_package() } == sut.resources_from_package()