meissa/dda-devops-build
6
1
Fork 0
Code Issues Pull requests Projects Releases 33 Packages Wiki Activity

Merge branch 'add-aws-backend' into 'main'

Browse source 
Add aws backend

See merge request domaindrivenarchitecture/dda-devops-build!14
This commit is contained in:
Michael Jerger 2023-07-06 07:29:47 +00:00
commit b11defa5ce
21 changed files with 206 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
build.py
View file

@ -33,7 +33,7 @@ default_task = "dev"
name = "ddadevops"
MODULE = "not-used"
PROJECT_ROOT_PATH = "."
version = "4.0.0-dev74"
version = "4.0.0-dev79"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]
@ -142,7 +142,6 @@ def tag_bump_and_push_release(project):
def build(project, release_type):
build = get_devops_build(project)
# TODO: release_type is here a string!
build.update_release_type(release_type)
test(project)
lint(project)

111
doc/CredentialsMappin.md Normal file
View file

@ -0,0 +1,111 @@
# CredentialsMapping
Credentials used for builds can be received from various places during build object initialization.
Precedence is
1. if there is a ENV, that is used
2. if there is a gopass_path only is given, the gopass password from path is used
3. if there is a gopass_path and gopass_field is given, the field from path is used.
```mermaid
classDiagram
class Credentials {
<<AggregateRoot>>
}
class CredentialMapping {
name
gopass_path
gopass_field
gopass_type()
name_for_input() - name used in context of build (according to python conventions converted to lower case snake case)
name_for_environment () - name used in context of ENV (according to ENV conventions converted to upper case snake case)
}
Credentials *-- "0..n" CredentialMapping: mappings[name]
Credentials *-- "0..n" CredentialMapping: default_mappings
```
## Input
| name | description | default |
| ------------ | -------------------------------------------------------- | ------- |
| gopass_path | path of secret in gopass | - |
| gopass_field | optional: if given, the field is read, the password else | - |
| name | name in context of build & ENV | - | |
## Example Usage
### build.py
```python
from os import environ
from pybuilder.core import task, init
from ddadevops import *
name = 'my-project'
MODULE = 'my-module'
PROJECT_ROOT_PATH = '..'
class MyBuild(DevopsTerraformBuild, C4kBuild, ProvsK3sBuild):
pass
@init
def initialize(project):
project.build_depends_on("ddadevops>=4.0.0")
stage = environ["STAGE"]
if stage == "test":
tmp_letsencrypt_endpoint = "staging"
tmp_jitsi_secrets_path = "server/jitsi-test"
elif stage == "prod":
tmp_letsencrypt_endpoint = "prod"
tmp_jitsi_secrets_path = "server/jitsi-prod"
c4k_config = {"issuer": tmp_letsencrypt_endpoint}
c4k_auth = {
"jvb-auth-password": gopass_field_from_path(
tmp_jitsi_secrets_path, "jvb-auth-password"
),
"jicofo-auth-password": gopass_field_from_path(
tmp_jitsi_secrets_path, "jvb-auth-password"
),
"jicofo-component-secret": gopass_field_from_path(
tmp_jitsi_secrets_path, "jicofo-component-secret"
),
}
config = {
"credentials_mapping": [
{
"gopass_path": environ.get("DIGITALOCEAN_TOKEN_KEY_PATH", None),
"name": "do_api_key",
},
{
"gopass_path": environ.get("HETZNER_API_KEY_PATH", None),
"name": "hetzner_api_key",
},
],
"name": name,
"module": MODULE,
"stage": stage,
"project_root_path": PROJECT_ROOT_PATH,
"build_types": ["TERRAFORM", "C4K", "K3S"],
"mixin_types": [],
"tf_provider_types": ["DIGITALOCEAN", "HETZNER"],
"tf_use_workspace": False,
"tf_terraform_semantic_version": "1.4.2",
"do_as_backend": True,
"do_bucket": "my-configuration",
"k3s_app_filename_to_provision": "out_jitsi.yaml",
"k3s_letsencrypt_endpoint": tmp_letsencrypt_endpoint,
"k3s_letsencrypt_email": "admin@your.domain",
"c4k_config": c4k_config,
"c4k_auth": c4k_auth,
}
build = MyBuild(project, config)
build.initialize_build_dir()
```

2
doc/DevopsTerraformBuildWithAwsProvider.md
View file

@ -4,6 +4,8 @@
| name | description | default |
| --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------- |
| aws_access_key | your aws access-key | |
| aws_secret_key | your aws secret-key | |
| aws_as_backend | you can use aws s3 as backend state storage | False |
| aws_region | in case of backend usage | "eu-central-1" |
| aws_bucket | in case of backend usage, the bucket your state is stored in. the url is S3://{aws_bucket}/{aws_bucket_key}/{aws_account_name} | |

19
doc/architecture/Domain.md
View file

@ -50,7 +50,7 @@ classDiagram
tf_terraform_semantic_version
}
class Digitalocean {
class ProviderDigitalocean {
do_api_key
do_spaces_access_key
do_spaces_secret_key
@ -62,12 +62,19 @@ classDiagram
do_region
}
class Hetzner {
class ProviderHetzner {
hetzner_api_key
}
class Aws {
class ProviderAws {
aws_access_key
aws_secret_key
aws_bucket
aws_bucket_kms_key_id
aws_account_name
aws_bucket_key
aws_as_backend
aws_region
}
class DnsRecord {
@ -118,9 +125,9 @@ classDiagram
Devops *-- "0..1" ProvsK3s: specialized_builds
Devops *-- "0..1" TerraformDomain: specialized_builds
Devops *-- "0..1" Release: mixins
TerraformDomain *-- "0..1" Digitalocean: providers
TerraformDomain *-- "0..1" Hetzner: providers
TerraformDomain *-- "0..1" Aws: providers
TerraformDomain *-- "0..1" ProviderDigitalocean: providers
TerraformDomain *-- "0..1" ProviderHetzner: providers
TerraformDomain *-- "0..1" ProviderAws: providers
Release o-- "0..1" BuildFile: primary_build_file
Release o-- "0..n" BuildFile: secondary_build_files
BuildFile *-- "1" Version

38
src/main/python/ddadevops/domain/provider_aws.py
View file

@ -9,6 +9,8 @@ class Aws(Validateable, CredentialMappingDefault):
):
self.stage = inp.get("stage")
self.module = inp.get("module")
self.aws_access_key = inp.get("aws_access_key")
self.aws_secret_key = inp.get("aws_secret_key")
self.aws_bucket = inp.get("aws_bucket")
self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id")
self.aws_account_name = inp.get("aws_account_name", self.stage)
@ -20,12 +22,13 @@ class Aws(Validateable, CredentialMappingDefault):
result = []
result += self.__validate_is_not_empty__("stage")
result += self.__validate_is_not_empty__("module")
result += self.__validate_is_not_empty__("aws_access_key")
result += self.__validate_is_not_empty__("aws_secret_key")
result += self.__validate_is_not_empty__("aws_account_name")
result += self.__validate_is_not_empty__("aws_as_backend")
if self.aws_as_backend:
result += self.__validate_is_not_empty__("aws_bucket")
result += self.__validate_is_not_empty__("aws_bucket_key")
result += self.__validate_is_not_empty__("aws_bucket_kms_key_id")
result += self.__validate_is_not_empty__("aws_region")
return result
@ -42,25 +45,44 @@ class Aws(Validateable, CredentialMappingDefault):
return result
def resources_from_package(self) -> Set[str]:
result = {"provider_registry.tf", "aws_provider.tf"}
result = {"provider_registry.tf", "aws_provider.tf", "aws_provider_vars.tf"}
if self.aws_as_backend:
result.update(
{"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"}
)
if self.aws_bucket_kms_key_id:
result.update(
{
"aws_backend_wkms_vars.tf",
"aws_backend.tf",
}
)
else:
result.update(
{
"aws_backend_wokms_vars.tf",
"aws_backend.tf",
}
)
return result
def project_vars(self):
result = {}
result = {
"aws_access_key": self.aws_access_key,
"aws_secret_key": self.aws_secret_key,
"aws_region": self.aws_region,
}
if self.aws_as_backend:
result.update(
{
"account_name": self.aws_account_name,
"bucket": self.aws_bucket,
"key": self.__bucket_key__(),
"kms_key_id": self.aws_bucket_kms_key_id,
"region": self.aws_region,
}
)
if self.aws_bucket_kms_key_id:
result.update(
{
"kms_key_id": self.aws_bucket_kms_key_id,
}
)
return result
def is_local_state(self):

4
src/main/python/ddadevops/domain/provider_digitalocean.py
View file

@ -49,10 +49,10 @@ class Digitalocean(Validateable, CredentialMappingDefault):
return result
def resources_from_package(self) -> Set[str]:
result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"}
result = {"provider_registry.tf", "do_provider.tf", "do_provider_vars.tf"}
if self.do_as_backend:
result.update(
{"do_backend_properties_vars.tf", "do_backend_with_properties.tf"}
{"do_backend_vars.tf", "do_backend.tf"}
)
return result

6
src/main/python/ddadevops/domain/provider_hetzner.py
View file

@ -18,7 +18,11 @@ class Hetzner(Validateable, CredentialMappingDefault):
return {}
def resources_from_package(self) -> Set[str]:
return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"}
return {
"provider_registry.tf",
"hetzner_provider.tf",
"hetzner_provider_vars.tf",
}
def project_vars(self):
return {"hetzner_api_key": self.hetzner_api_key}

0
src/main/resources/terraform/aws_backend_with_properties.tf → src/main/resources/terraform/aws_backend.tf
View file

3
src/main/resources/terraform/aws_backend_properties_vars.tf → src/main/resources/terraform/aws_backend_wkms_vars.tf
View file

@ -1,5 +1,4 @@
variable "account_name" {}
variable "bucket" {}
variable "key" {}
variable "kms_key_id" {}
variable "region" {}
variable "kms_key_id" {}

3
src/main/resources/terraform/aws_backend_wokms_vars.tf Normal file
View file

@ -0,0 +1,3 @@
variable "account_name" {}
variable "bucket" {}
variable "key" {}

4
src/main/resources/terraform/aws_provider.tf
View file

@ -1,3 +1,5 @@
provider "aws" {
region = var.region
region = var.aws_region
access_key = var.aws_access_key
secret_key = var.aws_secret_key
}

3
src/main/resources/terraform/aws_provider_vars.tf Normal file
View file

@ -0,0 +1,3 @@
variable "aws_access_key" {}
variable "aws_secret_key" {}
variable "aws_region" {}

0
src/main/resources/terraform/do_backend_with_properties.tf → src/main/resources/terraform/do_backend.tf
View file

0
src/main/resources/terraform/do_backend_properties_vars.tf → src/main/resources/terraform/do_backend_vars.tf
View file

0
src/main/resources/terraform/do_mixin_vars.tf → src/main/resources/terraform/do_provider_vars.tf
View file

0
src/main/resources/terraform/hetzner_mixin_vars.tf → src/main/resources/terraform/hetzner_provider_vars.tf
View file

13
src/main/resources/terraform/provider_registry.tf
View file

@ -1,29 +1,24 @@
terraform {
required_providers {
exoscale = {
source = "exoscale/exoscale"
version = ">= 0.29.0"
}
hcloud = {
source = "hetznercloud/hcloud"
version = ">= 1.23.0"
version = ">= 1.41"
}
aws = {
source = "hashicorp/aws"
version = "~> 3.0"
version = "~> 5.6"
}
hetznerdns = {
source = "timohirt/hetznerdns"
version = ">= 1.1.0"
version = ">= 2.2"
}
digitalocean = {
source = "digitalocean/digitalocean"
version = "~> 2.0"
version = "~> 2.28"
}
}
}

2
src/test/python/domain/helper.py
View file

@ -42,6 +42,8 @@ def devops_config(overrides: dict) -> dict:
"do_bucket": "bucket",
"do_region": "region",
"hetzner_api_key": "hetzner_api_key",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_as_backend": True,
"aws_bucket": "bucket",
"aws_region": "region",

14
src/test/python/domain/test_provider_aws.py
View file

@ -12,6 +12,8 @@ def test_aws_creation():
{
"module": "module",
"stage": "test",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_account_name": "aws_account_name",
}
)
@ -22,6 +24,8 @@ def test_aws_creation():
{
"module": "module",
"stage": "test",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_as_backend": True,
"aws_bucket": "bucket",
"aws_bucket_kms_key_id": "aws_bucket_kms_key_id",
@ -59,7 +63,11 @@ def test_should_calculate_project_vars():
}
)
)
assert {} == sut.project_vars()
assert {
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_region": "region",
} == sut.project_vars()
sut = Aws(
devops_config(
@ -69,9 +77,11 @@ def test_should_calculate_project_vars():
)
)
assert {
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_region": "region",
"account_name": "test",
"bucket": "bucket",
"key": "test/module",
"kms_key_id": "aws_bucket_kms_key_id",
"region": "region",
} == sut.project_vars()

2
src/test/python/domain/test_provider_digitalocean.py
View file

@ -76,9 +76,9 @@ def test_should_calculate_project_vars():
"do_api_key": "api_key",
"do_spaces_access_id": "spaces_id",
"do_spaces_secret_key": "spaces_secret",
"region": "region",
"account_name": "test",
"endpoint": "endpoint",
"bucket": "bucket",
"key": "test/module",
"region": "region",
} == sut.project_vars()

22
src/test/python/domain/test_terraform.py
View file

@ -87,6 +87,9 @@ def test_should_calculate_project_vars():
"do_spaces_access_id": "spaces_id",
"do_spaces_secret_key": "spaces_secret",
"hetzner_api_key": "hetzner_api_key",
"aws_access_key": "aws_access_key",
"aws_secret_key": "aws_secret_key",
"aws_region": "region",
} == sut.project_vars()
@ -111,7 +114,7 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf",
"provider_registry.tf",
"do_provider.tf",
"do_mixin_vars.tf",
"do_provider_vars.tf",
} == sut.resources_from_package()
sut = TerraformDomain(
@ -127,9 +130,9 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf",
"provider_registry.tf",
"do_provider.tf",
"do_mixin_vars.tf",
"do_backend_properties_vars.tf",
"do_backend_with_properties.tf",
"do_provider_vars.tf",
"do_backend_vars.tf",
"do_backend.tf",
} == sut.resources_from_package()
config = devops_config({"tf_provider_types": ["HETZNER"]})
@ -139,7 +142,7 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf",
"provider_registry.tf",
"hetzner_provider.tf",
"hetzner_mixin_vars.tf",
"hetzner_provider_vars.tf",
} == sut.resources_from_package()
config = devops_config(
@ -154,13 +157,14 @@ def test_should_calculate_resources_from_package():
"terraform_build_vars.tf",
"provider_registry.tf",
"do_provider.tf",
"do_mixin_vars.tf",
"do_provider_vars.tf",
"provider_registry.tf",
"hetzner_provider.tf",
"hetzner_mixin_vars.tf",
"aws_backend_with_properties.tf",
"hetzner_provider_vars.tf",
"aws_backend.tf",
"aws_provider.tf",
"aws_backend_properties_vars.tf",
"aws_provider_vars.tf",
"aws_backend_wkms_vars.tf",
"my.file",
} == sut.resources_from_package()