meissa/dda-devops-build
6
1
Fork 0
Code Issues Pull requests Projects Releases 33 Packages Wiki Activity

introduce aws_provider

Browse source
This commit is contained in:
Michael Jerger 2023-05-26 16:56:02 +02:00
parent da872af76e
commit 0b577597e8
14 changed files with 209 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build.py
View file

@ -28,7 +28,7 @@ use_plugin("python.distutils")
default_task = "publish"
name = "ddadevops"
version = "4.0.0-dev59"
version = "4.0.0-dev61"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]

5
doc/architecture/Domain.md
View file

@ -65,6 +65,10 @@ classDiagram
class Hetzner {
hetzner_api_key
}
class Aws {
aws_account_name
}
class DnsRecord {
fqdn
@ -116,6 +120,7 @@ classDiagram
Devops *-- "0..1" Release: mixins
TerraformDomain *-- "0..1" Digitalocean: providers
TerraformDomain *-- "0..1" Hetzner: providers
TerraformDomain *-- "0..1" Aws: providers
Release o-- "0..1" BuildFile: primary_build_file
Release o-- "0..n" BuildFile: secondary_build_files
BuildFile *-- "1" Version

4
src/main/python/ddadevops/__init__.py
View file

@ -4,10 +4,8 @@ terraform, dda-pallet, aws & hetzner-cloud.
"""
from .python_util import execute
from .provs_k3s_build import ProvsK3sBuild
from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config
from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config
#from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config
from .c4k_build import C4kBuild
from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config
from .devops_image_build import DevopsImageBuild

52
src/main/python/ddadevops/aws_backend_properties_mixin.py
View file

@ -1,52 +0,0 @@
from dda_python_terraform import Terraform
from .devops_terraform_build import DevopsTerraformBuild
def add_aws_backend_properties_mixin_config(config, account_name):
config.update({"AwsBackendPropertiesMixin": {"account_name": account_name}})
return config
class AwsBackendPropertiesMixin(DevopsTerraformBuild):
def __init__(self, project, config):
super().__init__(project, config)
aws_mixin_config = config["AwsBackendPropertiesMixin"]
self.account_name = aws_mixin_config["account_name"]
self.backend_config = (
"backend." + self.account_name + "." + self.stage + ".properties"
)
self.additional_tfvar_files.append(self.backend_config)
def project_vars(self):
ret = super().project_vars()
ret.update({"account_name": self.account_name})
return ret
def copy_build_resources_from_package(self):
super().copy_build_resources_from_package()
self.copy_build_resource_file_from_package("provider_registry.tf")
self.copy_build_resource_file_from_package("aws_provider.tf")
self.copy_build_resource_file_from_package("aws_backend_properties_vars.tf")
self.copy_build_resource_file_from_package("aws_backend_with_properties.tf")
def copy_local_state(self):
pass
def rescue_local_state(self):
pass
def init_client(self):
terraform = Terraform(
working_dir=self.build_path(),
terraform_semantic_version=self.terraform_semantic_version,
)
terraform.init(backend_config=self.backend_config)
self.print_terraform_command(terraform)
if self.use_workspace:
try:
terraform.workspace("select", self.stage)
self.print_terraform_command(terraform)
except:
terraform.workspace("new", self.stage)
self.print_terraform_command(terraform)
return terraform

1
src/main/python/ddadevops/domain/__init__.py
View file

@ -5,6 +5,7 @@ from .c4k import C4k
from .terraform import TerraformDomain
from .provider_digitalocean import Digitalocean
from .provider_hetzner import Hetzner
from .provider_aws import Aws
from .provs_k3s import K3s
from .release import Release
from .credentials import Credentials, CredentialMapping, GopassType

1
src/main/python/ddadevops/domain/common.py
View file

@ -16,6 +16,7 @@ class BuildType(Enum):
class ProviderType(Enum):
DIGITALOCEAN = 0
HETZNER = 1
AWS = 2
class MixinType(Enum):

80
src/main/python/ddadevops/domain/provider_aws.py Normal file
View file

@ -0,0 +1,80 @@
from typing import List, Dict, Set, Any
from .common import Validateable, CredentialMappingDefault
class Aws(Validateable, CredentialMappingDefault):
def __init__(
self,
inp: dict,
):
self.stage = inp.get("stage")
self.module = inp.get("module")
self.aws_bucket = inp.get("aws_bucket")
self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id")
self.aws_account_name = inp.get("aws_account_name", self.stage)
self.aws_bucket_key = inp.get("aws_bucket_key", self.module)
self.aws_as_backend = inp.get("aws_as_backend", False)
self.aws_region = inp.get("aws_region", "eu-central-1")
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("stage")
result += self.__validate_is_not_empty__("module")
result += self.__validate_is_not_empty__("aws_account_name")
result += self.__validate_is_not_empty__("aws_as_backend")
if self.aws_as_backend:
result += self.__validate_is_not_empty__("aws_bucket")
result += self.__validate_is_not_empty__("aws_bucket_key")
result += self.__validate_is_not_empty__("aws_bucket_kms_key_id")
result += self.__validate_is_not_empty__("aws_region")
return result
def backend_config(self) -> Dict[str, Any]:
result = {}
if self.aws_as_backend:
result = {
"bucket": self.aws_bucket,
"key": self.__bucket_key__(),
"region": self.aws_region,
}
if self.aws_bucket_kms_key_id:
result["kms_key_id"] = self.aws_bucket_kms_key_id
return result
def resources_from_package(self) -> Set[str]:
result = {"provider_registry.tf", "aws_provider.tf"}
if self.aws_as_backend:
result.update(
{"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"}
)
return result
def project_vars(self):
result = {}
if self.aws_as_backend:
result.update(
{
"account_name": self.aws_account_name,
"bucket": self.aws_bucket,
"key": self.__bucket_key__(),
"kms_key_id": self.aws_bucket_kms_key_id,
"region": self.aws_region,
}
)
return result
def is_local_state(self):
return not self.aws_as_backend
def __bucket_key__(self):
result = ""
if self.aws_as_backend:
if self.aws_account_name and self.aws_bucket_key:
result = f"{self.aws_account_name}/{self.aws_bucket_key}"
else:
result = f"{self.stage}/{self.module}"
return result
@classmethod
def get_mapping_default(cls) -> List[Dict[str, str]]:
return []

26
src/main/python/ddadevops/domain/provider_digitalocean.py
View file

@ -35,6 +35,19 @@ class Digitalocean(Validateable, CredentialMappingDefault):
result += self.__validate_is_not_empty__("do_region")
return result
def backend_config(self) -> Dict[str, Any]:
result = {}
if self.do_as_backend:
result = {
"access_key": self.do_spaces_access_id,
"secret_key": self.do_spaces_secret_key,
"endpoint": self.do_endpoint,
"bucket": self.do_bucket,
"key": self.__bucket_key__(),
"region": self.do_region,
}
return result
def resources_from_package(self) -> Set[str]:
result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"}
if self.do_as_backend:
@ -61,19 +74,6 @@ class Digitalocean(Validateable, CredentialMappingDefault):
)
return result
def backend_config(self) -> Dict[str, Any]:
result = {}
if self.do_as_backend:
result = {
"access_key": self.do_spaces_access_id,
"secret_key": self.do_spaces_secret_key,
"endpoint": self.do_endpoint,
"bucket": self.do_bucket,
"key": self.__bucket_key__(),
"region": self.do_region,
}
return result
def is_local_state(self):
return not self.do_as_backend

3
src/main/python/ddadevops/domain/terraform.py
View file

@ -7,6 +7,7 @@ from .common import (
)
from .provider_digitalocean import Digitalocean
from .provider_hetzner import Hetzner
from .provider_aws import Aws
class TerraformDomain(Validateable):
@ -39,6 +40,8 @@ class TerraformDomain(Validateable):
self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp)
if ProviderType.HETZNER in provider_types:
self.providers[ProviderType.HETZNER] = Hetzner(inp)
if ProviderType.AWS in provider_types:
self.providers[ProviderType.AWS] = Aws(inp)
def validate(self) -> List[str]:
result = []

4
src/main/resources/terraform/aws_backend_properties_vars.tf
View file

@ -1,5 +1,5 @@
variable "account_name" {}
variable "bucket" {}
variable "key" {}
variable "kms_key_id" {}
variable "region" {}
variable "account_name" {}
variable "region" {}

4
src/main/resources/terraform/do_backend_properties_vars.tf
View file

@ -1,5 +1,5 @@
variable "account_name" {}
variable "endpoint" {}
variable "bucket" {}
variable "key" {}
variable "region" {}
variable "account_name" {}
variable "region" {}

6
src/test/python/domain/helper.py
View file

@ -24,7 +24,7 @@ def devops_config(overrides: dict) -> dict:
"k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint",
"k3s_enable_echo": "false",
"k3s_app_filename_to_provision": "k3s_app.yaml",
"tf_provider_types": ["DIGITALOCEAN", "HETZNER"],
"tf_provider_types": ["DIGITALOCEAN", "HETZNER", "AWS"],
"tf_additional_vars": [],
"tf_output_json_name": "the_out.json",
"tf_use_workspace": None,
@ -45,6 +45,10 @@ def devops_config(overrides: dict) -> dict:
"do_bucket": "bucket",
"do_region": "region",
"hetzner_api_key": "hetzner_api_key",
"aws_as_backend": True,
"aws_bucket": "bucket",
"aws_region": "region",
"aws_bucket_kms_key_id": "aws_bucket_kms_key_id",
"release_type": "NONE",
"release_main_branch": "main",
"release_current_branch": "my_feature",

77
src/test/python/domain/test_provider_aws.py Normal file
View file

@ -0,0 +1,77 @@
from pybuilder.core import Project
from pathlib import Path
from src.main.python.ddadevops.domain import (
BuildType,
Aws,
)
from .helper import devops_config
def test_aws_creation():
sut = Aws(
{
"module": "module",
"stage": "test",
"aws_account_name": "aws_account_name",
}
)
assert sut is not None
assert sut.is_valid()
sut = Aws(
{
"module": "module",
"stage": "test",
"aws_as_backend": True,
"aws_bucket": "bucket",
"aws_bucket_kms_key_id": "aws_bucket_kms_key_id",
}
)
assert sut is not None
assert sut.is_valid()
def test_should_calculate_backend_config():
sut = Aws(
devops_config(
{
"module": "dns_aws",
"stage": "prod",
"aws_bucket": "meissa-configuration",
"aws_bucket_kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration",
"aws_region": "eu-central-1",
}
)
)
assert {
"bucket": "meissa-configuration",
"key": "prod/dns_aws",
"kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration",
"region": "eu-central-1",
} == sut.backend_config()
def test_should_calculate_project_vars():
sut = Aws(
devops_config(
{
"aws_as_backend": False,
}
)
)
assert {} == sut.project_vars()
sut = Aws(
devops_config(
{
"aws_as_backend": True,
}
)
)
assert {
"account_name": "test",
"bucket": "bucket",
"key": "test/module",
"kms_key_id": "aws_bucket_kms_key_id",
"region": "region",
} == sut.project_vars()

19
src/test/python/domain/test_terraform.py
View file

@ -16,6 +16,7 @@ def test_creation():
assert sut
assert sut.providers[ProviderType.DIGITALOCEAN]
assert sut.providers[ProviderType.HETZNER]
assert sut.providers[ProviderType.AWS]
def test_should_calculate_output_json_name():
@ -42,6 +43,14 @@ def test_should_validate():
sut = TerraformDomain(config)
assert not sut.is_valid()
config = devops_config(
{
"aws_account_name": "",
}
)
sut = TerraformDomain(config)
assert not sut.is_valid()
def test_should_calculate_terraform_build_commons_path():
config = devops_config({})
@ -64,7 +73,12 @@ def test_should_calculate_project_vars():
sut = TerraformDomain(config)
assert {"module": "module", "stage": "test"} == sut.project_vars()
config = devops_config({"do_as_backend": False,})
config = devops_config(
{
"do_as_backend": False,
"aws_as_backend": False,
}
)
sut = TerraformDomain(config)
assert {
"module": "module",
@ -144,6 +158,9 @@ def test_should_calculate_resources_from_package():
"provider_registry.tf",
"hetzner_provider.tf",
"hetzner_mixin_vars.tf",
"aws_backend_with_properties.tf",
"aws_provider.tf",
"aws_backend_properties_vars.tf",
"my.file",
} == sut.resources_from_package()