meissa/dda-devops-build
6
1
Fork 0
Code Issues Pull requests Projects Releases 33 Packages Wiki Activity

introduce terraform

Browse source
This commit is contained in:
Michael Jerger 2023-05-24 08:34:43 +02:00
parent 483d98a3d5
commit 15335ae73f
8 changed files with 211 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
doc/architecture/Domain.md
View file

@ -37,6 +37,18 @@ classDiagram
k3s_app_filename_to_provision k3s_app_filename_to_provision
} }
class Terraform {
tf_additional_vars
tf_output_json_name
tf_use_workspace
tf_use_package_common_files
tf_build_commons_path
tf_commons_dir_name
tf_debug_print_terraform_command
tf_additional_tfvar_files
tf_terraform_semantic_version
}
class DnsRecord { class DnsRecord {
fqdn fqdn
ipv4 ipv4
@ -82,6 +94,7 @@ classDiagram
Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" Image: spcialized_builds
Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds
Devops *-- "0..1" ProvsK3s: spcialized_builds Devops *-- "0..1" ProvsK3s: spcialized_builds
Devops *-- "0..1" Terraform: spcialized_builds
Devops *-- "0..1" Release: mixins Devops *-- "0..1" Release: mixins
Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..1" BuildFile: primary_build_file
Release o-- "0..n" BuildFile: secondary_build_files Release o-- "0..n" BuildFile: secondary_build_files

226
src/main/python/ddadevops/devops_terraform_build.py
View file

@ -9,94 +9,87 @@ from .python_util import filter_none
from .devops_build import DevopsBuild, create_devops_build_config from .devops_build import DevopsBuild, create_devops_build_config
def create_devops_terraform_build_config(
stage,
def create_devops_terraform_build_config(stage, project_root_path,
project_root_path, module,
module, additional_vars,
additional_vars, build_dir_name="target",
build_dir_name='target', output_json_name=None,
output_json_name=None, use_workspace=True,
use_workspace=True, use_package_common_files=True,
use_package_common_files=True, build_commons_path=None,
build_commons_path=None, terraform_build_commons_dir_name="terraform",
terraform_build_commons_dir_name='terraform', debug_print_terraform_command=False,
debug_print_terraform_command=False, additional_tfvar_files=None,
additional_tfvar_files=None, terraform_semantic_version="1.0.8",
terraform_semantic_version="1.0.8"): ):
if not output_json_name: if not output_json_name:
output_json_name = 'out_' + module + '.json' output_json_name = "out_" + module + ".json"
if not additional_tfvar_files: if not additional_tfvar_files:
additional_tfvar_files = [] additional_tfvar_files = []
ret = create_devops_build_config( ret = create_devops_build_config(stage, project_root_path, module, build_dir_name)
stage, project_root_path, module, build_dir_name) ret.update(
ret.update({'additional_vars': additional_vars, {
'output_json_name': output_json_name, "additional_vars": additional_vars,
'use_workspace': use_workspace, "output_json_name": output_json_name,
'use_package_common_files': use_package_common_files, "use_workspace": use_workspace,
'build_commons_path': build_commons_path, "use_package_common_files": use_package_common_files,
'terraform_build_commons_dir_name': terraform_build_commons_dir_name, "build_commons_path": build_commons_path,
'debug_print_terraform_command': debug_print_terraform_command, "terraform_build_commons_dir_name": terraform_build_commons_dir_name,
'additional_tfvar_files': additional_tfvar_files, "debug_print_terraform_command": debug_print_terraform_command,
'terraform_semantic_version': terraform_semantic_version}) "additional_tfvar_files": additional_tfvar_files,
"terraform_semantic_version": terraform_semantic_version,
}
)
return ret return ret
class DevopsTerraformBuild(DevopsBuild):
class DevopsTerraformBuild(DevopsBuild):
def __init__(self, project, config): def __init__(self, project, config):
inp = config.copy() inp = config.copy()
inp["name"]=project.name inp["name"] = project.name
inp["module"]=config.get("module") inp["module"] = config.get("module")
inp["stage"]=config.get("stage") inp["stage"] = config.get("stage")
inp["project_root_path"]=config.get("project_root_path") inp["project_root_path"] = config.get("project_root_path")
inp["build_types"]=config.get("build_types", []) inp["build_types"] = config.get("build_types", [])
inp["mixin_types"]=config.get("mixin_types", []) inp["mixin_types"] = config.get("mixin_types", [])
super().__init__(project, inp) super().__init__(project, inp)
project.build_depends_on('dda-python-terraform') project.build_depends_on("dda-python-terraform")
self.additional_vars = config['additional_vars']
self.output_json_name = config['output_json_name']
self.use_workspace = config['use_workspace']
self.use_package_common_files = config['use_package_common_files']
self.build_commons_path = config['build_commons_path']
self.terraform_build_commons_dir_name = config['terraform_build_commons_dir_name']
self.debug_print_terraform_command = config['debug_print_terraform_command']
self.additional_tfvar_files = config['additional_tfvar_files']
self.terraform_semantic_version = config['terraform_semantic_version']
self.stage = config["stage"]
self.module = config["module"]
def terraform_build_commons_path(self): def terraform_build_commons_path(self):
mylist = [self.build_commons_path, mylist = [self.build_commons_path, self.terraform_build_commons_dir_name]
self.terraform_build_commons_dir_name] return "/".join(filter_none(mylist)) + "/"
return '/'.join(filter_none(mylist)) + '/'
def project_vars(self): def project_vars(self):
ret = {'stage': self.stage} ret = {"stage": self.stage}
if self.module: if self.module:
ret['module'] = self.module ret["module"] = self.module
if self.additional_vars: if self.additional_vars:
ret.update(self.additional_vars) ret.update(self.additional_vars)
return ret return ret
def copy_build_resource_file_from_package(self, name): def copy_build_resource_file_from_package(self, name):
my_data = resource_string( my_data = resource_string(__name__, "src/main/resources/terraform/" + name)
__name__, "src/main/resources/terraform/" + name) with open(self.build_path() + "/" + name, "w", encoding="utf-8") as output_file:
with open(self.build_path() + '/' + name, "w", encoding="utf-8") as output_file:
output_file.write(my_data.decode(sys.stdout.encoding)) output_file.write(my_data.decode(sys.stdout.encoding))
def copy_build_resources_from_package(self): def copy_build_resources_from_package(self):
self.copy_build_resource_file_from_package('versions.tf') self.copy_build_resource_file_from_package("versions.tf")
self.copy_build_resource_file_from_package('terraform_build_vars.tf') self.copy_build_resource_file_from_package("terraform_build_vars.tf")
def copy_build_resources_from_dir(self): def copy_build_resources_from_dir(self):
run('cp -f ' + self.terraform_build_commons_path() + run(
'* ' + self.build_path(), shell=True, check=False) "cp -f " + self.terraform_build_commons_path() + "* " + self.build_path(),
shell=True,
check=False,
)
def copy_local_state(self): def copy_local_state(self):
run('cp terraform.tfstate ' + self.build_path(), shell=True, check=False) run("cp terraform.tfstate " + self.build_path(), shell=True, check=False)
def rescue_local_state(self): def rescue_local_state(self):
run('cp ' + self.build_path() + '/terraform.tfstate .', shell=True, check=False) run("cp " + self.build_path() + "/terraform.tfstate .", shell=True, check=False)
def initialize_build_dir(self): def initialize_build_dir(self):
super().initialize_build_dir() super().initialize_build_dir()
@ -105,43 +98,54 @@ class DevopsTerraformBuild(DevopsBuild):
else: else:
self.copy_build_resources_from_dir() self.copy_build_resources_from_dir()
self.copy_local_state() self.copy_local_state()
run('cp *.tf ' + self.build_path(), shell=True, check=False) run("cp *.tf " + self.build_path(), shell=True, check=False)
run('cp *.properties ' + self.build_path(), shell=True, check=False) run("cp *.properties " + self.build_path(), shell=True, check=False)
run('cp *.tfvars ' + self.build_path(), shell=True, check=False) run("cp *.tfvars " + self.build_path(), shell=True, check=False)
run('cp -r scripts ' + self.build_path(), shell=True, check=False) run("cp -r scripts " + self.build_path(), shell=True, check=False)
def post_build(self): def post_build(self):
self.rescue_local_state() self.rescue_local_state()
def init_client(self): def init_client(self):
terraform = Terraform(working_dir=self.build_path(), terraform_semantic_version=self.terraform_semantic_version) terraform = Terraform(
working_dir=self.build_path(),
terraform_semantic_version=self.terraform_semantic_version,
)
terraform.init() terraform.init()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if self.use_workspace: if self.use_workspace:
try: try:
terraform.workspace('select', self.stage) terraform.workspace("select", self.stage)
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
except: except:
terraform.workspace('new', self.stage) terraform.workspace("new", self.stage)
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
return terraform return terraform
def write_output(self, terraform): def write_output(self, terraform):
result = terraform.output(json=IsFlagged) result = terraform.output(json=IsFlagged)
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
with open(self.build_path() + self.output_json_name, "w", encoding="utf-8") as output_file: with open(
self.build_path() + self.output_json_name, "w", encoding="utf-8"
) as output_file:
output_file.write(dumps(result)) output_file.write(dumps(result))
chmod(self.build_path() + self.output_json_name, 0o600) chmod(self.build_path() + self.output_json_name, 0o600)
def read_output_json(self): def read_output_json(self):
with open(self.build_path() + self.output_json_name, 'r', encoding="utf-8") as file: with open(
self.build_path() + self.output_json_name, "r", encoding="utf-8"
) as file:
return load(file) return load(file)
def plan(self): def plan(self):
terraform = self.init_client() terraform = self.init_client()
return_code, _, stderr = terraform.plan(detailed_exitcode=None, capture_output=False, raise_on_error=False, return_code, _, stderr = terraform.plan(
var=self.project_vars(), detailed_exitcode=None,
var_file=self.additional_tfvar_files) capture_output=False,
raise_on_error=False,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
@ -149,9 +153,13 @@ class DevopsTerraformBuild(DevopsBuild):
def plan_fail_on_diff(self): def plan_fail_on_diff(self):
terraform = self.init_client() terraform = self.init_client()
return_code, _, stderr = terraform.plan(detailed_exitcode=IsFlagged, capture_output=False, raise_on_error=False, return_code, _, stderr = terraform.plan(
var=self.project_vars(), detailed_exitcode=IsFlagged,
var_file=self.additional_tfvar_files) capture_output=False,
raise_on_error=False,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code not in (0, 2): if return_code not in (0, 2):
@ -166,15 +174,21 @@ class DevopsTerraformBuild(DevopsBuild):
else: else:
auto_approve_flag = None auto_approve_flag = None
if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"):
return_code, _, stderr = terraform.apply(capture_output=False, raise_on_error=True, return_code, _, stderr = terraform.apply(
auto_approve=auto_approve_flag, capture_output=False,
var=self.project_vars(), raise_on_error=True,
var_file=self.additional_tfvar_files) auto_approve=auto_approve_flag,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
else: else:
return_code, _, stderr = terraform.apply(capture_output=False, raise_on_error=True, return_code, _, stderr = terraform.apply(
skip_plan=auto_approve, capture_output=False,
var=self.project_vars(), raise_on_error=True,
var_file=self.additional_tfvar_files) skip_plan=auto_approve,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.write_output(terraform) self.write_output(terraform)
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
@ -184,8 +198,8 @@ class DevopsTerraformBuild(DevopsBuild):
def refresh(self): def refresh(self):
terraform = self.init_client() terraform = self.init_client()
return_code, _, stderr = terraform.refresh( return_code, _, stderr = terraform.refresh(
var=self.project_vars(), var=self.project_vars(), var_file=self.additional_tfvar_files
var_file=self.additional_tfvar_files) )
self.write_output(terraform) self.write_output(terraform)
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
@ -199,26 +213,40 @@ class DevopsTerraformBuild(DevopsBuild):
else: else:
auto_approve_flag = None auto_approve_flag = None
if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"):
return_code, _, stderr = terraform.destroy(capture_output=False, raise_on_error=True, return_code, _, stderr = terraform.destroy(
auto_approve=auto_approve_flag, capture_output=False,
var=self.project_vars(), raise_on_error=True,
var_file=self.additional_tfvar_files) auto_approve=auto_approve_flag,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
else: else:
return_code, _, stderr = terraform.destroy(capture_output=False, raise_on_error=True, return_code, _, stderr = terraform.destroy(
force=auto_approve_flag, capture_output=False,
var=self.project_vars(), raise_on_error=True,
var_file=self.additional_tfvar_files) force=auto_approve_flag,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr) raise RuntimeError(return_code, "terraform error:", stderr)
def tf_import(self, tf_import_name, tf_import_resource,): def tf_import(
self,
tf_import_name,
tf_import_resource,
):
terraform = self.init_client() terraform = self.init_client()
return_code, _, stderr = terraform.import_cmd(tf_import_name, tf_import_resource, return_code, _, stderr = terraform.import_cmd(
capture_output=False, raise_on_error=True, tf_import_name,
var=self.project_vars(), tf_import_resource,
var_file=self.additional_tfvar_files) capture_output=False,
raise_on_error=True,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
@ -226,5 +254,5 @@ class DevopsTerraformBuild(DevopsBuild):
def print_terraform_command(self, terraform): def print_terraform_command(self, terraform):
if self.debug_print_terraform_command: if self.debug_print_terraform_command:
output = 'cd ' + self.build_path() + ' && ' + terraform.latest_cmd() output = "cd " + self.build_path() + " && " + terraform.latest_cmd()
print(output) print(output)

1
src/main/python/ddadevops/domain/__init__.py
View file

@ -2,6 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea
from .devops_factory import DevopsFactory from .devops_factory import DevopsFactory
from .image import Image from .image import Image
from .c4k import C4k from .c4k import C4k
from .terraform import Terraform
from .provs_k3s import K3s from .provs_k3s import K3s
from .release import Release from .release import Release
from .credentials import Credentials, CredentialMapping, GopassType from .credentials import Credentials, CredentialMapping, GopassType

1
src/main/python/ddadevops/domain/common.py
View file

@ -10,6 +10,7 @@ class BuildType(Enum):
IMAGE = 0 IMAGE = 0
C4K = 1 C4K = 1
K3S = 2 K3S = 2
TERRAFORM = 3
class MixinType(Enum): class MixinType(Enum):

3
src/main/python/ddadevops/domain/devops_factory.py
View file

@ -3,6 +3,7 @@ from .common import Validateable, Devops, BuildType, MixinType
from .image import Image from .image import Image
from .c4k import C4k from .c4k import C4k
from .provs_k3s import K3s from .provs_k3s import K3s
from .terraform import Terraform
from .release import Release from .release import Release
from .version import Version from .version import Version
@ -22,6 +23,8 @@ class DevopsFactory:
specialized_builds[BuildType.C4K] = C4k(inp) specialized_builds[BuildType.C4K] = C4k(inp)
if BuildType.K3S in build_types: if BuildType.K3S in build_types:
specialized_builds[BuildType.K3S] = K3s(inp) specialized_builds[BuildType.K3S] = K3s(inp)
if BuildType.K3S in build_types:
specialized_builds[BuildType.TERRAFORM] = Terraform(inp)
mixins: Dict[MixinType, Validateable] = {} mixins: Dict[MixinType, Validateable] = {}
if MixinType.RELEASE in mixin_types: if MixinType.RELEASE in mixin_types:

36
src/main/python/ddadevops/domain/terraform.py Normal file
View file

@ -0,0 +1,36 @@
from typing import List, Optional
from .common import (
Validateable,
DnsRecord,
Devops,
)
class Terraform(Validateable):
def __init__(self, inp: dict):
self.module = inp.get("module")
self.stage = inp.get("stage")
self.tf_additional_vars = inp.get("tf_additional_vars")
self.tf_output_json_name = inp.get("tf_output_json_name")
self.tf_build_commons_path = inp.get("tf_build_commons_path")
self.tf_additional_tfvar_files = inp.get("tf_additional_tfvar_files", [])
self.tf_use_workspace = inp.get("tf_use_workspace", True)
self.tf_debug_print_terraform_command = inp.get(
"tf_debug_print_terraform_command", False
)
self.tf_commons_dir_name = inp.get("tf_commons_dir_name", "terraform")
self.tf_terraform_semantic_version = inp.get(
"tf_terraform_semantic_version", "1.0.8"
)
self.tf_use_package_common_files = inp.get("tf_use_package_common_files", True)
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("module")
return result
def output_json_name(self) -> str:
if self.tf_output_json_name:
return self.tf_output_json_name
else:
return f"out_{self.module}.json"

12
src/test/python/domain/helper.py
View file

@ -9,7 +9,7 @@ def devops_config(overrides: dict) -> dict:
"stage": "test", "stage": "test",
"project_root_path": "root_path", "project_root_path": "root_path",
"build_dir_name": "target", "build_dir_name": "target",
"build_types": ["IMAGE", "C4K", "K3S"], "build_types": ["IMAGE", "C4K", "K3S", "TERRAFORM"],
"mixin_types": ["RELEASE"], "mixin_types": ["RELEASE"],
"image_dockerhub_user": "dockerhub_user", "image_dockerhub_user": "dockerhub_user",
"image_dockerhub_password": "dockerhub_password", "image_dockerhub_password": "dockerhub_password",
@ -24,6 +24,16 @@ def devops_config(overrides: dict) -> dict:
"k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint",
"k3s_enable_echo": "false", "k3s_enable_echo": "false",
"k3s_app_filename_to_provision": "k3s_app.yaml", "k3s_app_filename_to_provision": "k3s_app.yaml",
"tf_additional_vars": None,
"tf_output_json_name": "the_out.json",
"tf_use_workspace": None,
"tf_use_package_common_files": None,
"tf_build_commons_path": None,
"tf_commons_dir_name": None,
"tf_debug_print_terraform_command": None,
"tf_additional_tfvar_files": None,
"tf_terraform_semantic_version": None,
"release_type": "NONE", "release_type": "NONE",
"release_main_branch": "main", "release_main_branch": "main",
"release_current_branch": "my_feature", "release_current_branch": "my_feature",

19
src/test/python/domain/test_terraform.py Normal file
View file

@ -0,0 +1,19 @@
import pytest
from pathlib import Path
from src.main.python.ddadevops.domain import DnsRecord, BuildType, Terraform
from .helper import build_devops
def test_creation():
sut = build_devops({})
assert BuildType.TERRAFORM in sut.specialized_builds
assert sut.specialized_builds[BuildType.TERRAFORM]
def test_should_calculate_output_json_name():
devops = build_devops({})
sut = devops.specialized_builds[BuildType.TERRAFORM]
assert 'the_out.json' == sut.output_json_name()
devops = build_devops({"tf_output_json_name": None,})
sut = devops.specialized_builds[BuildType.TERRAFORM]
assert 'out_module.json' == sut.output_json_name()