devops_terraform_build now might work

This commit is contained in:
Michael Jerger 2023-05-24 16:04:42 +02:00
parent 7ede8e345a
commit 62464afb83
7 changed files with 189 additions and 151 deletions

View file

@ -1,22 +1,26 @@
from pathlib import Path from pathlib import Path
from ..domain import Devops, BuildType from dda_python_terraform import Terraform, IsFlagged
from ..infrastructure import FileApi, ResourceApi, ImageApi from packaging import version
from ..domain import Devops, BuildType, TerraformDomain
from ..infrastructure import FileApi, ResourceApi, TerraformApi
# TODO: mv more fkt to Terraform_api ?
class TerraformService: class TerraformService:
def __init__( def __init__(
self, file_api: FileApi, resource_api: ResourceApi, image_api: ImageApi self, file_api: FileApi, resource_api: ResourceApi, terraform_api: TerraformApi
): ):
self.file_api = file_api self.file_api = file_api
self.resource_api = resource_api self.resource_api = resource_api
self.image_api = image_api self.terraform_api = terraform_api
@classmethod @classmethod
def prod(cls): def prod(cls):
return cls( return cls(
FileApi(), FileApi(),
ResourceApi(), ResourceApi(),
ImageApi(), TerraformApi(),
) )
def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops):
@ -39,6 +43,12 @@ class TerraformService:
f"{terraform.build_commons_path()}/*", devops.build_path() f"{terraform.build_commons_path()}/*", devops.build_path()
) )
def __print_terraform_command__(self, terraform: Terraform, devops: Devops):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
if terraform_domain.tf_debug_print_terraform_command:
output = f"cd {devops.build_path()} && {terraform.latest_cmd()}"
print(output)
def copy_local_state(self, devops: Devops): def copy_local_state(self, devops: Devops):
# TODO: orignal was unchecked ... # TODO: orignal was unchecked ...
self.file_api.cp("terraform.tfstate", devops.build_path()) self.file_api.cp("terraform.tfstate", devops.build_path())
@ -60,3 +70,143 @@ class TerraformService:
self.file_api.cp("*.tfvars", devops.build_path()) self.file_api.cp("*.tfvars", devops.build_path())
self.file_api.cp_recursive("scripts", devops.build_path()) self.file_api.cp_recursive("scripts", devops.build_path())
def init_client(self, devops: Devops):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
terraform = Terraform(
working_dir=devops.build_path(),
terraform_semantic_version=terraform_domain.tf_terraform_semantic_version,
)
terraform.init()
self.__print_terraform_command__(terraform, devops)
if terraform_domain.tf_use_workspace:
try:
terraform.workspace("select", self.stage)
self.__print_terraform_command__(terraform, devops)
except:
terraform.workspace("new", self.stage)
self.__print_terraform_command__(terraform, devops)
return terraform
def write_output(self, terraform, devops: Devops):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
result = terraform.output(json=IsFlagged)
self.__print_terraform_command__(terraform, devops)
self.file_api.write_json_to_file(
Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}"), result
)
def read_output(self, devops: Devops):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
return self.file_api.read_json_fro_file(
Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}")
)
def plan(self, devops: Devops, fail_on_diff=False):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
if fail_on_diff:
detailed_exitcode = IsFlagged
else:
detailed_exitcode = None
terraform = self.init_client(devops)
return_code, _, stderr = terraform.plan(
detailed_exitcode=detailed_exitcode,
capture_output=False,
raise_on_error=False,
var=terraform_domain.project_vars(),
var_file=terraform_domain.tf_additional_tfvar_files,
)
self.__print_terraform_command__(terraform)
if return_code not in (0, 2):
raise RuntimeError(return_code, "terraform error:", stderr)
if return_code == 2:
raise RuntimeError(return_code, "diff in config found:", stderr)
def apply(self, devops: Devops, auto_approve=False):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
if auto_approve:
auto_approve_flag = IsFlagged
else:
auto_approve_flag = None
terraform = self.init_client(devops)
if version.parse(
terraform_domain.tf_terraform_semantic_version
) >= version.parse("1.0.0"):
return_code, _, stderr = terraform.apply(
capture_output=False,
raise_on_error=True,
auto_approve=auto_approve_flag,
var=terraform_domain.project_vars(),
var_file=terraform_domain.tf_additional_tfvar_files,
)
else:
return_code, _, stderr = terraform.apply(
capture_output=False,
raise_on_error=True,
skip_plan=auto_approve,
var=terraform_domain.project_vars(),
var_file=terraform_domain.tf_additional_tfvar_files,
)
self.__print_terraform_command__(terraform, devops)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
self.write_output(terraform, devops)
def refresh(self, devops: Devops):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
terraform = self.init_client(devops)
return_code, _, stderr = terraform.refresh(
var=terraform_domain.project_vars(),
var_file=terraform_domain.tf_additional_tfvar_files,
)
self.__print_terraform_command__(terraform, devops)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
self.write_output(terraform, devops)
def destroy(self, devops: Devops, auto_approve=False):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
if auto_approve:
auto_approve_flag = IsFlagged
else:
auto_approve_flag = None
terraform = self.init_client(devops)
if version.parse(
terraform_domain.tf_terraform_semantic_version
) >= version.parse("1.0.0"):
return_code, _, stderr = terraform.destroy(
capture_output=False,
raise_on_error=True,
auto_approve=auto_approve_flag,
var=terraform_domain.project_vars(),
var_file=terraform_domain.tf_additional_tfvar_files,
)
else:
return_code, _, stderr = terraform.destroy(
capture_output=False,
raise_on_error=True,
force=auto_approve_flag,
var=terraform_domain.project_vars(),
var_file=terraform_domain.tf_additional_tfvar_files,
)
self.__print_terraform_command__(terraform, devops)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
def tf_import(
self,
devops: Devops,
tf_import_name,
tf_import_resource,
):
terraform_domain = devops.specialized_builds[BuildType.TERRAFORM]
return_code, _, stderr = terraform.import_cmd(
tf_import_name,
tf_import_resource,
capture_output=False,
raise_on_error=True,
var=terraform_domain.project_vars(),
var_file=terraform_domain.tf_additional_tfvar_files,
)
self.print_terraform_command(terraform, devops)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)

View file

@ -1,11 +1,3 @@
import sys
from os import chmod
from json import load, dumps
from subprocess import run
from packaging import version
from pkg_resources import resource_string
from dda_python_terraform import Terraform, IsFlagged
from .python_util import filter_none
from .devops_build import DevopsBuild, create_devops_build_config from .devops_build import DevopsBuild, create_devops_build_config
@ -67,153 +59,40 @@ class DevopsTerraformBuild(DevopsBuild):
devops = self.devops_repo.get_devops(self.project) devops = self.devops_repo.get_devops(self.project)
self.teraform_service.rescue_local_state(devops) self.teraform_service.rescue_local_state(devops)
def init_client(self): def read_output_json(self) -> map:
terraform = Terraform( devops = self.devops_repo.get_devops(self.project)
working_dir=self.build_path(), return self.teraform_service.read_output(devops)
terraform_semantic_version=self.terraform_semantic_version,
)
terraform.init()
self.print_terraform_command(terraform)
if self.use_workspace:
try:
terraform.workspace("select", self.stage)
self.print_terraform_command(terraform)
except:
terraform.workspace("new", self.stage)
self.print_terraform_command(terraform)
return terraform
def write_output(self, terraform):
result = terraform.output(json=IsFlagged)
self.print_terraform_command(terraform)
with open(
self.build_path() + self.output_json_name, "w", encoding="utf-8"
) as output_file:
output_file.write(dumps(result))
chmod(self.build_path() + self.output_json_name, 0o600)
def read_output_json(self):
with open(
self.build_path() + self.output_json_name, "r", encoding="utf-8"
) as file:
return load(file)
def plan(self): def plan(self):
terraform = self.init_client() devops = self.devops_repo.get_devops(self.project)
return_code, _, stderr = terraform.plan( self.teraform_service.plan(devops)
detailed_exitcode=None,
capture_output=False,
raise_on_error=False,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
def plan_fail_on_diff(self): def plan_fail_on_diff(self):
terraform = self.init_client() devops = self.devops_repo.get_devops(self.project)
return_code, _, stderr = terraform.plan( self.teraform_service.plan(devops, fail_on_diff=True)
detailed_exitcode=IsFlagged,
capture_output=False,
raise_on_error=False,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform)
if return_code not in (0, 2):
raise RuntimeError(return_code, "terraform error:", stderr)
if return_code == 2:
raise RuntimeError(return_code, "diff in config found:", stderr)
def apply(self, auto_approve=False): def apply(self, auto_approve=False):
terraform = self.init_client() devops = self.devops_repo.get_devops(self.project)
if auto_approve: self.teraform_service.apply(devops, auto_approve=auto_approve)
auto_approve_flag = IsFlagged
else:
auto_approve_flag = None
if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"):
return_code, _, stderr = terraform.apply(
capture_output=False,
raise_on_error=True,
auto_approve=auto_approve_flag,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
else:
return_code, _, stderr = terraform.apply(
capture_output=False,
raise_on_error=True,
skip_plan=auto_approve,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.write_output(terraform)
self.post_build() self.post_build()
self.print_terraform_command(terraform)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
def refresh(self): def refresh(self):
terraform = self.init_client() devops = self.devops_repo.get_devops(self.project)
return_code, _, stderr = terraform.refresh( self.teraform_service.refresh(devops)
var=self.project_vars(), var_file=self.additional_tfvar_files
)
self.write_output(terraform)
self.post_build() self.post_build()
self.print_terraform_command(terraform)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
def destroy(self, auto_approve=False): def destroy(self, auto_approve=False):
terraform = self.init_client() devops = self.devops_repo.get_devops(self.project)
if auto_approve: self.teraform_service.refresh(devops)
auto_approve_flag = IsFlagged
else:
auto_approve_flag = None
if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"):
return_code, _, stderr = terraform.destroy(
capture_output=False,
raise_on_error=True,
auto_approve=auto_approve_flag,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
else:
return_code, _, stderr = terraform.destroy(
capture_output=False,
raise_on_error=True,
force=auto_approve_flag,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
def tf_import( def tf_import(
self, self,
tf_import_name, tf_import_name,
tf_import_resource, tf_import_resource,
): ):
terraform = self.init_client() devops = self.devops_repo.get_devops(self.project)
return_code, _, stderr = terraform.import_cmd( self.teraform_service.tf_import(devops, tf_import_name, tf_import_resource)
tf_import_name,
tf_import_resource,
capture_output=False,
raise_on_error=True,
var=self.project_vars(),
var_file=self.additional_tfvar_files,
)
self.post_build() self.post_build()
self.print_terraform_command(terraform)
if return_code > 0:
raise RuntimeError(return_code, "terraform error:", stderr)
def print_terraform_command(self, terraform):
if self.debug_print_terraform_command:
output = "cd " + self.build_path() + " && " + terraform.latest_cmd()
print(output)

View file

@ -2,7 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea
from .devops_factory import DevopsFactory from .devops_factory import DevopsFactory
from .image import Image from .image import Image
from .c4k import C4k from .c4k import C4k
from .terraform import Terraform from .terraform import TerraformDomain
from .provs_k3s import K3s from .provs_k3s import K3s
from .release import Release from .release import Release
from .credentials import Credentials, CredentialMapping, GopassType from .credentials import Credentials, CredentialMapping, GopassType

View file

@ -3,7 +3,7 @@ from .common import Validateable, Devops, BuildType, MixinType
from .image import Image from .image import Image
from .c4k import C4k from .c4k import C4k
from .provs_k3s import K3s from .provs_k3s import K3s
from .terraform import Terraform from .terraform import TerraformDomain
from .release import Release from .release import Release
from .version import Version from .version import Version
@ -24,7 +24,7 @@ class DevopsFactory:
if BuildType.K3S in build_types: if BuildType.K3S in build_types:
specialized_builds[BuildType.K3S] = K3s(inp) specialized_builds[BuildType.K3S] = K3s(inp)
if BuildType.K3S in build_types: if BuildType.K3S in build_types:
specialized_builds[BuildType.TERRAFORM] = Terraform(inp) specialized_builds[BuildType.TERRAFORM] = TerraformDomain(inp)
mixins: Dict[MixinType, Validateable] = {} mixins: Dict[MixinType, Validateable] = {}
if MixinType.RELEASE in mixin_types: if MixinType.RELEASE in mixin_types:

View file

@ -8,7 +8,7 @@ from .common import (
) )
class Terraform(Validateable): class TerraformDomain(Validateable):
def __init__(self, inp: dict): def __init__(self, inp: dict):
self.module = inp.get("module") self.module = inp.get("module")
self.stage = inp.get("stage") self.stage = inp.get("stage")

View file

@ -3,6 +3,7 @@ from pathlib import Path
from sys import stdout from sys import stdout
from os import chmod, environ from os import chmod, environ
from pkg_resources import resource_string from pkg_resources import resource_string
from json import load, dumps
import yaml import yaml
@ -37,6 +38,14 @@ class FileApi:
yaml.dump(data, output_file) yaml.dump(data, output_file)
chmod(path, 0o600) chmod(path, 0o600)
def write_json_to_file(self, path: Path, data: map):
with open(path, "w", encoding="utf-8") as output_file:
output_file.write(dumps(data))
chmod(path, 0o600)
def read_json_fro_file(self, path: Path) -> map:
with open(path, "r", encoding="utf-8") as input_file:
return load(input_file)
class ImageApi: class ImageApi:
def image(self, name: str, path: Path): def image(self, name: str, path: Path):

View file

@ -1,6 +1,6 @@
import pytest import pytest
from pathlib import Path from pathlib import Path
from src.main.python.ddadevops.domain import DnsRecord, BuildType, Terraform from src.main.python.ddadevops.domain import DnsRecord, BuildType, TerraformDomain
from .helper import build_devops, devops_config from .helper import build_devops, devops_config
@ -12,12 +12,12 @@ def test_creation():
def test_should_calculate_output_json_name(): def test_should_calculate_output_json_name():
config = devops_config({}) config = devops_config({})
sut = Terraform(config) sut = TerraformDomain(config)
assert "the_out.json" == sut.output_json_name() assert "the_out.json" == sut.output_json_name()
config = devops_config({}) config = devops_config({})
del config["tf_output_json_name"] del config["tf_output_json_name"]
sut = Terraform(config) sut = TerraformDomain(config)
assert "out_module.json" == sut.output_json_name() assert "out_module.json" == sut.output_json_name()
@ -25,14 +25,14 @@ def test_should_calculate_terraform_build_commons_path():
config = devops_config({}) config = devops_config({})
del config["tf_build_commons_path"] del config["tf_build_commons_path"]
del config["tf_build_commons_dir_name"] del config["tf_build_commons_dir_name"]
sut = Terraform(config) sut = TerraformDomain(config)
assert Path("terraform") == sut.terraform_build_commons_path() assert Path("terraform") == sut.terraform_build_commons_path()
config = devops_config({}) config = devops_config({})
sut = Terraform(config) sut = TerraformDomain(config)
assert Path("build_commons_path/terraform") == sut.terraform_build_commons_path() assert Path("build_commons_path/terraform") == sut.terraform_build_commons_path()
def test_should_calculate_project_vars(): def test_should_calculate_project_vars():
config = devops_config({}) config = devops_config({})
sut = Terraform(config) sut = TerraformDomain(config)
assert {'module': 'module', 'stage': 'test'} == sut.project_vars() assert {'module': 'module', 'stage': 'test'} == sut.project_vars()