new layout with base build & mixins

This commit is contained in:
jem 2020-03-03 16:34:12 +01:00
parent c77c434794
commit 86fb7740a1
8 changed files with 194 additions and 139 deletions

View file

@ -27,7 +27,7 @@ use_plugin("python.distutils")
default_task = "publish" default_task = "publish"
name = "ddadevops" name = "ddadevops"
version = "0.3.7.dev4" version = "0.4.0.dev1"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__ description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]

View file

@ -4,13 +4,11 @@ terraform, dda-pallet, aws & hetzner-cloud.
""" """
from .build import init_project, stage, name, module, \
project_root_path, build_commons_path, build_target_path, \
initialize_target, tf_import_name, tf_import_resource
from .meissa_build import meissa_init_project, hetzner_api_key
from .credential import gopass_credential_from_env_path, gopass_credential_from_path from .credential import gopass_credential_from_env_path, gopass_credential_from_path
from .dda_pallet import dda_write_target, dda_write_domain, dda_install, dda_configure from .devops_build import DevopsBuild, create_devops_build_config
from .terraform import tf_copy_common, tf_plan, tf_import, tf_apply, tf_output, tf_destroy, \ from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config
tf_read_output_json from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config
from .aws_mixin import AwsMixin, add_aws_mixin_config
from .dda_pallet_mixin import DdaPalletMixin, add_dda_pallet_mixin_config
__version__ = "${version}" __version__ = "${version}"

View file

@ -0,0 +1,37 @@
from python_terraform import *
from .credential import gopass_credential_from_env_path
from .devops_terraform_build import DevopsTerraformBuild
def add_aws_mixin_config(config, account_name):
return config.update({'AwsMixin':
{'account_name': account_name}})
class AwsMixin(DevopsTerraformBuild):
def __init__(self, project, config):
super().__init__(self, project, config)
aws_mixin_config = config['AwsMixin']
self.account_name = aws_mixin_config['account_name']
def backend_config(self):
return "backend." + self.account_name + "." + self.stage + ".properties"
def project_vars(self):
ret = super().project_vars()
return ret.update({'account_name': self.account_name})
def init_client(self):
tf = Terraform(working_dir=self.build_path())
tf.init(backend_config=self.backend_config)
try:
tf.workspace('select', slef.stage)
except:
tf.workspace('new', self.stage)
return tf
def plan(self):
tf = self.init_client()
tf.plan(capture_output=False, var=self.project_vars,
var_file=self.backend_config)

View file

@ -1,46 +0,0 @@
from .python_util import *
from .meissa_build import build_target_path, project_root_path
from string import Template
TARGET = 'target.edn'
TEMPLATE_TARGET_CONTENT = Template("""
{:existing [{:node-name "k8s"
:node-ip "$ipv4"}]
:provisioning-user {:login "root"}}
""")
def dda_write_target(project, ipv4):
with open(build_target_path(project) + TARGET, "w") as output_file:
output_file.write(TEMPLATE_TARGET_CONTENT.substitute({'ipv4' : ipv4}))
def dda_write_domain(project, domain_file_name, substitues):
with open(build_target_path(project) + domain_file_name, "r") as input_file:
domain_input = input_file.read()
domain_template = Template(domain_input)
with open(build_target_path(project) + 'out_' + domain_file_name, "w") as output_file:
output_file.write(domain_template.substitute(substitues))
def dda_install(project, tenant, application, domain_file_name):
return dda_uberjar(project, tenant, application, domain_file_name)
def dda_configure(project, tenant, application, domain_file_name):
return dda_uberjar(project, tenant, application, domain_file_name, True)
def dda_uberjar(project, tenant, application, domain_file_name, configure_switch=None):
if configure_switch:
cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \
'--targets', build_target_path(project) + TARGET, \
'--tenant', tenant, '--application', application, \
'--configure', \
build_target_path(project) + 'out_' + domain_file_name]
else:
cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \
'--targets', build_target_path(project) + TARGET, \
'--tenant', tenant, '--application', application, \
build_target_path(project) + 'out_' + domain_file_name]
prn_cmd=list(cmd)
print(" ".join(prn_cmd))
output = execute(cmd)
print(output)
return output

View file

@ -0,0 +1,68 @@
from string import Template
from .python_util import *
from .devops_build import DevopsBuild
def add_dda_pallet_mixin_config(config, tenant, application, domain_file_name):
return config.update({'DdaPalletMixin':
{'tenant': tenant,
'application': application,
'domain_file_name': domain_file_name,
'target_edn_name': 'target.edn',
'jar_file': 'target/meissa-tenant-server.jar',
'target_template':
"""
{:existing [{:node-name "$node_name"
:node-ip "$ipv4"}]
:provisioning-user {:login "root"}}
""", }})
class DdaPalletMixin(DevopsBuild):
def __init__(self, project, config):
super().__init__(self, project, config)
dda_pallet_mixin_config = config['DdaPalletMixin']
self.tenant = dda_pallet_mixin_config['tenant']
self.application = dda_pallet_mixin_config['application']
self.domain_file_name = dda_pallet_mixin_config['domain_file_name']
self.target_edn_name = dda_pallet_mixin_config['target_edn_name']
self.jar_file = dda_pallet_mixin_config['jar_file']
self.target_template = Template(
dda_pallet_mixin_config['target_template'])
def dda_write_target(self, node_name, ipv4):
with open(self.build_path() + self.target_edn_name, "w") as output_file:
output_file.write(
self.target_template.substitute({'ipv4': ipv4, 'node_name': node_name}))
def dda_write_domain(self, domain_file_name, substitues):
with open(self.build_path() + domain_file_name, "r") as input_file:
domain_input = input_file.read()
domain_template = Template(domain_input)
with open(self.build_path() + 'out_' + domain_file_name, "w") as output_file:
output_file.write(domain_template.substitute(substitues))
def dda_uberjar(self, configure_switch=None):
if configure_switch:
cmd = ['java', '-jar', self.project_root_path() + self.jar_file,
'--targets', self.build_path() + self.target_edn_name,
'--tenant', self.tenant, '--application', self.application,
'--configure',
self.build_path() + 'out_' + self.domain_file_name]
else:
cmd = ['java', '-jar', self.project_root_path() + self.jar_file,
'--targets', self.build_path() + self.target_edn_name,
'--tenant', self.tenant, '--application', self.application,
self.build_path() + 'out_' + self.domain_file_name]
prn_cmd = list(cmd)
print(" ".join(prn_cmd))
output = execute(cmd)
print(output)
return output
def dda_install(self):
return self.dda_uberjar()
def dda_configure(self):
return self.dda_uberjar(True)

View file

@ -1,15 +1,23 @@
from .credential import gopass_credential_from_env_path
from subprocess import run from subprocess import run
def create_devops_build_config(stage, project_root_path, build_commons_path, module):
return {'stage': stage,
'project_root_path': project_root_path,
'build_commons_path': build_commons_path,
'module': module,
'build_dir_name': 'target'}
class DevopsBuild: class DevopsBuild:
def __init__(self, project, project_root_path, build_commons_path, module, stage): def __init__(self, project, config):
self.stage = stage self.stage = config['stage']
self.project_root_path = project_root_path self.project_root_path = config['project_root_path']
self.build_commons_path = build_commons_path self.build_commons_path = config['build_commons_path']
self.module = module self.module = config['module']
self.build_dir_name = config['build_dir_name']
self.project = project self.project = project
self.build_dir_name = 'target'
project.set_property("devops_build", self) project.set_property("devops_build", self)
def name(self): def name(self):
@ -26,6 +34,6 @@ class DevopsBuild:
def tf_import_name(project): def tf_import_name(project):
return project.get_property('tf_import_name') return project.get_property('tf_import_name')
def tf_import_resource(project): def tf_import_resource(project):
return project.get_property('tf_import_resource') return project.get_property('tf_import_resource')

View file

@ -1,20 +1,31 @@
from os import path from os import path
from json import load from json import load
from subprocess import run from subprocess import run
from .devops_build import DevopsBuild
from .python_util import execute
from python_terraform import * from python_terraform import *
from .devops_build import DevopsBuild, create_devops_build_config
def create_devops_terraform_build_config(stage, project_root_path, build_commons_path, module,
account_name, additional_vars, tf_import_name, tf_import_resource):
ret = create_devops_build_config(
stage, project_root_path, build_commons_path, module)
return ret.update({'account_name': account_name,
'additional_vars': additional_vars,
'tf_import_name': tf_import_name,
'tf_import_resource': tf_import_resource,
'terraform_build_commons_dir_name': 'terraform',
'output_json_name': 'output.json'})
class DevopsTerraformBuild(DevopsBuild): class DevopsTerraformBuild(DevopsBuild):
def __init__(self, project, project_root_path, build_commons_path, module, stage, account_name, additional_vars): def __init__(self, project, config):
super().__init__(self, project, project_root_path, build_commons_path, module, stage) super().__init__(self, project, config)
self.account_name = account_name self.additional_vars = config['additional_vars']
self.additional_vars = additional_vars self.tf_import_name = config['tf_import_name']
self.terraform_build_commons_dir_name= 'terraform' self.tf_import_resource = config['tf_import_resource']
self.terraform_build_commons_dir_name = config['terraform_build_commons_dir_name']
def backend_config(self): self.output_json_name = config['output_json_name']
return "backend." + self.account_name + "." + self.stage + ".properties"
def terraform_build_commons_path(self): def terraform_build_commons_path(self):
return self.build_commons_path() + '/' + self.terraform_build_commons_dir_name return self.build_commons_path() + '/' + self.terraform_build_commons_dir_name
@ -29,7 +40,8 @@ class DevopsTerraformBuild(DevopsBuild):
def initialize_build_dir(self): def initialize_build_dir(self):
super().initialize_build_dir() super().initialize_build_dir()
run('cp -f ' + self.terraform_build_commons_path + '* ' + self.build_path, shell=True) run('cp -f ' + self.terraform_build_commons_path +
'* ' + self.build_path, shell=True)
run('cp *.tf ' + self.build_path, shell=True) run('cp *.tf ' + self.build_path, shell=True)
run('cp *.properties ' + self.build_path, shell=True) run('cp *.properties ' + self.build_path, shell=True)
run('cp *.tfars ' + self.build_path, shell=True) run('cp *.tfars ' + self.build_path, shell=True)
@ -37,69 +49,38 @@ class DevopsTerraformBuild(DevopsBuild):
def init_client(self): def init_client(self):
tf = Terraform(working_dir=self.build_path()) tf = Terraform(working_dir=self.build_path())
tf.init(backend_config=self.backend_config) tf.init()
try: try:
tf.workspace('select', slef.stage) tf.workspace('select', slef.stage)
except: except:
tf.workspace('new', self.stage) tf.workspace('new', self.stage)
return tf return tf
def plan(self): def write_output(self, tf):
tf = self.init_client()
tf.plan(capture_output=False, var=self.project_vars, var_file=self.backend_config)
OUTPUT_JSON = "output.json"
def tf_copy_common(project):
run(['cp', '-f', build_commons_path(project) + 'terraform/aws_provider.tf', \
build_target_path(project) + 'aws_provider.tf'])
run(['cp', '-f', build_commons_path(project) + 'terraform/variables.tf', \
build_target_path(project) + 'variables.tf'])
def tf_plan(project):
init(project)
tf = Terraform(working_dir=build_target_path(project))
tf.plan(capture_output=False, var=project_vars(project))
def tf_import(project):
init(project)
tf = Terraform(working_dir=build_target_path(project))
tf.import_cmd(tf_import_name(project), tf_import_resource(project), \
capture_output=False, var=project_vars(project))
def tf_apply(project, p_auto_approve=False):
init(project)
tf = Terraform(working_dir=build_target_path(project))
tf.apply(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project))
tf_output(project)
def tf_output(project):
init(project)
tf = Terraform(working_dir=build_target_path(project))
result = tf.output(json=IsFlagged) result = tf.output(json=IsFlagged)
with open(build_target_path(project) + OUTPUT_JSON, "w") as output_file: with open(self.build_path() + self.output_json_name, "w") as output_file:
output_file.write(json.dumps(result)) output_file.write(json.dumps(result))
def tf_destroy(project, p_auto_approve=False): def read_output_json(self):
init(project) with open(self.build_path() + self.output_json_name, 'r') as f:
tf = Terraform(working_dir=build_target_path(project))
tf.destroy(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project))
def tf_read_output_json(project):
with open(build_target_path(project) + OUTPUT_JSON, 'r') as f:
return load(f) return load(f)
def project_vars(project): def plan(self):
my_hetzner_api_key = hetzner_api_key(project) tf = self.init_client()
my_module = project.name tf.plan(capture_output=False, var=self.project_vars)
ret = {'stage' : stage(project)}
# TODO: move to meissa specific part
if my_hetzner_api_key:
ret['hetzner_api_key'] = my_hetzner_api_key
if my_module:
ret['module'] = my_module
return ret
def apply(self, p_auto_approve=False):
tf = self.init_client()
tf.apply(capture_output=False, auto_approve=p_auto_approve,
var=self.project_vars())
self.write_output(tf)
def destroy(self, p_auto_approve=False):
tf = self.init_client()
tf.destroy(capture_output=False, auto_approve=p_auto_approve,
var=self.project_vars())
def tf_import(self):
tf = self.init_client()
tf.import_cmd(self.tf_import_name, self.tf_import_resource,
capture_output=False, var=self.project_vars())

View file

@ -1,10 +1,19 @@
from .credential import gopass_credential_from_env_path from .credential import gopass_credential_from_env_path
from .devops_terraform_build import DevopsTerraformBuild
class HetznerMixin:
def __init__(self, project, project_root_path, build_commons_path, module, stage): def add_hetzner_mixin_config(config):
super().__init__(self, project, project_root_path, build_commons_path, module, stage) return config.update({'HetznerMixin':
self.hetzner_api_key = gopass_credential_from_env_path('HETZNER_API_KEY_PATH') {'HETZNER_API_KEY_PATH_ENVIRONMENT': 'HETZNER_API_KEY_PATH'}})
class HetznerMixin(DevopsTerraformBuild):
def __init__(self, project, config):
super().__init__(self, project, config)
hetzner_mixin_config = config['HetznerMixin']
self.hetzner_api_key = gopass_credential_from_env_path(
hetzner_mixin_config['HETZNER_API_KEY_PATH_ENVIRONMENT'])
def project_vars(self): def project_vars(self):
ret = super().project_vars ret = super().project_vars