new layout with base build & mixins

This commit is contained in:
jem 2020-03-03 16:34:12 +01:00
parent c77c434794
commit 86fb7740a1
8 changed files with 194 additions and 139 deletions

View file

@ -27,7 +27,7 @@ use_plugin("python.distutils")
default_task = "publish"
name = "ddadevops"
version = "0.3.7.dev4"
version = "0.4.0.dev1"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]

View file

@ -4,13 +4,11 @@ terraform, dda-pallet, aws & hetzner-cloud.
"""
from .build import init_project, stage, name, module, \
project_root_path, build_commons_path, build_target_path, \
initialize_target, tf_import_name, tf_import_resource
from .meissa_build import meissa_init_project, hetzner_api_key
from .credential import gopass_credential_from_env_path, gopass_credential_from_path
from .dda_pallet import dda_write_target, dda_write_domain, dda_install, dda_configure
from .terraform import tf_copy_common, tf_plan, tf_import, tf_apply, tf_output, tf_destroy, \
tf_read_output_json
from .devops_build import DevopsBuild, create_devops_build_config
from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config
from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config
from .aws_mixin import AwsMixin, add_aws_mixin_config
from .dda_pallet_mixin import DdaPalletMixin, add_dda_pallet_mixin_config
__version__ = "${version}"

View file

@ -0,0 +1,37 @@
from python_terraform import *
from .credential import gopass_credential_from_env_path
from .devops_terraform_build import DevopsTerraformBuild
def add_aws_mixin_config(config, account_name):
return config.update({'AwsMixin':
{'account_name': account_name}})
class AwsMixin(DevopsTerraformBuild):
def __init__(self, project, config):
super().__init__(self, project, config)
aws_mixin_config = config['AwsMixin']
self.account_name = aws_mixin_config['account_name']
def backend_config(self):
return "backend." + self.account_name + "." + self.stage + ".properties"
def project_vars(self):
ret = super().project_vars()
return ret.update({'account_name': self.account_name})
def init_client(self):
tf = Terraform(working_dir=self.build_path())
tf.init(backend_config=self.backend_config)
try:
tf.workspace('select', slef.stage)
except:
tf.workspace('new', self.stage)
return tf
def plan(self):
tf = self.init_client()
tf.plan(capture_output=False, var=self.project_vars,
var_file=self.backend_config)

View file

@ -1,46 +0,0 @@
from .python_util import *
from .meissa_build import build_target_path, project_root_path
from string import Template
TARGET = 'target.edn'
TEMPLATE_TARGET_CONTENT = Template("""
{:existing [{:node-name "k8s"
:node-ip "$ipv4"}]
:provisioning-user {:login "root"}}
""")
def dda_write_target(project, ipv4):
with open(build_target_path(project) + TARGET, "w") as output_file:
output_file.write(TEMPLATE_TARGET_CONTENT.substitute({'ipv4' : ipv4}))
def dda_write_domain(project, domain_file_name, substitues):
with open(build_target_path(project) + domain_file_name, "r") as input_file:
domain_input = input_file.read()
domain_template = Template(domain_input)
with open(build_target_path(project) + 'out_' + domain_file_name, "w") as output_file:
output_file.write(domain_template.substitute(substitues))
def dda_install(project, tenant, application, domain_file_name):
return dda_uberjar(project, tenant, application, domain_file_name)
def dda_configure(project, tenant, application, domain_file_name):
return dda_uberjar(project, tenant, application, domain_file_name, True)
def dda_uberjar(project, tenant, application, domain_file_name, configure_switch=None):
if configure_switch:
cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \
'--targets', build_target_path(project) + TARGET, \
'--tenant', tenant, '--application', application, \
'--configure', \
build_target_path(project) + 'out_' + domain_file_name]
else:
cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \
'--targets', build_target_path(project) + TARGET, \
'--tenant', tenant, '--application', application, \
build_target_path(project) + 'out_' + domain_file_name]
prn_cmd=list(cmd)
print(" ".join(prn_cmd))
output = execute(cmd)
print(output)
return output

View file

@ -0,0 +1,68 @@
from string import Template
from .python_util import *
from .devops_build import DevopsBuild
def add_dda_pallet_mixin_config(config, tenant, application, domain_file_name):
return config.update({'DdaPalletMixin':
{'tenant': tenant,
'application': application,
'domain_file_name': domain_file_name,
'target_edn_name': 'target.edn',
'jar_file': 'target/meissa-tenant-server.jar',
'target_template':
"""
{:existing [{:node-name "$node_name"
:node-ip "$ipv4"}]
:provisioning-user {:login "root"}}
""", }})
class DdaPalletMixin(DevopsBuild):
def __init__(self, project, config):
super().__init__(self, project, config)
dda_pallet_mixin_config = config['DdaPalletMixin']
self.tenant = dda_pallet_mixin_config['tenant']
self.application = dda_pallet_mixin_config['application']
self.domain_file_name = dda_pallet_mixin_config['domain_file_name']
self.target_edn_name = dda_pallet_mixin_config['target_edn_name']
self.jar_file = dda_pallet_mixin_config['jar_file']
self.target_template = Template(
dda_pallet_mixin_config['target_template'])
def dda_write_target(self, node_name, ipv4):
with open(self.build_path() + self.target_edn_name, "w") as output_file:
output_file.write(
self.target_template.substitute({'ipv4': ipv4, 'node_name': node_name}))
def dda_write_domain(self, domain_file_name, substitues):
with open(self.build_path() + domain_file_name, "r") as input_file:
domain_input = input_file.read()
domain_template = Template(domain_input)
with open(self.build_path() + 'out_' + domain_file_name, "w") as output_file:
output_file.write(domain_template.substitute(substitues))
def dda_uberjar(self, configure_switch=None):
if configure_switch:
cmd = ['java', '-jar', self.project_root_path() + self.jar_file,
'--targets', self.build_path() + self.target_edn_name,
'--tenant', self.tenant, '--application', self.application,
'--configure',
self.build_path() + 'out_' + self.domain_file_name]
else:
cmd = ['java', '-jar', self.project_root_path() + self.jar_file,
'--targets', self.build_path() + self.target_edn_name,
'--tenant', self.tenant, '--application', self.application,
self.build_path() + 'out_' + self.domain_file_name]
prn_cmd = list(cmd)
print(" ".join(prn_cmd))
output = execute(cmd)
print(output)
return output
def dda_install(self):
return self.dda_uberjar()
def dda_configure(self):
return self.dda_uberjar(True)

View file

@ -1,17 +1,25 @@
from .credential import gopass_credential_from_env_path
from subprocess import run
def create_devops_build_config(stage, project_root_path, build_commons_path, module):
return {'stage': stage,
'project_root_path': project_root_path,
'build_commons_path': build_commons_path,
'module': module,
'build_dir_name': 'target'}
class DevopsBuild:
def __init__(self, project, project_root_path, build_commons_path, module, stage):
self.stage = stage
self.project_root_path = project_root_path
self.build_commons_path = build_commons_path
self.module = module
def __init__(self, project, config):
self.stage = config['stage']
self.project_root_path = config['project_root_path']
self.build_commons_path = config['build_commons_path']
self.module = config['module']
self.build_dir_name = config['build_dir_name']
self.project = project
self.build_dir_name = 'target'
project.set_property("devops_build", self)
def name(self):
return self.project.get_property('name')
@ -20,12 +28,12 @@ class DevopsBuild:
def initialize_build_dir(self):
run('rm -rf ' + self.build_path(), shell=True)
run('mkdir -p ' + self.build_path(), shell=True)
run('mkdir -p ' + self.build_path(), shell=True)
def tf_import_name(project):
return project.get_property('tf_import_name')
def tf_import_resource(project):
return project.get_property('tf_import_resource')

View file

@ -1,26 +1,37 @@
from os import path
from json import load
from subprocess import run
from .devops_build import DevopsBuild
from .python_util import execute
from python_terraform import *
from .devops_build import DevopsBuild, create_devops_build_config
def create_devops_terraform_build_config(stage, project_root_path, build_commons_path, module,
account_name, additional_vars, tf_import_name, tf_import_resource):
ret = create_devops_build_config(
stage, project_root_path, build_commons_path, module)
return ret.update({'account_name': account_name,
'additional_vars': additional_vars,
'tf_import_name': tf_import_name,
'tf_import_resource': tf_import_resource,
'terraform_build_commons_dir_name': 'terraform',
'output_json_name': 'output.json'})
class DevopsTerraformBuild(DevopsBuild):
def __init__(self, project, project_root_path, build_commons_path, module, stage, account_name, additional_vars):
super().__init__(self, project, project_root_path, build_commons_path, module, stage)
self.account_name = account_name
self.additional_vars = additional_vars
self.terraform_build_commons_dir_name= 'terraform'
def backend_config(self):
return "backend." + self.account_name + "." + self.stage + ".properties"
def __init__(self, project, config):
super().__init__(self, project, config)
self.additional_vars = config['additional_vars']
self.tf_import_name = config['tf_import_name']
self.tf_import_resource = config['tf_import_resource']
self.terraform_build_commons_dir_name = config['terraform_build_commons_dir_name']
self.output_json_name = config['output_json_name']
def terraform_build_commons_path(self):
return self.build_commons_path() + '/' + self.terraform_build_commons_dir_name
def project_vars(self):
ret = {'stage' : self.stage}
ret = {'stage': self.stage}
if self.module:
ret['module'] = self.module
if self.additional_vars:
@ -29,77 +40,47 @@ class DevopsTerraformBuild(DevopsBuild):
def initialize_build_dir(self):
super().initialize_build_dir()
run('cp -f ' + self.terraform_build_commons_path + '* ' + self.build_path, shell=True)
run('cp *.tf ' + self.build_path, shell=True)
run('cp -f ' + self.terraform_build_commons_path +
'* ' + self.build_path, shell=True)
run('cp *.tf ' + self.build_path, shell=True)
run('cp *.properties ' + self.build_path, shell=True)
run('cp *.tfars ' + self.build_path, shell=True)
run('cp *.edn ' + self.build_path, shell=True)
run('cp *.tfars ' + self.build_path, shell=True)
run('cp *.edn ' + self.build_path, shell=True)
def init_client(self):
tf = Terraform(working_dir=self.build_path())
tf.init(backend_config=self.backend_config)
tf.init()
try:
tf.workspace('select', slef.stage)
except:
tf.workspace('new', self.stage)
return tf
def write_output(self, tf):
result = tf.output(json=IsFlagged)
with open(self.build_path() + self.output_json_name, "w") as output_file:
output_file.write(json.dumps(result))
def read_output_json(self):
with open(self.build_path() + self.output_json_name, 'r') as f:
return load(f)
def plan(self):
tf = self.init_client()
tf.plan(capture_output=False, var=self.project_vars, var_file=self.backend_config)
tf.plan(capture_output=False, var=self.project_vars)
def apply(self, p_auto_approve=False):
tf = self.init_client()
tf.apply(capture_output=False, auto_approve=p_auto_approve,
var=self.project_vars())
self.write_output(tf)
def destroy(self, p_auto_approve=False):
tf = self.init_client()
tf.destroy(capture_output=False, auto_approve=p_auto_approve,
var=self.project_vars())
OUTPUT_JSON = "output.json"
def tf_copy_common(project):
run(['cp', '-f', build_commons_path(project) + 'terraform/aws_provider.tf', \
build_target_path(project) + 'aws_provider.tf'])
run(['cp', '-f', build_commons_path(project) + 'terraform/variables.tf', \
build_target_path(project) + 'variables.tf'])
def tf_plan(project):
init(project)
tf = Terraform(working_dir=build_target_path(project))
tf.plan(capture_output=False, var=project_vars(project))
def tf_import(project):
init(project)
tf = Terraform(working_dir=build_target_path(project))
tf.import_cmd(tf_import_name(project), tf_import_resource(project), \
capture_output=False, var=project_vars(project))
def tf_apply(project, p_auto_approve=False):
init(project)
tf = Terraform(working_dir=build_target_path(project))
tf.apply(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project))
tf_output(project)
def tf_output(project):
init(project)
tf = Terraform(working_dir=build_target_path(project))
result = tf.output(json=IsFlagged)
with open(build_target_path(project) + OUTPUT_JSON, "w") as output_file:
output_file.write(json.dumps(result))
def tf_destroy(project, p_auto_approve=False):
init(project)
tf = Terraform(working_dir=build_target_path(project))
tf.destroy(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project))
def tf_read_output_json(project):
with open(build_target_path(project) + OUTPUT_JSON, 'r') as f:
return load(f)
def project_vars(project):
my_hetzner_api_key = hetzner_api_key(project)
my_module = project.name
ret = {'stage' : stage(project)}
# TODO: move to meissa specific part
if my_hetzner_api_key:
ret['hetzner_api_key'] = my_hetzner_api_key
if my_module:
ret['module'] = my_module
return ret
def tf_import(self):
tf = self.init_client()
tf.import_cmd(self.tf_import_name, self.tf_import_resource,
capture_output=False, var=self.project_vars())

View file

@ -1,13 +1,22 @@
from .credential import gopass_credential_from_env_path
from .devops_terraform_build import DevopsTerraformBuild
class HetznerMixin:
def __init__(self, project, project_root_path, build_commons_path, module, stage):
super().__init__(self, project, project_root_path, build_commons_path, module, stage)
self.hetzner_api_key = gopass_credential_from_env_path('HETZNER_API_KEY_PATH')
def add_hetzner_mixin_config(config):
return config.update({'HetznerMixin':
{'HETZNER_API_KEY_PATH_ENVIRONMENT': 'HETZNER_API_KEY_PATH'}})
class HetznerMixin(DevopsTerraformBuild):
def __init__(self, project, config):
super().__init__(self, project, config)
hetzner_mixin_config = config['HetznerMixin']
self.hetzner_api_key = gopass_credential_from_env_path(
hetzner_mixin_config['HETZNER_API_KEY_PATH_ENVIRONMENT'])
def project_vars(self):
ret = super().project_vars
if self.hetzner_api_key:
ret['hetzner_api_key'] = self.hetzner_api_key
return ret
return ret