From 7637f65779f7aca7bddfd28d8ed5e03239f4796b Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 13:50:15 +0200 Subject: [PATCH 001/173] [Sip-CI] Update Type Hints --- src/main/python/ddadevops/domain/common.py | 2 +- src/main/python/ddadevops/domain/release.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index b077512..d944d32 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -47,7 +47,7 @@ class Devops(Validateable): self.name = module self.build_dir_name = build_dir_name # Deprecated - no longer use generic stack ... - self.stack = {} + self.stack : dict = {} @deprecation.deprecated(deprecated_in="3.2") # use .name instead diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index c340154..040db05 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -25,7 +25,7 @@ class Version(): self.version_string: Optional[str | None] = None self.is_snapshot: Optional[bool | None] = None - def increment(self, release_type: ReleaseType): + def increment(self, release_type: ReleaseType | None): self.is_snapshot = False match release_type: case ReleaseType.BUMP: @@ -51,7 +51,7 @@ class Version(): self.version_string += "-SNAPSHOT" return self.version_string - def create_release_version(self, release_type: ReleaseType): + def create_release_version(self, release_type: ReleaseType | None): release_version = Version(self.id, self.version_list.copy()) release_version.is_snapshot = self.is_snapshot release_version.increment(release_type) @@ -98,9 +98,9 @@ class Release(Validateable): self.devops = devops self.main_branch = main_branch self.config_file = config_file - self.release_context = None + self.release_context: ReleaseContext | None = None - def set_release_context(self, set_release_context: ReleaseContext) -> None: + def set_release_context(self, set_release_context: ReleaseContext): self.release_context = set_release_context def release_version(self): From f4fed5293abc3b241f6ad29312c606edf0dd8027 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 13:33:54 +0200 Subject: [PATCH 002/173] Add and use throw_if_invalid function Validates objects and raises an error if they are invalid --- src/main/python/ddadevops/domain/common.py | 5 +++++ .../python/ddadevops/infrastructure/release_mixin/repo.py | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index d944d32..a5f1fc8 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -18,6 +18,11 @@ class Validateable: def is_valid(self) -> bool: return len(self.validate()) < 1 + + def throw_if_invalid(self): + if not self.is_valid(): + issues = '\n'.join(self.validate()) + raise ValueError(f"Invalid Validateable: {issues}") class DnsRecord(Validateable): diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 4ab3ae5..2bfbb37 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -124,7 +124,5 @@ class ReleaseContextRepository: self.version_repository.get_version(), self.main_branch, ) - if not result.is_valid(): - issues = '\n'.join(result.validate()) - raise ValueError(f"invalid release: {issues}") + result.throw_if_invalid() return result From d801a71b49e795dc69a349d0e7c3ec48c4d3a6a8 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 13:38:22 +0200 Subject: [PATCH 003/173] Make ReleaseContextRepository stateless --- .../python/ddadevops/infrastructure/release_mixin/repo.py | 7 ++----- src/main/python/ddadevops/release_mixin.py | 5 +++-- src/test/python/release_mixin/test_infrastructure.py | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 2bfbb37..308d6c0 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -106,23 +106,20 @@ class ReleaseTypeRepository: raise ValueError("No valid api passed to ReleaseTypeRepository") -# TODO: Repo has state & repository should exist only for AggregateRoot class ReleaseContextRepository: def __init__( self, version_repository: VersionRepository, release_type_repository: ReleaseTypeRepository, - main_branch: str, ): self.version_repository = version_repository self.release_type_repository = release_type_repository - self.main_branch = main_branch - def get_release(self) -> ReleaseContext: + def get_release(self, main_branch: str) -> ReleaseContext: result = ReleaseContext( self.release_type_repository.get_release_type(), self.version_repository.get_version(), - self.main_branch, + main_branch, ) result.throw_if_invalid() return result diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index c992a1f..c7eddb7 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -9,6 +9,7 @@ class ReleaseMixin(DevopsBuild): def __init__(self, project: Project, release: Release): super().__init__(project, devops=release.devops) self.repo.set_release(self.project, release) + self.main_branch = release.main_branch git_api = GitApi() environment_api = EnvironmentApi() @@ -21,13 +22,13 @@ class ReleaseMixin(DevopsBuild): release_type_repo = ReleaseTypeRepository.from_git(git_api) version_repo = VersionRepository(release.config_file) - self.release_repo = ReleaseContextRepository(version_repo, release_type_repo, release.main_branch) + self.release_repo = ReleaseContextRepository(version_repo, release_type_repo) self.prepare_release_service = PrepareReleaseService() self.tag_and_push_release_service = TagAndPushReleaseService(git_api) def prepare_release(self): - release = self.release_repo.get_release() + release = self.release_repo.get_release(self.main_branch) self.prepare_release_service.write_and_commit_release(release, self.release_repo.version_repository) self.prepare_release_service.write_and_commit_bump(release, self.release_repo.version_repository) diff --git a/src/test/python/release_mixin/test_infrastructure.py b/src/test/python/release_mixin/test_infrastructure.py index 5bfe27d..6233c64 100644 --- a/src/test/python/release_mixin/test_infrastructure.py +++ b/src/test/python/release_mixin/test_infrastructure.py @@ -22,9 +22,9 @@ def test_release_repository(tmp_path): release_type_repo = ReleaseTypeRepository.from_git(MockGitApi('MINOR test')) # test - sut = ReleaseContextRepository(version_repo, release_type_repo, 'main') + sut = ReleaseContextRepository(version_repo, release_type_repo) - release = sut.get_release() + release = sut.get_release('main') assert release is not None From 953e05ecfb4fbe837ab51dc5c7a58ea7a44dee46 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:27:41 +0200 Subject: [PATCH 004/173] Remove unused imports --- src/main/python/ddadevops/devops_build.py | 1 - src/main/python/ddadevops/domain/image.py | 1 - src/main/python/ddadevops/domain/release.py | 1 - .../infrastructure/release_mixin/infrastructure_api.py | 1 - src/main/python/ddadevops/infrastructure/release_mixin/repo.py | 2 +- src/main/python/ddadevops/release_mixin.py | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index 36ddb84..a239ae2 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,5 +1,4 @@ from typing import Optional -from subprocess import run, CalledProcessError import deprecation from .domain import Devops from .infrastructure import ProjectRepository, FileApi diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index 49ca555..37a23aa 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -1,4 +1,3 @@ -from typing import Optional from .common import ( filter_none, Validateable, diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 040db05..882a8fc 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -2,7 +2,6 @@ from enum import Enum from typing import Optional from pathlib import Path from .common import ( - filter_none, Validateable, Devops, ) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index 09587b9..5d297b9 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -1,6 +1,5 @@ import json import re -import subprocess as sub from abc import ABC, abstractmethod from typing import Optional from pathlib import Path diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 308d6c0..4fd65df 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -1,4 +1,4 @@ -from typing import Optional + from src.main.python.ddadevops.domain import ( ReleaseContext, Version, diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index c7eddb7..597ee6a 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -1,4 +1,3 @@ -from typing import Optional from pybuilder.core import Project from src.main.python.ddadevops.devops_build import DevopsBuild from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, ReleaseTypeRepository, VersionRepository, GitApi, EnvironmentApi From 0214cdd99a527341f05081b149f95b52cfedc372 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:33:49 +0200 Subject: [PATCH 005/173] Update Ignore Settings --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c2920c..023a56b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d C0301,W0614,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ pytest: stage: lint&test From 9db91c0a16c40b656af23ef9460e882498f6c94f Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:41:37 +0200 Subject: [PATCH 006/173] Fix pylint messages for repo --- .../infrastructure/release_mixin/repo.py | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 4fd65df..2dab0c1 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -23,9 +23,8 @@ class VersionRepository: def write_file(self, version_string): if self.file_handler is None: - raise Exception("Version was not created by load_file method.") - else: - self.file_handler.write(version_string) + raise RuntimeError("Version was not created by load_file method.") + self.file_handler.write(version_string) def parse_file(self): version_list, is_snapshot = self.file_handler.parse() @@ -53,29 +52,30 @@ class ReleaseTypeRepository: @classmethod def from_git(cls, git_api: GitApi): - releaseTypeRepo = cls(git_api=git_api) - releaseTypeRepo.get_from_git = True - return releaseTypeRepo + release_type_repo = cls(git_api=git_api) + release_type_repo.get_from_git = True + return release_type_repo @classmethod def from_environment(cls, environment_api: EnvironmentApi): - releaseTypeRepo = cls(environment_api=environment_api) - releaseTypeRepo.get_from_env = True - return releaseTypeRepo + release_type_repo = cls(environment_api=environment_api) + release_type_repo.get_from_env = True + return release_type_repo def __get_release_type_git(self) -> ReleaseType | None: latest_commit = self.git_api.get_latest_commit() - if ReleaseType.MAJOR.name in latest_commit.upper(): - return ReleaseType.MAJOR - elif ReleaseType.MINOR.name in latest_commit.upper(): - return ReleaseType.MINOR - elif ReleaseType.PATCH.name in latest_commit.upper(): - return ReleaseType.PATCH - elif ReleaseType.SNAPSHOT.name in latest_commit.upper(): - return ReleaseType.SNAPSHOT - else: - return None + match latest_commit.upper(): + case ReleaseType.MAJOR.name: + return ReleaseType.MAJOR + case ReleaseType.MINOR.name: + return ReleaseType.MINOR + case ReleaseType.PATCH.name: + return ReleaseType.PATCH + case ReleaseType.SNAPSHOT.name: + return ReleaseType.SNAPSHOT + case _: + return None def __get_release_type_environment(self) -> ReleaseType | None: release_name = self.environment_api.get( @@ -86,24 +86,25 @@ class ReleaseTypeRepository: raise ValueError( "Release Name not found. Is the Environment correctly configured?" ) - elif ReleaseType.MAJOR.name in release_name.upper(): - return ReleaseType.MAJOR - elif ReleaseType.MINOR.name in release_name.upper(): - return ReleaseType.MINOR - elif ReleaseType.PATCH.name in release_name.upper(): - return ReleaseType.PATCH - elif ReleaseType.SNAPSHOT.name in release_name.upper(): - return ReleaseType.SNAPSHOT - else: - return None + + match release_name.upper(): + case ReleaseType.MAJOR.name: + return ReleaseType.MAJOR + case ReleaseType.MINOR.name: + return ReleaseType.MINOR + case ReleaseType.PATCH.name: + return ReleaseType.PATCH + case ReleaseType.SNAPSHOT.name: + return ReleaseType.SNAPSHOT + case _: + return None def get_release_type(self) -> ReleaseType | None: if self.get_from_git: return self.__get_release_type_git() - elif self.get_from_env: + if self.get_from_env: return self.__get_release_type_environment() - else: - raise ValueError("No valid api passed to ReleaseTypeRepository") + raise ValueError("No valid api passed to ReleaseTypeRepository") class ReleaseContextRepository: From 2463d3667498a8a18415cd55d2df69abdad4bb68 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:42:36 +0200 Subject: [PATCH 007/173] [Skip-CI] More specific exception --- .../infrastructure/release_mixin/infrastructure_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index 5d297b9..33bb455 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -25,7 +25,7 @@ class FileHandler(ABC): case '.py': file_handler = PythonFileHandler() case _: - raise Exception( + raise RuntimeError( f'The file type "{config_file_type}" is not implemented') # TODO: Attribute is only set in classmethod. Should this be initialized outside of this class? file_handler.config_file_path = file_path From 481d20a14ce5911ef9a1abf10a4d8bd12a5efc43 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 28 Apr 2023 14:49:23 +0200 Subject: [PATCH 008/173] introduce devops factory --- doc/architecture/Domain.md | 5 ++- src/main/python/ddadevops/domain/__init__.py | 2 +- src/main/python/ddadevops/domain/common.py | 39 ++++++++++++++++++++ src/test/python/domain/test_devops.py | 16 ++++++++ src/test/python/domain/test_domain.py | 31 ++++++---------- 5 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 src/test/python/domain/test_devops.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index dd5687f..92d12dc 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -3,6 +3,7 @@ ```mermaid classDiagram class Devops { + <> stage name project_root_path @@ -42,8 +43,10 @@ classDiagram current_branch } + Devops *-- Image: spcialized_build + Devops *-- C4k: spcialized_build + Devops *-- Release: release C4k *-- DnsRecord - Image *-- Devops Release *-- "0..1" ReleaseContext ``` diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 2e4f6b1..9601d03 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,4 +1,4 @@ -from .common import Validateable, DnsRecord, Devops +from .common import Validateable, DnsRecord, Devops, DevopsFactory from .image import Image from .c4k import C4k from .release import Release, ReleaseContext, ReleaseType, Version, EnvironmentKeys diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index b077512..d038ec6 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,4 +1,6 @@ import deprecation +from enum import Enum +# TODO: remove logging from domain! import logging from typing import List @@ -70,3 +72,40 @@ class Devops(Validateable): for key in keys: result[key] = self.__get__(key) return result + + +class BuildType(Enum): + IMAGE = 0 + C4K = 1 + +class DevopsFactory(): + def __init__(self): + pass + + def build_devops(self, input) -> Devops: + build_type = BuildType[input['build_type']] + specialized_build = None + if build_type == BuildType.IMAGE: + specialized_build = Image( + dockerhub_user=input['dockerhub_user'], + dockerhub_password=input['dockerhub_password'], + docker_publish_tag=input['tag'] + ) + elif build_type == BuildType.C4K: + pass + + devops = Devops( + stage=input['stage'], + project_root_path=input['project_root_path'], + module=input['module'], + name=input['name'], + specialized_build=specialized_build + ) + + # TODO: validate devops + + return devops + + def merge(input, autorization, context) -> dict: + pass + diff --git a/src/test/python/domain/test_devops.py b/src/test/python/domain/test_devops.py new file mode 100644 index 0000000..3889454 --- /dev/null +++ b/src/test/python/domain/test_devops.py @@ -0,0 +1,16 @@ +import pytest +from src.main.python.ddadevops.domain.common import ( + Devops, + DevopsFactory, +) + +def test_devops_factory(): + with pytest.raises(Exception): + DevopsFactory().build_devops({'build_type': 'NOTEXISTING'}) + + +def test_devops_buildpath(): + sut = Devops( + stage="test", project_root_path="../../..", module="cloud", name="meissa" + ) + assert "../../../target/meissa/cloud" == sut.build_path() diff --git a/src/test/python/domain/test_domain.py b/src/test/python/domain/test_domain.py index b94b492..230e130 100644 --- a/src/test/python/domain/test_domain.py +++ b/src/test/python/domain/test_domain.py @@ -62,26 +62,6 @@ def test_should_validate_DnsRecord(): sut = DnsRecord("name", ipv6="1::") assert sut.is_valid() - -def test_devops_buildpath(): - sut = Devops( - stage="test", project_root_path="../../..", module="cloud", name="meissa" - ) - assert "../../../target/meissa/cloud" == sut.build_path() - - -def test_devops_build_commons_path(): - devops = Devops( - stage="test", project_root_path="../../..", module="cloud", name="meissa" - ) - sut = Image( - dockerhub_user="user", - dockerhub_password="password", - devops = devops, - ) - assert "docker/" == sut.docker_build_commons_path() - - def test_c4k_build_should_update_fqdn(): project_config = { "stage": "test", @@ -224,3 +204,14 @@ def test_release(tmp_path): sut.set_release_context(ReleaseContext(ReleaseType.MINOR, Version("id", [1,2,3]), "main")) assert sut.is_valid() + +def test_devops_build_commons_path(): + devops = Devops( + stage="test", project_root_path="../../..", module="cloud", name="meissa" + ) + sut = Image( + dockerhub_user="user", + dockerhub_password="password", + devops = devops, + ) + assert "docker/" == sut.docker_build_commons_path() From c70daa346d3d6a6e687d362d697888e9ec8ec312 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 14:50:35 +0200 Subject: [PATCH 009/173] Resolve pylint errors in release.py --- .gitlab-ci.yml | 2 +- src/main/python/ddadevops/domain/release.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 023a56b..6d8e287 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0719,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ pytest: stage: lint&test diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 882a8fc..2c27b4a 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -18,8 +18,8 @@ class EnvironmentKeys(Enum): class Version(): - def __init__(self, id: Path, version_list: list): - self.id = id + def __init__(self, path: Path, version_list: list): + self.path = path self.version_list = version_list self.version_string: Optional[str | None] = None self.is_snapshot: Optional[bool | None] = None @@ -42,7 +42,7 @@ class Version(): self.version_list[ReleaseType.MINOR.value] = 0 self.version_list[ReleaseType.MAJOR.value] += 1 case None: - raise Exception("Release Type was not set!") + raise RuntimeError("Release Type was not set!") def get_version_string(self) -> str: self.version_string = ".".join([str(x) for x in self.version_list]) @@ -51,13 +51,13 @@ class Version(): return self.version_string def create_release_version(self, release_type: ReleaseType | None): - release_version = Version(self.id, self.version_list.copy()) + release_version = Version(self.path, self.version_list.copy()) release_version.is_snapshot = self.is_snapshot release_version.increment(release_type) return release_version def create_bump_version(self): - bump_version = Version(self.id, self.version_list.copy()) + bump_version = Version(self.path, self.version_list.copy()) bump_version.is_snapshot = self.is_snapshot bump_version.increment(ReleaseType.BUMP) return bump_version @@ -99,9 +99,9 @@ class Release(Validateable): self.config_file = config_file self.release_context: ReleaseContext | None = None - def set_release_context(self, set_release_context: ReleaseContext): + def set_release_context(self, set_release_context: ReleaseContext): self.release_context = set_release_context - + def release_version(self): return self.release_context.release_version() From 837b58d5b893d3deddbf87c67f3f7b99d4761c0d Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 14:57:19 +0200 Subject: [PATCH 010/173] Resolve pylint errors in common.py --- .gitlab-ci.yml | 2 +- src/main/python/ddadevops/domain/common.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d8e287..1abb7b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0719,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0719,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ pytest: stage: lint&test diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index a5f1fc8..5ffdb14 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,6 +1,6 @@ -import deprecation -import logging from typing import List +import logging +import deprecation def filter_none(list_to_filter): return [x for x in list_to_filter if x is not None] @@ -10,15 +10,15 @@ class Validateable: value = self.__dict__[field_name] if value is None or value == "": return [f"Field '{field_name}' must not be empty."] - else: - return [] + + return [] def validate(self) -> List[str]: return [] def is_valid(self) -> bool: return len(self.validate()) < 1 - + def throw_if_invalid(self): if not self.is_valid(): issues = '\n'.join(self.validate()) @@ -46,7 +46,7 @@ class Devops(Validateable): self.stage = stage self.name = name self.project_root_path = project_root_path - logging.warn(f"Set project root in DevOps {self.project_root_path}") + logging.warning(f"Set project root in DevOps {self.project_root_path}") self.module = module if not name: self.name = module @@ -56,22 +56,23 @@ class Devops(Validateable): @deprecation.deprecated(deprecated_in="3.2") # use .name instead + #pylint: disable=method-hidden def name(self): return self.name def build_path(self): path = [self.project_root_path, self.build_dir_name, self.name, self.module] - logging.warn(f"Set project build_path in Devops {path}") + logging.warning(f"Set project build_path in Devops {path}") return "/".join(filter_none(path)) def __put__(self, key, value): self.stack[key] = value - def __get__(self, key): + def __get(self, key): return self.stack[key] def __get_keys__(self, keys): result = {} for key in keys: - result[key] = self.__get__(key) + result[key] = self.__get(key) return result From af0d6cc43e8e9f8037846767c87e1e590ba80d0d Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 15:01:03 +0200 Subject: [PATCH 011/173] Resolve pylint errors in infrastructure_api.py --- .../release_mixin/infrastructure_api.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index 33bb455..f65140e 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -46,7 +46,7 @@ class JsonFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as json_file: + with open(self.config_file_path, 'r', encoding='utf-8') as json_file: json_version = json.load(json_file)['version'] is_snapshot = False if '-SNAPSHOT' in json_version: @@ -56,7 +56,7 @@ class JsonFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as json_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as json_file: json_data = json.load(json_file) json_data['version'] = version_string json_file.seek(0) @@ -69,7 +69,7 @@ class GradleFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as gradle_file: + with open(self.config_file_path, 'r', encoding='utf-8') as gradle_file: contents = gradle_file.read() version_line = re.search("\nversion = .*", contents) exception = Exception("Version not found in gradle file") @@ -93,7 +93,7 @@ class GradleFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as gradle_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as gradle_file: contents = gradle_file.read() version_substitute = re.sub( '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents) @@ -107,7 +107,7 @@ class PythonFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as python_file: + with open(self.config_file_path, 'r', encoding='utf-8') as python_file: contents = python_file.read() version_line = re.search("\nversion = .*\n", contents) exception = Exception("Version not found in gradle file") @@ -131,7 +131,7 @@ class PythonFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as python_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as python_file: contents = python_file.read() version_substitute = re.sub( '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents) @@ -145,7 +145,7 @@ class ClojureFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as clj_file: + with open(self.config_file_path, 'r', encoding='utf-8') as clj_file: contents = clj_file.read() version_line = re.search("^\\(defproject .*\n", contents) exception = Exception("Version not found in clj file") @@ -169,7 +169,7 @@ class ClojureFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as clj_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as clj_file: clj_first = clj_file.readline() clj_rest = clj_file.read() version_substitute = re.sub( @@ -185,6 +185,7 @@ class GitApi(): def __init__(self): self.execution_api = ExecutionApi() + # pylint: disable=invalid-name def get_latest_n_commits(self, n: int): return self.execution_api.execute( f'git log --oneline --format="%s %b" -n {n}') @@ -203,11 +204,10 @@ class GitApi(): return self.execution_api.execute('git describe --tags --abbrev=0') def get_current_branch(self): - self.execution_api.execute('git branch --show-current') - return ''.join(self.execution_api.stdout).rstrip() + return ''.join(self.execution_api.execute('git branch --show-current')).rstrip() def init(self, default_branch: str = "main"): - self.execution_api.execute(f'git init') + self.execution_api.execute('git init') self.execution_api.execute(f'git checkout -b {default_branch}') def set_user_config(self, email: str, name: str): From 93d6063e6e85ec4f98bf10b385e010cb0437c233 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 15:31:54 +0200 Subject: [PATCH 012/173] Remove Cyclic import --- src/main/python/ddadevops/infrastructure/release_mixin/repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 2dab0c1..588118a 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -5,7 +5,7 @@ from src.main.python.ddadevops.domain import ( ReleaseType, EnvironmentKeys, ) -from src.main.python.ddadevops.infrastructure.release_mixin import ( +from src.main.python.ddadevops.infrastructure.release_mixin.infrastructure_api import ( FileHandler, GitApi, EnvironmentApi, From 172d0b848f0491e99d1487f759b9d95d79d83f4d Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 15:32:15 +0200 Subject: [PATCH 013/173] Format --- src/main/python/ddadevops/domain/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 5ffdb14..fd15e27 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -24,7 +24,6 @@ class Validateable: issues = '\n'.join(self.validate()) raise ValueError(f"Invalid Validateable: {issues}") - class DnsRecord(Validateable): def __init__(self, fqdn, ipv4=None, ipv6=None): self.fqdn = fqdn From 9e519519cbfbc07d5274d4c371593ad989bc7484 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 15:33:06 +0200 Subject: [PATCH 014/173] [Skip-CI] Fix method override --- src/main/python/ddadevops/domain/image.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index 37a23aa..62e7c63 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -10,7 +10,6 @@ class Image(Validateable): dockerhub_user, dockerhub_password, devops: Devops, - build_dir_name="target", use_package_common_files=True, build_commons_path=None, docker_build_commons_dir_name="docker", @@ -25,5 +24,5 @@ class Image(Validateable): self.devops = devops def docker_build_commons_path(self): - list = [self.build_commons_path, self.docker_build_commons_dir_name] - return "/".join(filter_none(list)) + "/" + commons_path = [self.build_commons_path, self.docker_build_commons_dir_name] + return "/".join(filter_none(commons_path)) + "/" From 79de9073964e02514c6c8df0a96e6bcd41fa586e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 28 Apr 2023 17:49:28 +0200 Subject: [PATCH 015/173] refactor to domain-object creation by dict --- src/main/python/ddadevops/devops_build.py | 19 ++--- .../python/ddadevops/devops_image_build.py | 19 +---- src/main/python/ddadevops/domain/__init__.py | 3 +- src/main/python/ddadevops/domain/common.py | 71 +++++-------------- .../python/ddadevops/domain/devops_factory.py | 32 +++++++++ src/main/python/ddadevops/domain/image.py | 31 ++++---- src/test/python/domain/test_devops.py | 6 -- src/test/python/domain/test_devops_factory.py | 27 +++++++ src/test/python/domain/test_helper.py | 9 +++ 9 files changed, 110 insertions(+), 107 deletions(-) create mode 100644 src/main/python/ddadevops/domain/devops_factory.py create mode 100644 src/test/python/domain/test_devops_factory.py create mode 100644 src/test/python/domain/test_helper.py diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index cc3f860..19f2ab5 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,7 +1,9 @@ from typing import Optional from subprocess import run, CalledProcessError import deprecation -from .domain import Devops +from .domain import ( + Devops, DevopsFactory +) from .infrastructure import ProjectRepository, FileApi @@ -36,21 +38,12 @@ def get_tag_from_latest_commit(): class DevopsBuild: - def __init__(self, project, config: Optional[dict] = None, devops: Optional[Devops] = None): + def __init__(self, project, input: dict): self.project = project self.file_api = FileApi() + self.devops_factory = DevopsFactory() self.repo = ProjectRepository() - if not devops: - if not config: - raise ValueError("Build parameters could not be set!") - devops = Devops( - stage=config["stage"], - project_root_path=config["project_root_path"], - module=config["module"], - name=project.name, - build_dir_name=config["build_dir_name"], - ) - + devops = self.devops_factory.build_devops(input) self.repo.set_devops(self.project, devops) self.repo.set_build(self.project, self) diff --git a/src/main/python/ddadevops/devops_image_build.py b/src/main/python/ddadevops/devops_image_build.py index a5e6e65..f1a2e30 100644 --- a/src/main/python/ddadevops/devops_image_build.py +++ b/src/main/python/ddadevops/devops_image_build.py @@ -33,24 +33,9 @@ def create_devops_docker_build_config( class DevopsImageBuild(DevopsBuild): - def __init__(self, project, config: Optional[dict] = None, image: Optional[Image] = None): + def __init__(self, project, input: dict): + super().__init__(project, input) self.image_build_service = ImageBuildService() - if not image: - if not config: - raise ValueError("Image parameters could not be set.") - super().__init__(project, config=config) - image = Image( - dockerhub_user=config["dockerhub_user"], - dockerhub_password=config["dockerhub_password"], - devops=self.repo.get_devops(project), - use_package_common_files=config["use_package_common_files"], - build_commons_path=config["build_commons_path"], - docker_build_commons_dir_name=config["docker_build_commons_dir_name"], - docker_publish_tag=config["docker_publish_tag"], - ) - else: - super().__init__(project, devops=image.devops) - self.repo.set_docker(self.project, image) def initialize_build_dir(self): super().initialize_build_dir() diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 9601d03..d6b5a11 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,4 +1,5 @@ -from .common import Validateable, DnsRecord, Devops, DevopsFactory +from .common import Validateable, DnsRecord, Devops +from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k from .release import Release, ReleaseContext, ReleaseType, Version, EnvironmentKeys diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index d038ec6..98bb33c 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,7 +1,5 @@ import deprecation from enum import Enum -# TODO: remove logging from domain! -import logging from typing import List def filter_none(list_to_filter): @@ -37,30 +35,28 @@ class DnsRecord(Validateable): class Devops(Validateable): - def __init__( - self, stage: str, project_root_path: str, module: str, name: str | None =None, build_dir_name: str="target" - ): - self.stage = stage - self.name = name - self.project_root_path = project_root_path - logging.warn(f"Set project root in DevOps {self.project_root_path}") - self.module = module - if not name: - self.name = module - self.build_dir_name = build_dir_name - # Deprecated - no longer use generic stack ... - self.stack = {} - - @deprecation.deprecated(deprecated_in="3.2") - # use .name instead - def name(self): - return self.name + def __init__(self, input: dict, specialized_build: Validateable): + self.stage = input.get('stage') + self.project_root_path = input.get('project_root_path') + self.module = input.get('module') + self.name = input.get('name', self.module) + self.build_dir_name = input.get('build_dir_name', 'target') + self.specialized_build=specialized_build def build_path(self): path = [self.project_root_path, self.build_dir_name, self.name, self.module] - logging.warn(f"Set project build_path in Devops {path}") return "/".join(filter_none(path)) + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("stage") + result += self.__validate_is_not_empty__("project_root_path") + result += self.__validate_is_not_empty__("module") + result += self.__validate_is_not_empty__("specialized_build") + if self.specialized_build: + result += self.specialized_build.validate() + return result + def __put__(self, key, value): self.stack[key] = value @@ -73,39 +69,6 @@ class Devops(Validateable): result[key] = self.__get__(key) return result - class BuildType(Enum): IMAGE = 0 C4K = 1 - -class DevopsFactory(): - def __init__(self): - pass - - def build_devops(self, input) -> Devops: - build_type = BuildType[input['build_type']] - specialized_build = None - if build_type == BuildType.IMAGE: - specialized_build = Image( - dockerhub_user=input['dockerhub_user'], - dockerhub_password=input['dockerhub_password'], - docker_publish_tag=input['tag'] - ) - elif build_type == BuildType.C4K: - pass - - devops = Devops( - stage=input['stage'], - project_root_path=input['project_root_path'], - module=input['module'], - name=input['name'], - specialized_build=specialized_build - ) - - # TODO: validate devops - - return devops - - def merge(input, autorization, context) -> dict: - pass - diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py new file mode 100644 index 0000000..61a3369 --- /dev/null +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -0,0 +1,32 @@ +import deprecation +from enum import Enum +from typing import List +from .common import ( + Devops, + BuildType +) +from .image import ( + Image, +) + +class DevopsFactory(): + def __init__(self): + pass + + def build_devops(self, input) -> Devops: + build_type = BuildType[input['build_type']] + specialized_build = None + if build_type == BuildType.IMAGE: + specialized_build = Image(input) + elif build_type == BuildType.C4K: + pass + + devops = Devops(input, specialized_build=specialized_build) + + # TODO: validate devops + + return devops + + def merge(input, autorization, context) -> dict: + pass + diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index 49ca555..4a67ee4 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import (Optional, List) from .common import ( filter_none, Validateable, @@ -8,22 +8,21 @@ from .common import ( class Image(Validateable): def __init__( self, - dockerhub_user, - dockerhub_password, - devops: Devops, - build_dir_name="target", - use_package_common_files=True, - build_commons_path=None, - docker_build_commons_dir_name="docker", - docker_publish_tag=None, + input, ): - self.dockerhub_user = dockerhub_user - self.dockerhub_password = dockerhub_password - self.use_package_common_files = use_package_common_files - self.build_commons_path = build_commons_path - self.docker_build_commons_dir_name = docker_build_commons_dir_name - self.docker_publish_tag = docker_publish_tag - self.devops = devops + self.dockerhub_user=input.get('dockerhub_user') + self.dockerhub_password=input.get('dockerhub_password') + self.docker_publish_tag=input.get('docker_publish_tag') + self.build_commons_path = input.get('build_commons_path') + self.docker_publish_tag = input.get('docker_publish_tag') + self.use_package_common_files = input.get('use_package_common_files', True) + self.docker_build_commons_dir_name = input.get('docker_build_commons_dir_name', 'docker') + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("dockerhub_user") + result += self.__validate_is_not_empty__("dockerhub_password") + return result def docker_build_commons_path(self): list = [self.build_commons_path, self.docker_build_commons_dir_name] diff --git a/src/test/python/domain/test_devops.py b/src/test/python/domain/test_devops.py index 3889454..29f94ad 100644 --- a/src/test/python/domain/test_devops.py +++ b/src/test/python/domain/test_devops.py @@ -1,14 +1,8 @@ import pytest from src.main.python.ddadevops.domain.common import ( Devops, - DevopsFactory, ) -def test_devops_factory(): - with pytest.raises(Exception): - DevopsFactory().build_devops({'build_type': 'NOTEXISTING'}) - - def test_devops_buildpath(): sut = Devops( stage="test", project_root_path="../../..", module="cloud", name="meissa" diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py new file mode 100644 index 0000000..df7e15f --- /dev/null +++ b/src/test/python/domain/test_devops_factory.py @@ -0,0 +1,27 @@ +import pytest +from src.main.python.ddadevops.domain.devops_factory import ( + DevopsFactory, +) + +def test_devops_factory(): + with pytest.raises(Exception): + DevopsFactory().build_devops( + {'build_type': 'NOTEXISTING'} + ) + + with pytest.raises(Exception): + DevopsFactory().build_devops( + {'build_type': 'IMAGE'} + ) + + sut = DevopsFactory().build_devops( + {'build_type': 'IMAGE', + 'stage': 'test', + 'project_root_path': "../../..", + 'name': 'mybuild', + 'module': 'test_image', + 'dockerhub_user': 'dockerhub_user', + 'dockerhub_password': 'dockerhub_password', + 'docker_image_tag': 'docker_image_tag',} + ) + assert sut != None diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py new file mode 100644 index 0000000..aec7093 --- /dev/null +++ b/src/test/python/domain/test_helper.py @@ -0,0 +1,9 @@ +from src.main.python.ddadevops.domain import ( + DevopsFactory, Devops +) + +def build_devops(overrides: dict) -> Devops: + default = {} + input = default.copy() + input.update(overrides) + return DevopsFactory().build_devops(input) From 7e23080cff0dfe0021e2f1ca224aa804c55ab5cc Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 13:28:40 +0200 Subject: [PATCH 016/173] Refactor to C4kBuild --- src/main/python/ddadevops/__init__.py | 2 +- src/main/python/ddadevops/c4k_mixin.py | 4 +--- src/test/python/test_c4k_mixin.py | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index fc4363f..f62b93d 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -9,7 +9,7 @@ from .provs_k3s_mixin import ProvsK3sMixin, add_provs_k3s_mixin_config from .aws_rds_pg_mixin import AwsRdsPgMixin, add_aws_rds_pg_mixin_config from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config -from .c4k_mixin import C4kMixin, add_c4k_mixin_config +from .c4k_mixin import C4kBuild, add_c4k_mixin_config from .exoscale_mixin import ExoscaleMixin, add_exoscale_mixin_config from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config diff --git a/src/main/python/ddadevops/c4k_mixin.py b/src/main/python/ddadevops/c4k_mixin.py index 61edc8c..8a5f3f2 100644 --- a/src/main/python/ddadevops/c4k_mixin.py +++ b/src/main/python/ddadevops/c4k_mixin.py @@ -44,9 +44,7 @@ def add_c4k_mixin_config( ) return config - -#TODO: refactor this to C4kBuild -class C4kMixin(DevopsBuild): +class C4kBuild(DevopsBuild): def __init__(self, project, config): super().__init__(project, config) self.execution_api = ExecutionApi() diff --git a/src/test/python/test_c4k_mixin.py b/src/test/python/test_c4k_mixin.py index 59cf436..c4c2b17 100644 --- a/src/test/python/test_c4k_mixin.py +++ b/src/test/python/test_c4k_mixin.py @@ -1,9 +1,9 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain import DnsRecord -from src.main.python.ddadevops.c4k_mixin import C4kMixin, add_c4k_mixin_config +from src.main.python.ddadevops.c4k_mixin import C4kBuild, add_c4k_mixin_config -class MyC4kMixin(C4kMixin): +class MyC4kBuild(C4kBuild): pass def test_c4k_mixin(tmp_path): @@ -29,7 +29,7 @@ def test_c4k_mixin(tmp_path): assert project_config.get('C4kMixin') is not None - mixin = MyC4kMixin(project, project_config) + mixin = MyC4kBuild(project, project_config) mixin.initialize_build_dir() assert mixin.build_path() == f'{tmp_path_str}/{build_dir}/{project_name}/{module_name}' From f921a8a2aef82c5734f3d2332552fe934e245953 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 13:32:10 +0200 Subject: [PATCH 017/173] Remove deprecated function --- src/main/python/ddadevops/__init__.py | 2 +- src/main/python/ddadevops/devops_build.py | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index f62b93d..3cd3560 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -16,7 +16,7 @@ from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_dig from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config from .devops_image_build import DevopsImageBuild, create_devops_docker_build_config from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config -from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build, get_tag_from_latest_commit +from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build from .credential import gopass_password_from_path, gopass_field_from_path from .release_mixin import ReleaseMixin diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index 19f2ab5..7de1005 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -21,22 +21,6 @@ def create_devops_build_config( def get_devops_build(project): return project.get_property("devops_build") - -@deprecation.deprecated(deprecated_in="3.2") -# TODO: Remove from here! # pylint: disable=W0511 -def get_tag_from_latest_commit(): - try: - value = run( - "git describe --abbrev=0 --tags --exact-match", - shell=True, - capture_output=True, - check=True, - ) - return value.stdout.decode("UTF-8").rstrip() - except CalledProcessError: - return None - - class DevopsBuild: def __init__(self, project, input: dict): self.project = project From 62b6465d01c08f43acd487d170c67d87db84cbda Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 13:32:40 +0200 Subject: [PATCH 018/173] Remove unused files --- src/test/resources/alt_config.json | 4 --- src/test/resources/build.py | 49 ------------------------------ 2 files changed, 53 deletions(-) delete mode 100644 src/test/resources/alt_config.json delete mode 100644 src/test/resources/build.py diff --git a/src/test/resources/alt_config.json b/src/test/resources/alt_config.json deleted file mode 100644 index 359f087..0000000 --- a/src/test/resources/alt_config.json +++ /dev/null @@ -1,4 +0,0 @@ -// TODO: jem, zam - 2023_04_18: move this to an build-test repo ? -{ - "version": "123.125.1-SNAPSHOT" -} \ No newline at end of file diff --git a/src/test/resources/build.py b/src/test/resources/build.py deleted file mode 100644 index 641ae3a..0000000 --- a/src/test/resources/build.py +++ /dev/null @@ -1,49 +0,0 @@ -# TODO: jem, zam - 2023_04_18: move this to an build-test repo? - -import sys -import os -from pathlib import Path -from ddadevops import * - -# getting the name of the directory -# where the this file is present. -current = os.path.dirname(os.path.realpath(__file__)) - -# adding the current directory to -# the sys.path. -sys.path.append(current) - -# now we can import the module in the current -# directory. - -from pybuilder.core import task, init -from ddadevops import * -from release_mixin import ReleaseMixin, create_release_mixin_config - -CONFIG_FILE = Path('config.json') -MAIN_BRANCH = 'main' -STAGE = 'test' -PROJECT_ROOT_PATH = '.' -MODULE = 'test' -BUILD_DIR_NAME = "build_dir" - -class MyBuild(ReleaseMixin): - pass - -@init -def initialize(project): - project.build_depends_on('ddadevops>=3.1.2') - config = create_release_mixin_config(CONFIG_FILE, MAIN_BRANCH) - config.update({'stage': STAGE, - 'module': MODULE, - 'project_root_path': PROJECT_ROOT_PATH, - 'build_dir_name': BUILD_DIR_NAME}) - build = MyBuild(project, config) - build.initialize_build_dir() - -@task -def release(project): - build = get_devops_build(project) - - build.prepare_release() - build.tag_and_push_release() From 35a5a69844ee3df5f7de869fdf01e774b304dc0c Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 13:50:15 +0200 Subject: [PATCH 019/173] [Sip-CI] Update Type Hints --- src/main/python/ddadevops/domain/release.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index c340154..040db05 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -25,7 +25,7 @@ class Version(): self.version_string: Optional[str | None] = None self.is_snapshot: Optional[bool | None] = None - def increment(self, release_type: ReleaseType): + def increment(self, release_type: ReleaseType | None): self.is_snapshot = False match release_type: case ReleaseType.BUMP: @@ -51,7 +51,7 @@ class Version(): self.version_string += "-SNAPSHOT" return self.version_string - def create_release_version(self, release_type: ReleaseType): + def create_release_version(self, release_type: ReleaseType | None): release_version = Version(self.id, self.version_list.copy()) release_version.is_snapshot = self.is_snapshot release_version.increment(release_type) @@ -98,9 +98,9 @@ class Release(Validateable): self.devops = devops self.main_branch = main_branch self.config_file = config_file - self.release_context = None + self.release_context: ReleaseContext | None = None - def set_release_context(self, set_release_context: ReleaseContext) -> None: + def set_release_context(self, set_release_context: ReleaseContext): self.release_context = set_release_context def release_version(self): From b6b283d2b298ee7bf361a941716f551421d9c6eb Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 13:33:54 +0200 Subject: [PATCH 020/173] Add and use throw_if_invalid function Validates objects and raises an error if they are invalid --- src/main/python/ddadevops/domain/common.py | 5 +++++ .../python/ddadevops/infrastructure/release_mixin/repo.py | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 98bb33c..ea8c551 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -18,6 +18,11 @@ class Validateable: def is_valid(self) -> bool: return len(self.validate()) < 1 + + def throw_if_invalid(self): + if not self.is_valid(): + issues = '\n'.join(self.validate()) + raise ValueError(f"Invalid Validateable: {issues}") class DnsRecord(Validateable): diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 4ab3ae5..2bfbb37 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -124,7 +124,5 @@ class ReleaseContextRepository: self.version_repository.get_version(), self.main_branch, ) - if not result.is_valid(): - issues = '\n'.join(result.validate()) - raise ValueError(f"invalid release: {issues}") + result.throw_if_invalid() return result From a25453cdbfd1b6deb4ec192f4acbff6739beaabc Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 13:38:22 +0200 Subject: [PATCH 021/173] Make ReleaseContextRepository stateless --- .../python/ddadevops/infrastructure/release_mixin/repo.py | 7 ++----- src/main/python/ddadevops/release_mixin.py | 5 +++-- src/test/python/release_mixin/test_infrastructure.py | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 2bfbb37..308d6c0 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -106,23 +106,20 @@ class ReleaseTypeRepository: raise ValueError("No valid api passed to ReleaseTypeRepository") -# TODO: Repo has state & repository should exist only for AggregateRoot class ReleaseContextRepository: def __init__( self, version_repository: VersionRepository, release_type_repository: ReleaseTypeRepository, - main_branch: str, ): self.version_repository = version_repository self.release_type_repository = release_type_repository - self.main_branch = main_branch - def get_release(self) -> ReleaseContext: + def get_release(self, main_branch: str) -> ReleaseContext: result = ReleaseContext( self.release_type_repository.get_release_type(), self.version_repository.get_version(), - self.main_branch, + main_branch, ) result.throw_if_invalid() return result diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index c992a1f..c7eddb7 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -9,6 +9,7 @@ class ReleaseMixin(DevopsBuild): def __init__(self, project: Project, release: Release): super().__init__(project, devops=release.devops) self.repo.set_release(self.project, release) + self.main_branch = release.main_branch git_api = GitApi() environment_api = EnvironmentApi() @@ -21,13 +22,13 @@ class ReleaseMixin(DevopsBuild): release_type_repo = ReleaseTypeRepository.from_git(git_api) version_repo = VersionRepository(release.config_file) - self.release_repo = ReleaseContextRepository(version_repo, release_type_repo, release.main_branch) + self.release_repo = ReleaseContextRepository(version_repo, release_type_repo) self.prepare_release_service = PrepareReleaseService() self.tag_and_push_release_service = TagAndPushReleaseService(git_api) def prepare_release(self): - release = self.release_repo.get_release() + release = self.release_repo.get_release(self.main_branch) self.prepare_release_service.write_and_commit_release(release, self.release_repo.version_repository) self.prepare_release_service.write_and_commit_bump(release, self.release_repo.version_repository) diff --git a/src/test/python/release_mixin/test_infrastructure.py b/src/test/python/release_mixin/test_infrastructure.py index 5bfe27d..6233c64 100644 --- a/src/test/python/release_mixin/test_infrastructure.py +++ b/src/test/python/release_mixin/test_infrastructure.py @@ -22,9 +22,9 @@ def test_release_repository(tmp_path): release_type_repo = ReleaseTypeRepository.from_git(MockGitApi('MINOR test')) # test - sut = ReleaseContextRepository(version_repo, release_type_repo, 'main') + sut = ReleaseContextRepository(version_repo, release_type_repo) - release = sut.get_release() + release = sut.get_release('main') assert release is not None From aa34558040e04dcf6604690af644afaf33148e95 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:27:41 +0200 Subject: [PATCH 022/173] Remove unused imports --- src/main/python/ddadevops/devops_build.py | 1 - src/main/python/ddadevops/domain/release.py | 1 - .../infrastructure/release_mixin/infrastructure_api.py | 1 - src/main/python/ddadevops/infrastructure/release_mixin/repo.py | 2 +- src/main/python/ddadevops/release_mixin.py | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index 7de1005..b9e83f6 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,5 +1,4 @@ from typing import Optional -from subprocess import run, CalledProcessError import deprecation from .domain import ( Devops, DevopsFactory diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 040db05..882a8fc 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -2,7 +2,6 @@ from enum import Enum from typing import Optional from pathlib import Path from .common import ( - filter_none, Validateable, Devops, ) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index 09587b9..5d297b9 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -1,6 +1,5 @@ import json import re -import subprocess as sub from abc import ABC, abstractmethod from typing import Optional from pathlib import Path diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 308d6c0..4fd65df 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -1,4 +1,4 @@ -from typing import Optional + from src.main.python.ddadevops.domain import ( ReleaseContext, Version, diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index c7eddb7..597ee6a 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -1,4 +1,3 @@ -from typing import Optional from pybuilder.core import Project from src.main.python.ddadevops.devops_build import DevopsBuild from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, ReleaseTypeRepository, VersionRepository, GitApi, EnvironmentApi From 6da4b3c27a0471cf67ed644bd1572ad3915d6fd5 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:33:49 +0200 Subject: [PATCH 023/173] Update Ignore Settings --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c2920c..023a56b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d C0301,W0614,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ pytest: stage: lint&test From 82279011fa56ea20a367b94e2929f981edf0daf6 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:41:37 +0200 Subject: [PATCH 024/173] Fix pylint messages for repo --- .../infrastructure/release_mixin/repo.py | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 4fd65df..2dab0c1 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -23,9 +23,8 @@ class VersionRepository: def write_file(self, version_string): if self.file_handler is None: - raise Exception("Version was not created by load_file method.") - else: - self.file_handler.write(version_string) + raise RuntimeError("Version was not created by load_file method.") + self.file_handler.write(version_string) def parse_file(self): version_list, is_snapshot = self.file_handler.parse() @@ -53,29 +52,30 @@ class ReleaseTypeRepository: @classmethod def from_git(cls, git_api: GitApi): - releaseTypeRepo = cls(git_api=git_api) - releaseTypeRepo.get_from_git = True - return releaseTypeRepo + release_type_repo = cls(git_api=git_api) + release_type_repo.get_from_git = True + return release_type_repo @classmethod def from_environment(cls, environment_api: EnvironmentApi): - releaseTypeRepo = cls(environment_api=environment_api) - releaseTypeRepo.get_from_env = True - return releaseTypeRepo + release_type_repo = cls(environment_api=environment_api) + release_type_repo.get_from_env = True + return release_type_repo def __get_release_type_git(self) -> ReleaseType | None: latest_commit = self.git_api.get_latest_commit() - if ReleaseType.MAJOR.name in latest_commit.upper(): - return ReleaseType.MAJOR - elif ReleaseType.MINOR.name in latest_commit.upper(): - return ReleaseType.MINOR - elif ReleaseType.PATCH.name in latest_commit.upper(): - return ReleaseType.PATCH - elif ReleaseType.SNAPSHOT.name in latest_commit.upper(): - return ReleaseType.SNAPSHOT - else: - return None + match latest_commit.upper(): + case ReleaseType.MAJOR.name: + return ReleaseType.MAJOR + case ReleaseType.MINOR.name: + return ReleaseType.MINOR + case ReleaseType.PATCH.name: + return ReleaseType.PATCH + case ReleaseType.SNAPSHOT.name: + return ReleaseType.SNAPSHOT + case _: + return None def __get_release_type_environment(self) -> ReleaseType | None: release_name = self.environment_api.get( @@ -86,24 +86,25 @@ class ReleaseTypeRepository: raise ValueError( "Release Name not found. Is the Environment correctly configured?" ) - elif ReleaseType.MAJOR.name in release_name.upper(): - return ReleaseType.MAJOR - elif ReleaseType.MINOR.name in release_name.upper(): - return ReleaseType.MINOR - elif ReleaseType.PATCH.name in release_name.upper(): - return ReleaseType.PATCH - elif ReleaseType.SNAPSHOT.name in release_name.upper(): - return ReleaseType.SNAPSHOT - else: - return None + + match release_name.upper(): + case ReleaseType.MAJOR.name: + return ReleaseType.MAJOR + case ReleaseType.MINOR.name: + return ReleaseType.MINOR + case ReleaseType.PATCH.name: + return ReleaseType.PATCH + case ReleaseType.SNAPSHOT.name: + return ReleaseType.SNAPSHOT + case _: + return None def get_release_type(self) -> ReleaseType | None: if self.get_from_git: return self.__get_release_type_git() - elif self.get_from_env: + if self.get_from_env: return self.__get_release_type_environment() - else: - raise ValueError("No valid api passed to ReleaseTypeRepository") + raise ValueError("No valid api passed to ReleaseTypeRepository") class ReleaseContextRepository: From 7cace08a871257340bc4bad0466035dfe5110d9f Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 14:42:36 +0200 Subject: [PATCH 025/173] [Skip-CI] More specific exception --- .../infrastructure/release_mixin/infrastructure_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index 5d297b9..33bb455 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -25,7 +25,7 @@ class FileHandler(ABC): case '.py': file_handler = PythonFileHandler() case _: - raise Exception( + raise RuntimeError( f'The file type "{config_file_type}" is not implemented') # TODO: Attribute is only set in classmethod. Should this be initialized outside of this class? file_handler.config_file_path = file_path From bc7d8e1d87de72aca7ae092a822a5234900c4415 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 14:50:35 +0200 Subject: [PATCH 026/173] Resolve pylint errors in release.py --- .gitlab-ci.yml | 2 +- src/main/python/ddadevops/domain/release.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 023a56b..6d8e287 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0719,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ pytest: stage: lint&test diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 882a8fc..2c27b4a 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -18,8 +18,8 @@ class EnvironmentKeys(Enum): class Version(): - def __init__(self, id: Path, version_list: list): - self.id = id + def __init__(self, path: Path, version_list: list): + self.path = path self.version_list = version_list self.version_string: Optional[str | None] = None self.is_snapshot: Optional[bool | None] = None @@ -42,7 +42,7 @@ class Version(): self.version_list[ReleaseType.MINOR.value] = 0 self.version_list[ReleaseType.MAJOR.value] += 1 case None: - raise Exception("Release Type was not set!") + raise RuntimeError("Release Type was not set!") def get_version_string(self) -> str: self.version_string = ".".join([str(x) for x in self.version_list]) @@ -51,13 +51,13 @@ class Version(): return self.version_string def create_release_version(self, release_type: ReleaseType | None): - release_version = Version(self.id, self.version_list.copy()) + release_version = Version(self.path, self.version_list.copy()) release_version.is_snapshot = self.is_snapshot release_version.increment(release_type) return release_version def create_bump_version(self): - bump_version = Version(self.id, self.version_list.copy()) + bump_version = Version(self.path, self.version_list.copy()) bump_version.is_snapshot = self.is_snapshot bump_version.increment(ReleaseType.BUMP) return bump_version @@ -99,9 +99,9 @@ class Release(Validateable): self.config_file = config_file self.release_context: ReleaseContext | None = None - def set_release_context(self, set_release_context: ReleaseContext): + def set_release_context(self, set_release_context: ReleaseContext): self.release_context = set_release_context - + def release_version(self): return self.release_context.release_version() From e0b30adf9fc71ade15868c7440f63be5b4c1bf63 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 14:57:19 +0200 Subject: [PATCH 027/173] Resolve pylint errors in common.py --- .gitlab-ci.yml | 2 +- src/main/python/ddadevops/domain/common.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d8e287..1abb7b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W0719,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0719,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ pytest: stage: lint&test diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index ea8c551..901f6fc 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,6 +1,8 @@ import deprecation from enum import Enum from typing import List +import logging +import deprecation def filter_none(list_to_filter): return [x for x in list_to_filter if x is not None] @@ -10,15 +12,15 @@ class Validateable: value = self.__dict__[field_name] if value is None or value == "": return [f"Field '{field_name}' must not be empty."] - else: - return [] + + return [] def validate(self) -> List[str]: return [] def is_valid(self) -> bool: return len(self.validate()) < 1 - + def throw_if_invalid(self): if not self.is_valid(): issues = '\n'.join(self.validate()) @@ -65,13 +67,13 @@ class Devops(Validateable): def __put__(self, key, value): self.stack[key] = value - def __get__(self, key): + def __get(self, key): return self.stack[key] def __get_keys__(self, keys): result = {} for key in keys: - result[key] = self.__get__(key) + result[key] = self.__get(key) return result class BuildType(Enum): From 70b5607d8efc69a9b4eb38a4a85703e47e332214 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 28 Apr 2023 15:01:03 +0200 Subject: [PATCH 028/173] Resolve pylint errors in infrastructure_api.py --- .../release_mixin/infrastructure_api.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index 33bb455..f65140e 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -46,7 +46,7 @@ class JsonFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as json_file: + with open(self.config_file_path, 'r', encoding='utf-8') as json_file: json_version = json.load(json_file)['version'] is_snapshot = False if '-SNAPSHOT' in json_version: @@ -56,7 +56,7 @@ class JsonFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as json_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as json_file: json_data = json.load(json_file) json_data['version'] = version_string json_file.seek(0) @@ -69,7 +69,7 @@ class GradleFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as gradle_file: + with open(self.config_file_path, 'r', encoding='utf-8') as gradle_file: contents = gradle_file.read() version_line = re.search("\nversion = .*", contents) exception = Exception("Version not found in gradle file") @@ -93,7 +93,7 @@ class GradleFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as gradle_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as gradle_file: contents = gradle_file.read() version_substitute = re.sub( '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents) @@ -107,7 +107,7 @@ class PythonFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as python_file: + with open(self.config_file_path, 'r', encoding='utf-8') as python_file: contents = python_file.read() version_line = re.search("\nversion = .*\n", contents) exception = Exception("Version not found in gradle file") @@ -131,7 +131,7 @@ class PythonFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as python_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as python_file: contents = python_file.read() version_substitute = re.sub( '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents) @@ -145,7 +145,7 @@ class ClojureFileHandler(FileHandler): def parse(self) -> tuple[list[int], bool]: if self.config_file_path is None: raise ValueError("No file name given.") - with open(self.config_file_path, 'r') as clj_file: + with open(self.config_file_path, 'r', encoding='utf-8') as clj_file: contents = clj_file.read() version_line = re.search("^\\(defproject .*\n", contents) exception = Exception("Version not found in clj file") @@ -169,7 +169,7 @@ class ClojureFileHandler(FileHandler): return version, is_snapshot def write(self, version_string): - with open(self.config_file_path, 'r+') as clj_file: + with open(self.config_file_path, 'r+', encoding='utf-8') as clj_file: clj_first = clj_file.readline() clj_rest = clj_file.read() version_substitute = re.sub( @@ -185,6 +185,7 @@ class GitApi(): def __init__(self): self.execution_api = ExecutionApi() + # pylint: disable=invalid-name def get_latest_n_commits(self, n: int): return self.execution_api.execute( f'git log --oneline --format="%s %b" -n {n}') @@ -203,11 +204,10 @@ class GitApi(): return self.execution_api.execute('git describe --tags --abbrev=0') def get_current_branch(self): - self.execution_api.execute('git branch --show-current') - return ''.join(self.execution_api.stdout).rstrip() + return ''.join(self.execution_api.execute('git branch --show-current')).rstrip() def init(self, default_branch: str = "main"): - self.execution_api.execute(f'git init') + self.execution_api.execute('git init') self.execution_api.execute(f'git checkout -b {default_branch}') def set_user_config(self, email: str, name: str): From 423166d8b9ea27cea889b214876f477ab4506438 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 15:31:54 +0200 Subject: [PATCH 029/173] Remove Cyclic import --- src/main/python/ddadevops/infrastructure/release_mixin/repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 2dab0c1..588118a 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -5,7 +5,7 @@ from src.main.python.ddadevops.domain import ( ReleaseType, EnvironmentKeys, ) -from src.main.python.ddadevops.infrastructure.release_mixin import ( +from src.main.python.ddadevops.infrastructure.release_mixin.infrastructure_api import ( FileHandler, GitApi, EnvironmentApi, From 40e09d246dcedf1bf9570eee48a7853e5b9d227f Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 15:32:15 +0200 Subject: [PATCH 030/173] Format --- src/main/python/ddadevops/domain/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 901f6fc..bc87184 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -26,7 +26,6 @@ class Validateable: issues = '\n'.join(self.validate()) raise ValueError(f"Invalid Validateable: {issues}") - class DnsRecord(Validateable): def __init__(self, fqdn, ipv4=None, ipv6=None): self.fqdn = fqdn From 806e1eb02dfa6b605d73fed97c5fe0da619eed55 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 28 Apr 2023 15:33:06 +0200 Subject: [PATCH 031/173] [Skip-CI] Fix method override --- src/main/python/ddadevops/domain/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index 4a67ee4..4344fb3 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -25,5 +25,5 @@ class Image(Validateable): return result def docker_build_commons_path(self): - list = [self.build_commons_path, self.docker_build_commons_dir_name] - return "/".join(filter_none(list)) + "/" + commons_path = [self.build_commons_path, self.docker_build_commons_dir_name] + return "/".join(filter_none(commons_path)) + "/" From 63409f03a99f12dc8adb1f83372fab373a17921b Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 28 Apr 2023 18:07:17 +0200 Subject: [PATCH 032/173] add domain validation --- src/main/python/ddadevops/domain/devops_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 61a3369..f99fd17 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -23,7 +23,7 @@ class DevopsFactory(): devops = Devops(input, specialized_build=specialized_build) - # TODO: validate devops + devops.throw_if_invalid() return devops From 1da73523d919919a719002c6fc71ce1f832f0f2d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 28 Apr 2023 18:07:55 +0200 Subject: [PATCH 033/173] add test_helper for improved testing & fix devops_test --- src/test/python/domain/test_devops.py | 5 ++--- src/test/python/domain/test_helper.py | 9 ++++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/test/python/domain/test_devops.py b/src/test/python/domain/test_devops.py index 29f94ad..0a881ea 100644 --- a/src/test/python/domain/test_devops.py +++ b/src/test/python/domain/test_devops.py @@ -1,10 +1,9 @@ import pytest +from .test_helper import build_devops from src.main.python.ddadevops.domain.common import ( Devops, ) def test_devops_buildpath(): - sut = Devops( - stage="test", project_root_path="../../..", module="cloud", name="meissa" - ) + sut = build_devops({'module': "cloud", 'name': "meissa"}) assert "../../../target/meissa/cloud" == sut.build_path() diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index aec7093..c93e3dd 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -3,7 +3,14 @@ from src.main.python.ddadevops.domain import ( ) def build_devops(overrides: dict) -> Devops: - default = {} + default = {'build_type': 'IMAGE', + 'stage': 'test', + 'project_root_path': "../../..", + 'name': 'mybuild', + 'module': 'test_image', + 'dockerhub_user': 'dockerhub_user', + 'dockerhub_password': 'dockerhub_password', + 'docker_image_tag': 'docker_image_tag',} input = default.copy() input.update(overrides) return DevopsFactory().build_devops(input) From e501525db24b542d3ae109d5903f87cb72cb8bd4 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 28 Apr 2023 18:13:45 +0200 Subject: [PATCH 034/173] fix tests --- src/test/python/domain/test_domain.py | 48 ++++++++++++++++----------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/test/python/domain/test_domain.py b/src/test/python/domain/test_domain.py index 230e130..bee073d 100644 --- a/src/test/python/domain/test_domain.py +++ b/src/test/python/domain/test_domain.py @@ -5,10 +5,16 @@ from src.main.python.ddadevops.domain.common import ( DnsRecord, Devops, ) -from src.main.python.ddadevops.domain import Version, ReleaseType, Release, ReleaseContext +from src.main.python.ddadevops.domain import ( + Version, + ReleaseType, + Release, + ReleaseContext, +) from src.main.python.ddadevops.domain.image import Image from src.main.python.ddadevops.domain.c4k import C4k from src.main.python.ddadevops.c4k_mixin import add_c4k_mixin_config +from .test_helper import build_devops class MockValidateable(Validateable): @@ -62,6 +68,7 @@ def test_should_validate_DnsRecord(): sut = DnsRecord("name", ipv6="1::") assert sut.is_valid() + def test_c4k_build_should_update_fqdn(): project_config = { "stage": "test", @@ -108,7 +115,9 @@ def test_c4k_build_should_update_fqdn(): def test_c4k_build_should_calculate_command(): - devops = Devops(stage="test", project_root_path="", module="module", name="name") + devops = build_devops( + {"project_root_path": ".", "module": "module", "name": "name"} + ) project_config = { "stage": "test", "name": "name", @@ -126,9 +135,9 @@ def test_c4k_build_should_calculate_command(): sut = C4k(project_config) assert ( "c4k-module-standalone.jar " - + "/target/name/module/out_c4k_config.yaml " - + "/target/name/module/out_c4k_auth.yaml > " - + "/target/name/module/out_module.yaml" + + "./target/name/module/out_c4k_config.yaml " + + "./target/name/module/out_c4k_auth.yaml > " + + "./target/name/module/out_module.yaml" == sut.command(devops) ) @@ -150,12 +159,13 @@ def test_c4k_build_should_calculate_command(): sut = C4k(project_config) assert ( "c4k-executabel_name-standalone.jar " - + "/target/name/module/out_c4k_config.yaml " - + "/target/name/module/out_c4k_auth.yaml > " - + "/target/name/module/out_module.yaml" + + "./target/name/module/out_c4k_config.yaml " + + "./target/name/module/out_c4k_auth.yaml > " + + "./target/name/module/out_module.yaml" == sut.command(devops) ) + def test_version(tmp_path: Path): version = Version(tmp_path, [1, 2, 3]) version.increment(ReleaseType.SNAPSHOT) @@ -187,31 +197,29 @@ def test_version(tmp_path: Path): assert version.version_list == [2, 0, 0] assert not version.is_snapshot + def test_release_context(tmp_path): version = Version(tmp_path, [1, 2, 3]) release = ReleaseContext(ReleaseType.MINOR, version, "main") release_version = release.release_version() - assert release_version.get_version_string() in '1.3.0' + assert release_version.get_version_string() in "1.3.0" bump_version = release.bump_version() assert bump_version.get_version_string() in "1.3.1-SNAPSHOT" + def test_release(tmp_path): - devops = Devops(stage="test", project_root_path="", module="module", name="name") + devops = build_devops({}) sut = Release(devops, "main", "config_file.json") assert not sut.is_valid() - sut.set_release_context(ReleaseContext(ReleaseType.MINOR, Version("id", [1,2,3]), "main")) + sut.set_release_context( + ReleaseContext(ReleaseType.MINOR, Version("id", [1, 2, 3]), "main") + ) assert sut.is_valid() + def test_devops_build_commons_path(): - devops = Devops( - stage="test", project_root_path="../../..", module="cloud", name="meissa" - ) - sut = Image( - dockerhub_user="user", - dockerhub_password="password", - devops = devops, - ) - assert "docker/" == sut.docker_build_commons_path() + sut = build_devops({}) + assert "docker/" == sut.specialized_build.docker_build_commons_path() From 2add0a76a2dace87bda447dc23ba7b2d22cea206 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 28 Apr 2023 18:13:58 +0200 Subject: [PATCH 035/173] format --- .../python/ddadevops/domain/devops_factory.py | 13 ++++------- src/main/python/ddadevops/domain/image.py | 19 ++++++++------- src/test/python/domain/test_helper.py | 23 ++++++++++--------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index f99fd17..33f6857 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -1,22 +1,20 @@ import deprecation from enum import Enum from typing import List -from .common import ( - Devops, - BuildType -) +from .common import Devops, BuildType from .image import ( Image, ) -class DevopsFactory(): + +class DevopsFactory: def __init__(self): pass def build_devops(self, input) -> Devops: - build_type = BuildType[input['build_type']] + build_type = BuildType[input["build_type"]] specialized_build = None - if build_type == BuildType.IMAGE: + if build_type == BuildType.IMAGE: specialized_build = Image(input) elif build_type == BuildType.C4K: pass @@ -29,4 +27,3 @@ class DevopsFactory(): def merge(input, autorization, context) -> dict: pass - diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index 4344fb3..a7ed1fe 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -1,22 +1,25 @@ -from typing import (Optional, List) +from typing import Optional, List from .common import ( filter_none, Validateable, Devops, ) + class Image(Validateable): def __init__( self, input, ): - self.dockerhub_user=input.get('dockerhub_user') - self.dockerhub_password=input.get('dockerhub_password') - self.docker_publish_tag=input.get('docker_publish_tag') - self.build_commons_path = input.get('build_commons_path') - self.docker_publish_tag = input.get('docker_publish_tag') - self.use_package_common_files = input.get('use_package_common_files', True) - self.docker_build_commons_dir_name = input.get('docker_build_commons_dir_name', 'docker') + self.dockerhub_user = input.get("dockerhub_user") + self.dockerhub_password = input.get("dockerhub_password") + self.docker_publish_tag = input.get("docker_publish_tag") + self.build_commons_path = input.get("build_commons_path") + self.docker_publish_tag = input.get("docker_publish_tag") + self.use_package_common_files = input.get("use_package_common_files", True) + self.docker_build_commons_dir_name = input.get( + "docker_build_commons_dir_name", "docker" + ) def validate(self) -> List[str]: result = [] diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index c93e3dd..6d40f36 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -1,16 +1,17 @@ -from src.main.python.ddadevops.domain import ( - DevopsFactory, Devops -) +from src.main.python.ddadevops.domain import DevopsFactory, Devops + def build_devops(overrides: dict) -> Devops: - default = {'build_type': 'IMAGE', - 'stage': 'test', - 'project_root_path': "../../..", - 'name': 'mybuild', - 'module': 'test_image', - 'dockerhub_user': 'dockerhub_user', - 'dockerhub_password': 'dockerhub_password', - 'docker_image_tag': 'docker_image_tag',} + default = { + "build_type": "IMAGE", + "stage": "test", + "project_root_path": "../../..", + "name": "mybuild", + "module": "test_image", + "dockerhub_user": "dockerhub_user", + "dockerhub_password": "dockerhub_password", + "docker_image_tag": "docker_image_tag", + } input = default.copy() input.update(overrides) return DevopsFactory().build_devops(input) From f4f27ed88ae7e066bfa1a1480ecc6a160cd29928 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 28 Apr 2023 18:26:02 +0200 Subject: [PATCH 036/173] fix test_devops_build --- src/test/python/domain/test_helper.py | 8 +++++--- src/test/python/test_devops_build.py | 10 ++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index 6d40f36..6af661f 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -1,7 +1,6 @@ from src.main.python.ddadevops.domain import DevopsFactory, Devops - -def build_devops(overrides: dict) -> Devops: +def devops_config(overrides: dict) -> dict: default = { "build_type": "IMAGE", "stage": "test", @@ -14,4 +13,7 @@ def build_devops(overrides: dict) -> Devops: } input = default.copy() input.update(overrides) - return DevopsFactory().build_devops(input) + return input + +def build_devops(overrides: dict) -> Devops: + return DevopsFactory().build_devops(devops_config(overrides)) diff --git a/src/test/python/test_devops_build.py b/src/test/python/test_devops_build.py index ca37caa..006d46b 100644 --- a/src/test/python/test_devops_build.py +++ b/src/test/python/test_devops_build.py @@ -2,6 +2,7 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain.common import Devops from src.main.python.ddadevops.devops_build import DevopsBuild +from .domain.test_helper import devops_config class MyDevopsBuild(DevopsBuild): @@ -15,13 +16,6 @@ def test_devops_build(tmp_path): tmp_path_str = str(tmp_path) project = Project(tmp_path_str, name=project_name) - devops = Devops( - stage="test", - project_root_path=tmp_path_str, - module=module_name, - build_dir_name=build_dir, - ) - - devops_build = DevopsBuild(project, devops=devops) + devops_build = DevopsBuild(project, devops_config({})) devops_build.initialize_build_dir() assert os.path.exists(f"{devops_build.build_path()}") From 5ce6e6c0cbf93e93966131168f4eab802773322b Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 12:02:03 +0200 Subject: [PATCH 037/173] add build-type to devops --- src/main/python/ddadevops/domain/__init__.py | 2 +- src/main/python/ddadevops/domain/common.py | 11 ++++++----- src/main/python/ddadevops/domain/devops_factory.py | 4 +++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index d6b5a11..4bd528a 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,4 +1,4 @@ -from .common import Validateable, DnsRecord, Devops +from .common import Validateable, DnsRecord, Devops, BuildType from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index bc87184..5792765 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -7,6 +7,10 @@ import deprecation def filter_none(list_to_filter): return [x for x in list_to_filter if x is not None] +class BuildType(Enum): + IMAGE = 0 + C4K = 1 + class Validateable: def __validate_is_not_empty__(self, field_name: str) -> List[str]: value = self.__dict__[field_name] @@ -41,12 +45,13 @@ class DnsRecord(Validateable): class Devops(Validateable): - def __init__(self, input: dict, specialized_build: Validateable): + def __init__(self, input: dict, build_type: BuildType, specialized_build: Validateable): self.stage = input.get('stage') self.project_root_path = input.get('project_root_path') self.module = input.get('module') self.name = input.get('name', self.module) self.build_dir_name = input.get('build_dir_name', 'target') + self.build_type = build_type self.specialized_build=specialized_build def build_path(self): @@ -74,7 +79,3 @@ class Devops(Validateable): for key in keys: result[key] = self.__get(key) return result - -class BuildType(Enum): - IMAGE = 0 - C4K = 1 diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 33f6857..3da7ea8 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -19,7 +19,9 @@ class DevopsFactory: elif build_type == BuildType.C4K: pass - devops = Devops(input, specialized_build=specialized_build) + devops = Devops( + input, build_type=build_type, specialized_build=specialized_build + ) devops.throw_if_invalid() From 462be0d49ab088fa19b7d6f317753185b717d4ba Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 12:09:04 +0200 Subject: [PATCH 038/173] with new structure we can clean up storage --- .../python/ddadevops/infrastructure/infrastructure.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 15e1e32..300180f 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -4,6 +4,7 @@ from os import chmod from subprocess import run from pkg_resources import resource_string import yaml +import deprecation from ..domain import Devops, Image, C4k, Release from ..python_util import execute @@ -18,21 +19,19 @@ class ProjectRepository: def set_devops(self, project, build: Devops): project.set_property("build", build) - def get_docker(self, project) -> Image: - return project.get_property("docker_build") - - def set_docker(self, project, build: Image): - project.set_property("docker_build", build) - + @deprecation.deprecated(deprecated_in="3.2") def get_c4k(self, project) -> C4k: return project.get_property("c4k_build") + @deprecation.deprecated(deprecated_in="3.2") def set_c4k(self, project, build: C4k): project.set_property("c4k_build", build) + @deprecation.deprecated(deprecated_in="3.2") def get_release(self, project) -> Release: return project.get_property("release_build") + @deprecation.deprecated(deprecated_in="3.2") def set_release(self, project, build: Release): project.set_property("release_build", build) From e2c0ae3a5496f75604a3e566d5dddbe8b5c00a7c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 12:09:23 +0200 Subject: [PATCH 039/173] switch image build to new structure --- .../application/image_build_service.py | 56 ++++++++++--------- .../python/ddadevops/devops_image_build.py | 31 +++++----- src/test/python/test_image_build.py | 15 +---- 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index a1fd210..b210699 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -1,4 +1,4 @@ -from src.main.python.ddadevops.domain import Image +from src.main.python.ddadevops.domain import Image, Devops from src.main.python.ddadevops.infrastructure import FileApi, ResourceApi, ImageApi @@ -6,49 +6,55 @@ class ImageBuildService: def __init__(self): self.file_api = FileApi() self.resource_api = ResourceApi() - self.docker_api = ImageApi() + self.image_api = ImageApi() - def __copy_build_resource_file_from_package__(self, resource_name, docker: Image): + def __copy_build_resource_file_from_package__(self, resource_name, image: Image): data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}") self.file_api.write_data_to_file( - f"{docker.devops.build_path()}/{resource_name}", data + f"{image.devops.build_path()}/{resource_name}", data ) - def __copy_build_resources_from_package__(self, docker: Image): + def __copy_build_resources_from_package__(self, image: Image): self.__copy_build_resource_file_from_package__( - "image/resources/install_functions.sh", docker + "image/resources/install_functions.sh", image ) - def __copy_build_resources_from_dir__(self, docker: Image): + def __copy_build_resources_from_dir__(self, image: Image): self.file_api.cp_force( - docker.docker_build_commons_path(), docker.devops.build_path() + image.docker_build_commons_path(), image.devops.build_path() ) - def initialize_build_dir(self, docker: Image): - build_path = docker.devops.build_path() + def initialize_build_dir(self, devops: Devops): + image = devops.specialized_build + build_path = image.devops.build_path() self.file_api.clean_dir(f"{build_path}/image/resources") - if docker.use_package_common_files: - self.__copy_build_resources_from_package__(docker) + if image.use_package_common_files: + self.__copy_build_resources_from_package__(image) else: - self.__copy_build_resources_from_dir__(docker) + self.__copy_build_resources_from_dir__(image) self.file_api.cp_recursive("image", build_path) self.file_api.cp_recursive("test", build_path) - def image(self, docker: Image): - self.docker_api.image(docker.devops.name, docker.devops.build_path()) + def image(self, devops: Devops): + image = devops.specialized_build + self.image_api.image(image.devops.name, image.devops.build_path()) - def drun(self, docker: Image): - self.docker_api.drun(docker.devops.name) + def drun(self, devops: Devops): + image = devops.specialized_build + self.image_api.drun(image.devops.name) - def dockerhub_login(self, docker: Image): - self.docker_api.dockerhub_login( - docker.dockerhub_user, docker.dockerhub_password + def dockerhub_login(self, devops: Devops): + image = devops.specialized_build + self.image_api.dockerhub_login( + image.dockerhub_user, image.dockerhub_password ) - def dockerhub_publish(self, docker: Image): - self.docker_api.dockerhub_publish( - docker.devops.name, docker.dockerhub_user, docker.docker_publish_tag + def dockerhub_publish(self, devops: Devops): + image = devops.specialized_build + self.image_api.dockerhub_publish( + image.devops.name, image.dockerhub_user, image.docker_publish_tag ) - def test(self, docker: Image): - self.docker_api.test(docker.devops.name, docker.devops.build_path()) + def test(self, devops: Devops): + image = devops.specialized_build + self.image_api.test(image.devops.name, image.devops.build_path()) diff --git a/src/main/python/ddadevops/devops_image_build.py b/src/main/python/ddadevops/devops_image_build.py index f1a2e30..b39855d 100644 --- a/src/main/python/ddadevops/devops_image_build.py +++ b/src/main/python/ddadevops/devops_image_build.py @@ -1,11 +1,11 @@ from typing import Optional import deprecation -from .domain import Image +from .domain import BuildType from .application import ImageBuildService from .devops_build import DevopsBuild, create_devops_build_config -@deprecation.deprecated(deprecated_in="3.2", details="create objects direct instead") +@deprecation.deprecated(deprecated_in="3.2", details="use direct dict instead") def create_devops_docker_build_config( stage, project_root_path, @@ -36,28 +36,31 @@ class DevopsImageBuild(DevopsBuild): def __init__(self, project, input: dict): super().__init__(project, input) self.image_build_service = ImageBuildService() + devops = self.repo.get_devops(self.project) + if devops.build_type != BuildType.IMAGE: + raise ValueError(f"ImageBuild requires BuildType.IMAGE but was: {devops.build_type}") def initialize_build_dir(self): super().initialize_build_dir() - image = self.repo.get_docker(self.project) - self.image_build_service.initialize_build_dir(image) + devops = self.repo.get_devops(self.project) + self.image_build_service.initialize_build_dir(devops) def image(self): - image = self.repo.get_docker(self.project) - self.image_build_service.image(image) + devops = self.repo.get_devops(self.project) + self.image_build_service.image(devops) def drun(self): - image = self.repo.get_docker(self.project) - self.image_build_service.drun(image) + devops = self.repo.get_devops(self.project) + self.image_build_service.drun(devops) def dockerhub_login(self): - image = self.repo.get_docker(self.project) - self.image_build_service.dockerhub_login(image) + devops = self.repo.get_devops(self.project) + self.image_build_service.dockerhub_login(devops) def dockerhub_publish(self): - image = self.repo.get_docker(self.project) - self.image_build_service.dockerhub_publish(image) + devops = self.repo.get_devops(self.project) + self.image_build_service.dockerhub_publish(devops) def test(self): - image = self.repo.get_docker(self.project) - self.image_build_service.test(image) + devops = self.repo.get_devops(self.project) + self.image_build_service.test(devops) diff --git a/src/test/python/test_image_build.py b/src/test/python/test_image_build.py index 9657085..acb2f3b 100644 --- a/src/test/python/test_image_build.py +++ b/src/test/python/test_image_build.py @@ -2,6 +2,7 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain import Image, Devops from src.main.python.ddadevops.devops_image_build import DevopsImageBuild +from .domain.test_helper import devops_config def test_devops_docker_build(tmp_path): @@ -11,15 +12,5 @@ def test_devops_docker_build(tmp_path): tmp_path_str = str(tmp_path) project = Project(tmp_path_str, name=project_name) - devops = Devops( - stage="test", - project_root_path=tmp_path_str, - module=module_name, - name=project_name, - build_dir_name=build_dir - ) - image = Image(dockerhub_user="user", dockerhub_password="password", devops=devops) - - docker_build = DevopsImageBuild(project, image=image) - # docker_build.initialize_build_dir() - # assert os.path.exists(f"{docker_build.build_path()}") + image_build = DevopsImageBuild(project, devops_config({})) + assert image_build From 634e89407eae0f4f810cdfb2eca2eff90b983611 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 14:53:59 +0200 Subject: [PATCH 040/173] multi inheritance requires more than one build extension --- doc/architecture/Domain.md | 4 +-- .../python/ddadevops/devops_image_build.py | 4 +-- src/main/python/ddadevops/domain/common.py | 21 +++++++++----- .../python/ddadevops/domain/devops_factory.py | 22 ++++++++------ src/test/python/domain/test_devops_factory.py | 29 +++++++++---------- src/test/python/domain/test_domain.py | 3 +- src/test/python/domain/test_helper.py | 2 +- 7 files changed, 47 insertions(+), 38 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 92d12dc..2e9cd5a 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -43,8 +43,8 @@ classDiagram current_branch } - Devops *-- Image: spcialized_build - Devops *-- C4k: spcialized_build + Devops *-- "0..1" Image: spcialized_builds + Devops *-- "0..1" C4k: spcialized_builds Devops *-- Release: release C4k *-- DnsRecord Release *-- "0..1" ReleaseContext diff --git a/src/main/python/ddadevops/devops_image_build.py b/src/main/python/ddadevops/devops_image_build.py index b39855d..db55b7e 100644 --- a/src/main/python/ddadevops/devops_image_build.py +++ b/src/main/python/ddadevops/devops_image_build.py @@ -37,8 +37,8 @@ class DevopsImageBuild(DevopsBuild): super().__init__(project, input) self.image_build_service = ImageBuildService() devops = self.repo.get_devops(self.project) - if devops.build_type != BuildType.IMAGE: - raise ValueError(f"ImageBuild requires BuildType.IMAGE but was: {devops.build_type}") + if BuildType.IMAGE not in devops.specialized_builds: + raise ValueError(f"ImageBuild requires BuildType.IMAGE") def initialize_build_dir(self): super().initialize_build_dir() diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 5792765..1a3594c 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,6 +1,6 @@ import deprecation from enum import Enum -from typing import List +from typing import List, TypedDict import logging import deprecation @@ -12,11 +12,16 @@ class BuildType(Enum): C4K = 1 class Validateable: + def __validate_is_not_none__(self, field_name: str) -> List[str]: + value = self.__dict__[field_name] + if value is None: + return [f"Field '{field_name}' must not be None."] + return [] + def __validate_is_not_empty__(self, field_name: str) -> List[str]: value = self.__dict__[field_name] if value is None or value == "": return [f"Field '{field_name}' must not be empty."] - return [] def validate(self) -> List[str]: @@ -45,14 +50,13 @@ class DnsRecord(Validateable): class Devops(Validateable): - def __init__(self, input: dict, build_type: BuildType, specialized_build: Validateable): + def __init__(self, input: dict, specialized_builds: dict[BuildType, Validateable]): self.stage = input.get('stage') self.project_root_path = input.get('project_root_path') self.module = input.get('module') self.name = input.get('name', self.module) self.build_dir_name = input.get('build_dir_name', 'target') - self.build_type = build_type - self.specialized_build=specialized_build + self.specialized_builds=specialized_builds def build_path(self): path = [self.project_root_path, self.build_dir_name, self.name, self.module] @@ -63,9 +67,10 @@ class Devops(Validateable): result += self.__validate_is_not_empty__("stage") result += self.__validate_is_not_empty__("project_root_path") result += self.__validate_is_not_empty__("module") - result += self.__validate_is_not_empty__("specialized_build") - if self.specialized_build: - result += self.specialized_build.validate() + result += self.__validate_is_not_none__("specialized_builds") + if self.specialized_builds: + for build in self.specialized_builds: + result += self.specialized_builds[build].validate() return result def __put__(self, key, value): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 3da7ea8..6324180 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -12,20 +12,24 @@ class DevopsFactory: pass def build_devops(self, input) -> Devops: - build_type = BuildType[input["build_type"]] - specialized_build = None - if build_type == BuildType.IMAGE: - specialized_build = Image(input) - elif build_type == BuildType.C4K: + build_types = self.__parse_build_types__(input["build_types"]) + specialized_builds = {} + if BuildType.IMAGE in build_types: + specialized_builds[BuildType.IMAGE] = Image(input) + elif BuildType.C4K in build_types: pass - devops = Devops( - input, build_type=build_type, specialized_build=specialized_build - ) + devops = Devops(input, specialized_builds=specialized_builds) devops.throw_if_invalid() return devops - def merge(input, autorization, context) -> dict: + def merge(self, input, autorization, context) -> dict: pass + + def __parse_build_types__(self, build_types: List[str]) -> List[BuildType]: + result = [] + for build_type in build_types: + result += [BuildType[build_type]] + return result diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index df7e15f..e6e3633 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -3,25 +3,24 @@ from src.main.python.ddadevops.domain.devops_factory import ( DevopsFactory, ) + def test_devops_factory(): with pytest.raises(Exception): - DevopsFactory().build_devops( - {'build_type': 'NOTEXISTING'} - ) + DevopsFactory().build_devops({"build_types": ["NOTEXISTING"]}) with pytest.raises(Exception): - DevopsFactory().build_devops( - {'build_type': 'IMAGE'} - ) + DevopsFactory().build_devops({'build_types': ['IMAGE'],}) sut = DevopsFactory().build_devops( - {'build_type': 'IMAGE', - 'stage': 'test', - 'project_root_path': "../../..", - 'name': 'mybuild', - 'module': 'test_image', - 'dockerhub_user': 'dockerhub_user', - 'dockerhub_password': 'dockerhub_password', - 'docker_image_tag': 'docker_image_tag',} - ) + { + "build_types": ["IMAGE"], + "stage": "test", + "project_root_path": "../../..", + "name": "mybuild", + "module": "test_image", + "dockerhub_user": "dockerhub_user", + "dockerhub_password": "dockerhub_password", + "docker_image_tag": "docker_image_tag", + } + ) assert sut != None diff --git a/src/test/python/domain/test_domain.py b/src/test/python/domain/test_domain.py index bee073d..375e10d 100644 --- a/src/test/python/domain/test_domain.py +++ b/src/test/python/domain/test_domain.py @@ -4,6 +4,7 @@ from src.main.python.ddadevops.domain.common import ( Validateable, DnsRecord, Devops, + BuildType, ) from src.main.python.ddadevops.domain import ( Version, @@ -222,4 +223,4 @@ def test_release(tmp_path): def test_devops_build_commons_path(): sut = build_devops({}) - assert "docker/" == sut.specialized_build.docker_build_commons_path() + assert "docker/" == sut.specialized_builds[BuildType.IMAGE].docker_build_commons_path() diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index 6af661f..4a5b171 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -2,7 +2,7 @@ from src.main.python.ddadevops.domain import DevopsFactory, Devops def devops_config(overrides: dict) -> dict: default = { - "build_type": "IMAGE", + "build_types": ["IMAGE"], "stage": "test", "project_root_path": "../../..", "name": "mybuild", From 3495f69a6bd20e9887426762a179c16dceb1b3b2 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 14:57:15 +0200 Subject: [PATCH 041/173] adjust filename --- src/main/python/ddadevops/__init__.py | 2 +- .../python/ddadevops/{c4k_mixin.py => c4k_build.py} | 11 ++++++----- src/test/python/domain/test_domain.py | 2 +- .../python/{test_c4k_mixin.py => test_c4k_build.py} | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) rename src/main/python/ddadevops/{c4k_mixin.py => c4k_build.py} (88%) rename src/test/python/{test_c4k_mixin.py => test_c4k_build.py} (95%) diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index 3cd3560..30ae280 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -9,7 +9,7 @@ from .provs_k3s_mixin import ProvsK3sMixin, add_provs_k3s_mixin_config from .aws_rds_pg_mixin import AwsRdsPgMixin, add_aws_rds_pg_mixin_config from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config -from .c4k_mixin import C4kBuild, add_c4k_mixin_config +from .c4k_build import C4kBuild, add_c4k_mixin_config from .exoscale_mixin import ExoscaleMixin, add_exoscale_mixin_config from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config diff --git a/src/main/python/ddadevops/c4k_mixin.py b/src/main/python/ddadevops/c4k_build.py similarity index 88% rename from src/main/python/ddadevops/c4k_mixin.py rename to src/main/python/ddadevops/c4k_build.py index 8a5f3f2..cbeb8bd 100644 --- a/src/main/python/ddadevops/c4k_mixin.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -1,12 +1,11 @@ import deprecation -from .domain import C4k, DnsRecord +from .domain import BuildType, DnsRecord from .devops_build import DevopsBuild from .credential import gopass_field_from_path, gopass_password_from_path from .infrastructure import ExecutionApi -@deprecation.deprecated(deprecated_in="3.2") -# create objects direct instead +@deprecation.deprecated(deprecated_in="3.2", details="use direct dict instead") def add_c4k_mixin_config( config, c4k_config_dict, @@ -44,12 +43,14 @@ def add_c4k_mixin_config( ) return config + class C4kBuild(DevopsBuild): def __init__(self, project, config): super().__init__(project, config) self.execution_api = ExecutionApi() - c4k_build = C4k(config) - self.repo.set_c4k(self.project, c4k_build) + devops = self.repo.get_devops(self.project) + if BuildType.C4K not in devops.specialized_builds: + raise ValueError(f"C4kBuild requires BuildType.C4K") def update_runtime_config(self, dns_record: DnsRecord): c4k_build = self.repo.get_c4k(self.project) diff --git a/src/test/python/domain/test_domain.py b/src/test/python/domain/test_domain.py index 375e10d..00ed640 100644 --- a/src/test/python/domain/test_domain.py +++ b/src/test/python/domain/test_domain.py @@ -14,7 +14,7 @@ from src.main.python.ddadevops.domain import ( ) from src.main.python.ddadevops.domain.image import Image from src.main.python.ddadevops.domain.c4k import C4k -from src.main.python.ddadevops.c4k_mixin import add_c4k_mixin_config +from src.main.python.ddadevops.c4k_build import add_c4k_mixin_config from .test_helper import build_devops diff --git a/src/test/python/test_c4k_mixin.py b/src/test/python/test_c4k_build.py similarity index 95% rename from src/test/python/test_c4k_mixin.py rename to src/test/python/test_c4k_build.py index c4c2b17..b1e90a0 100644 --- a/src/test/python/test_c4k_mixin.py +++ b/src/test/python/test_c4k_build.py @@ -1,7 +1,7 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain import DnsRecord -from src.main.python.ddadevops.c4k_mixin import C4kBuild, add_c4k_mixin_config +from src.main.python.ddadevops.c4k_build import C4kBuild, add_c4k_mixin_config class MyC4kBuild(C4kBuild): pass From 1ce070beac4f7f91ca6215e5aeae58fa63057fd4 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 16:44:10 +0200 Subject: [PATCH 042/173] split tests --- src/test/python/domain/test_c4k.py | 99 +++++++++++++++++ .../domain/{test_domain.py => test_common.py} | 100 ------------------ 2 files changed, 99 insertions(+), 100 deletions(-) create mode 100644 src/test/python/domain/test_c4k.py rename src/test/python/domain/{test_domain.py => test_common.py} (55%) diff --git a/src/test/python/domain/test_c4k.py b/src/test/python/domain/test_c4k.py new file mode 100644 index 0000000..175980e --- /dev/null +++ b/src/test/python/domain/test_c4k.py @@ -0,0 +1,99 @@ +import pytest +from pathlib import Path +from src.main.python.ddadevops.domain.common import ( + DnsRecord, + BuildType, +) +from src.main.python.ddadevops.domain.c4k import C4k +from .test_helper import build_devops + + +def test_creation(): + sut = build_devops({}) + assert BuildType.C4K in sut.specialized_builds + + +def test_c4k_should_calculate_config(): + sut = build_devops({}) + with pytest.raises(Exception): + sut.specialized_builds[BuildType.C4K].config() + + sut = build_devops({}) + c4k = sut.specialized_builds[BuildType.C4K] + c4k.update_runtime_config(DnsRecord("fqdn")) + assert { + "fqdn": "fqdn", + "mon-cfg": { + "cluster-name": "module", + "cluster-stage": "test", + "grafana-cloud-url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", + }, + } == c4k.config() + + sut = build_devops( + { + "c4k_config": {"test": "test"}, + } + ) + c4k = sut.specialized_builds[BuildType.C4K] + c4k.update_runtime_config(DnsRecord("fqdn")) + assert { + "test": "test", + "fqdn": "fqdn", + "mon-cfg": { + "cluster-name": "module", + "cluster-stage": "test", + "grafana-cloud-url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", + }, + } == c4k.config() + + +def test_c4k_should_calculate_auth(): + sut = build_devops({}) + c4k = sut.specialized_builds[BuildType.C4K] + assert { + "mon-auth": {"grafana-cloud-password": "password", "grafana-cloud-user": "user"} + } == c4k.auth() + + sut = build_devops( + { + "c4k_auth": {"test": "test"}, + } + ) + c4k = sut.specialized_builds[BuildType.C4K] + assert { + "test": "test", + "mon-auth": { + "grafana-cloud-password": "password", + "grafana-cloud-user": "user", + }, + } == c4k.auth() + + +def test_c4k_build_should_calculate_command(): + sut = build_devops( + { + "project_root_path": ".", + } + ) + assert ( + "c4k-module-standalone.jar " + + "./target/name/module/out_c4k_config.yaml " + + "./target/name/module/out_c4k_auth.yaml > " + + "./target/name/module/out_module.yaml" + == sut.specialized_builds[BuildType.C4K].command(sut) + ) + + sut = build_devops( + { + "project_root_path": ".", + "c4k_executabel_name": "executabel_name", + } + ) + assert ( + "c4k-executabel_name-standalone.jar " + + "./target/name/module/out_c4k_config.yaml " + + "./target/name/module/out_c4k_auth.yaml > " + + "./target/name/module/out_module.yaml" + == sut.specialized_builds[BuildType.C4K].command(sut) + ) diff --git a/src/test/python/domain/test_domain.py b/src/test/python/domain/test_common.py similarity index 55% rename from src/test/python/domain/test_domain.py rename to src/test/python/domain/test_common.py index 00ed640..5c2bd6d 100644 --- a/src/test/python/domain/test_domain.py +++ b/src/test/python/domain/test_common.py @@ -13,8 +13,6 @@ from src.main.python.ddadevops.domain import ( ReleaseContext, ) from src.main.python.ddadevops.domain.image import Image -from src.main.python.ddadevops.domain.c4k import C4k -from src.main.python.ddadevops.c4k_build import add_c4k_mixin_config from .test_helper import build_devops @@ -69,104 +67,6 @@ def test_should_validate_DnsRecord(): sut = DnsRecord("name", ipv6="1::") assert sut.is_valid() - -def test_c4k_build_should_update_fqdn(): - project_config = { - "stage": "test", - "name": "name", - "project_root_path": "mypath", - "module": "module", - "build_dir_name": "target", - } - config = {"issuer": "staging"} - auth = { - "jvb-auth-password": "pw1", - "jicofo-auth-password": "pw2", - "jicofo-component-secret": "pw3", - } - add_c4k_mixin_config( - project_config, - config, - auth, - grafana_cloud_user="user", - grafana_cloud_password="password", - ) - - sut = C4k(project_config) - sut.update_runtime_config(DnsRecord("test.de", ipv6="1::")) - - assert { - "issuer": "staging", - "fqdn": "test.de", - "mon-cfg": { - "cluster-name": "module", - "cluster-stage": "test", - "grafana-cloud-url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", - }, - } == sut.config() - assert { - "jicofo-auth-password": "pw2", - "jicofo-component-secret": "pw3", - "jvb-auth-password": "pw1", - "mon-auth": { - "grafana-cloud-password": "password", - "grafana-cloud-user": "user", - }, - } == sut.c4k_mixin_auth - - -def test_c4k_build_should_calculate_command(): - devops = build_devops( - {"project_root_path": ".", "module": "module", "name": "name"} - ) - project_config = { - "stage": "test", - "name": "name", - "project_root_path": "", - "module": "module", - "build_dir_name": "target", - } - add_c4k_mixin_config( - project_config, - {}, - {}, - grafana_cloud_user="user", - grafana_cloud_password="password", - ) - sut = C4k(project_config) - assert ( - "c4k-module-standalone.jar " - + "./target/name/module/out_c4k_config.yaml " - + "./target/name/module/out_c4k_auth.yaml > " - + "./target/name/module/out_module.yaml" - == sut.command(devops) - ) - - project_config = { - "stage": "test", - "name": "name", - "project_root_path": "", - "module": "module", - "build_dir_name": "target", - } - add_c4k_mixin_config( - project_config, - {}, - {}, - executabel_name="executabel_name", - grafana_cloud_user="user", - grafana_cloud_password="password", - ) - sut = C4k(project_config) - assert ( - "c4k-executabel_name-standalone.jar " - + "./target/name/module/out_c4k_config.yaml " - + "./target/name/module/out_c4k_auth.yaml > " - + "./target/name/module/out_module.yaml" - == sut.command(devops) - ) - - def test_version(tmp_path: Path): version = Version(tmp_path, [1, 2, 3]) version.increment(ReleaseType.SNAPSHOT) From ea77c84948be0e7cc7cbe55adbf18c26e72cd2f1 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 16:44:34 +0200 Subject: [PATCH 043/173] add some tests --- src/main/python/ddadevops/domain/c4k.py | 56 +++++++++++++------ src/test/python/domain/test_devops_factory.py | 29 ++++++++-- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/main/python/ddadevops/domain/c4k.py b/src/main/python/ddadevops/domain/c4k.py index 5ae0cbf..88e0a81 100644 --- a/src/main/python/ddadevops/domain/c4k.py +++ b/src/main/python/ddadevops/domain/c4k.py @@ -5,17 +5,20 @@ from .common import ( Devops, ) + class C4k(Validateable): - def __init__(self, config: dict): - tmp_executabel_name = config["C4kMixin"]["executabel_name"] - if not tmp_executabel_name: - tmp_executabel_name = config["module"] - self.executabel_name = tmp_executabel_name - self.c4k_mixin_config = config["C4kMixin"]["config"] - self.c4k_mixin_auth = config["C4kMixin"]["auth"] - tmp = self.c4k_mixin_config["mon-cfg"] - tmp.update({"cluster-name": config["module"], "cluster-stage": config["stage"]}) - self.c4k_mixin_config.update({"mon-cfg": tmp}) + def __init__(self, input: dict): + self.module = input.get("module") + self.stage = input.get("stage") + self.executabel_name = input.get("c4k_executabel_name", input.get("module")) + self.c4k_config = input.get("c4k_config", {}) + self.grafana_cloud_url = input.get( + "c4k_grafana_cloud_url", + "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", + ) + self.c4k_auth = input.get("c4k_auth", {}) + self.grafana_cloud_user = input.get('c4k_grafana_cloud_user') + self.grafana_cloud_password = input.get('c4k_grafana_cloud_password') self.dns_record: Optional[DnsRecord] = None # TODO: these functions should be located at TerraformBuild later on. @@ -24,19 +27,38 @@ class C4k(Validateable): def validate(self) -> List[str]: result = [] - result += self.__validate_is_not_empty__("fqdn") + result += self.__validate_is_not_empty__("module") + result += self.__validate_is_not_empty__("stage") + result += self.__validate_is_not_empty__("executabel_name") + result += self.__validate_is_not_empty__("grafana_cloud_user") + result += self.__validate_is_not_empty__("grafana_cloud_password") if self.dns_record: result += self.dns_record.validate() return result def config(self): - fqdn = self.dns_record.fqdn - self.c4k_mixin_config.update({"fqdn": fqdn}) - return self.c4k_mixin_config + if not self.dns_record: + raise ValueError("dns_reqord was not set.") + result = self.c4k_config.copy() + result["fqdn"] = self.dns_record.fqdn + result["mon-cfg"] = { + "cluster-name": self.module, + "cluster-stage": self.stage, + "grafana-cloud-url": self.grafana_cloud_url, + } + return result - def command(self, build: Devops): - module = build.module - build_path = build.build_path() + def auth(self): + result = self.c4k_auth.copy() + result["mon-auth"] = { + "grafana-cloud-user": self.grafana_cloud_user, + "grafana-cloud-password": self.grafana_cloud_password, + } + return result + + def command(self, devops: Devops): + module = devops.module + build_path = devops.build_path() config_path = f"{build_path}/out_c4k_config.yaml" auth_path = f"{build_path}/out_c4k_auth.yaml" output_path = f"{build_path}/out_{module}.yaml" diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index e6e3633..056e75c 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -9,18 +9,35 @@ def test_devops_factory(): DevopsFactory().build_devops({"build_types": ["NOTEXISTING"]}) with pytest.raises(Exception): - DevopsFactory().build_devops({'build_types': ['IMAGE'],}) + DevopsFactory().build_devops( + { + "build_types": ["IMAGE"], + } + ) sut = DevopsFactory().build_devops( { - "build_types": ["IMAGE"], "stage": "test", - "project_root_path": "../../..", "name": "mybuild", "module": "test_image", - "dockerhub_user": "dockerhub_user", - "dockerhub_password": "dockerhub_password", - "docker_image_tag": "docker_image_tag", + "project_root_path": "../../..", + "build_types": ["IMAGE"], + "image_dockerhub_user": "dockerhub_user", + "image_dockerhub_password": "dockerhub_password", + "image_tag": "docker_image_tag", + } + ) + assert sut != None + + sut = DevopsFactory().build_devops( + { + "stage": "test", + "name": "mybuild", + "module": "test_image", + "project_root_path": "../../..", + "build_types": ["C4K"], + "c4k_grafana_cloud_user": "user", + "c4k_grafana_cloud_password": "password", } ) assert sut != None From 2a4a2d25d84390d710b63079698420893140f8aa Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 16:45:17 +0200 Subject: [PATCH 044/173] implement --- .../python/ddadevops/domain/devops_factory.py | 10 ++- src/main/python/ddadevops/domain/image.py | 17 ++--- src/test/python/domain/test_helper.py | 20 ++++-- src/test/python/test_c4k_build.py | 65 ++++++++++--------- 4 files changed, 62 insertions(+), 50 deletions(-) diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 6324180..a16a0b9 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -2,10 +2,8 @@ import deprecation from enum import Enum from typing import List from .common import Devops, BuildType -from .image import ( - Image, -) - +from .image import Image +from .c4k import C4k class DevopsFactory: def __init__(self): @@ -16,8 +14,8 @@ class DevopsFactory: specialized_builds = {} if BuildType.IMAGE in build_types: specialized_builds[BuildType.IMAGE] = Image(input) - elif BuildType.C4K in build_types: - pass + if BuildType.C4K in build_types: + specialized_builds[BuildType.C4K] = C4k(input) devops = Devops(input, specialized_builds=specialized_builds) diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index a7ed1fe..d5f9cc2 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -9,16 +9,17 @@ from .common import ( class Image(Validateable): def __init__( self, - input, + input: dict, ): - self.dockerhub_user = input.get("dockerhub_user") - self.dockerhub_password = input.get("dockerhub_password") - self.docker_publish_tag = input.get("docker_publish_tag") - self.build_commons_path = input.get("build_commons_path") - self.docker_publish_tag = input.get("docker_publish_tag") - self.use_package_common_files = input.get("use_package_common_files", True) + self.dockerhub_user = input.get("image_dockerhub_user") + self.dockerhub_password = input.get("image_dockerhub_password") + # TODO: rename to image_tag + self.docker_publish_tag = input.get("image_tag") + self.build_commons_path = input.get("image_build_commons_path") + self.use_package_common_files = input.get("image_use_package_common_files", True) + # TODO: rename to image_build_commons_dir_name self.docker_build_commons_dir_name = input.get( - "docker_build_commons_dir_name", "docker" + "image_build_commons_dir_name", "docker" ) def validate(self) -> List[str]: diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index 4a5b171..4d96b4e 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -1,19 +1,27 @@ from src.main.python.ddadevops.domain import DevopsFactory, Devops + def devops_config(overrides: dict) -> dict: default = { - "build_types": ["IMAGE"], + "name": "name", + "module": "module", "stage": "test", "project_root_path": "../../..", - "name": "mybuild", - "module": "test_image", - "dockerhub_user": "dockerhub_user", - "dockerhub_password": "dockerhub_password", - "docker_image_tag": "docker_image_tag", + "build_dir_name": "target", + "build_types": ["IMAGE", "C4K"], + "image_dockerhub_user": "dockerhub_user", + "image_dockerhub_password": "dockerhub_password", + "image_tag": "image_tag", + 'c4k_config': {}, + "c4k_grafana_cloud_user": "user", + "c4k_grafana_cloud_password": "password", + "c4k_grafana_cloud_url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", + 'c4k_auth': {}, } input = default.copy() input.update(overrides) return input + def build_devops(overrides: dict) -> Devops: return DevopsFactory().build_devops(devops_config(overrides)) diff --git a/src/test/python/test_c4k_build.py b/src/test/python/test_c4k_build.py index b1e90a0..aab2e14 100644 --- a/src/test/python/test_c4k_build.py +++ b/src/test/python/test_c4k_build.py @@ -2,45 +2,50 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain import DnsRecord from src.main.python.ddadevops.c4k_build import C4kBuild, add_c4k_mixin_config +from .domain.test_helper import devops_config + class MyC4kBuild(C4kBuild): pass -def test_c4k_mixin(tmp_path): - build_dir = 'build' - project_name = 'testing-project' - module_name = 'c4k-test' +def test_c4k_mixin(tmp_path): + build_dir = "build" + project_name = "testing-project" + module_name = "c4k-test" tmp_path_str = str(tmp_path) project = Project(tmp_path_str, name=project_name) + sut = MyC4kBuild( + project, + devops_config( + { + "build_types": ["C4K"], + "project_root_path": tmp_path_str, + "module": "c4k-test", + "c4k_config": {"a": 1, "b": 2}, + "c4k_auth": {"c": 3, "d": 4}, + "grafana_cloud_user": "user", + "grafana_cloud_password": "password", + 'grafana_cloud_url': "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", + } + ), + ) - project_config = { - 'stage': 'test', - 'project_root_path': tmp_path_str, - 'module': module_name, - 'build_dir_name': build_dir - } + + sut.initialize_build_dir() + assert ( + sut.build_path() == f"{tmp_path_str}/target/mybuild/c4k-test" + ) - config = {'a': 1, 'b': 2} - auth = {'c': 3, 'd': 4} + # TODO: mv this to domain test + # sut.update_runtime_config(DnsRecord("test.de", ipv6="1::")) + # assert "fqdn" in sut.specialized_builds[BuildType.C4K].config() + # assert "mon-cfg" in sut.specialized_builds[BuildType.C4K].config() + # assert "mon-auth" in sut.specialized_builds[BuildType.C4K].c4k_mixin_auth - add_c4k_mixin_config(project_config, config, auth, grafana_cloud_user='user', grafana_cloud_password='password') + sut.write_c4k_config() + assert os.path.exists(f"{mixin.build_path()}/out_c4k_config.yaml") - assert project_config.get('C4kMixin') is not None - - mixin = MyC4kBuild(project, project_config) - mixin.initialize_build_dir() - assert mixin.build_path() == f'{tmp_path_str}/{build_dir}/{project_name}/{module_name}' - - mixin.update_runtime_config(DnsRecord('test.de', ipv6="1::")) - sut = mixin.repo.get_c4k(mixin.project) - assert 'fqdn' in sut.config() - assert 'mon-cfg' in sut.config() - assert 'mon-auth' in sut.c4k_mixin_auth - - mixin.write_c4k_config() - assert os.path.exists(f'{mixin.build_path()}/out_c4k_config.yaml') - - mixin.write_c4k_auth() - assert os.path.exists(f'{mixin.build_path()}/out_c4k_auth.yaml') + sut.write_c4k_auth() + assert os.path.exists(f"{mixin.build_path()}/out_c4k_auth.yaml") From 331b3bba6ea266aee68f2f9dc7928353f98e8da4 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 20:03:21 +0200 Subject: [PATCH 045/173] c4kBuild is working --- src/main/python/ddadevops/c4k_build.py | 31 ++++++++++++++------------ src/test/python/test_c4k_build.py | 27 ++++++---------------- 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/src/main/python/ddadevops/c4k_build.py b/src/main/python/ddadevops/c4k_build.py index cbeb8bd..9cd0f99 100644 --- a/src/main/python/ddadevops/c4k_build.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -53,23 +53,26 @@ class C4kBuild(DevopsBuild): raise ValueError(f"C4kBuild requires BuildType.C4K") def update_runtime_config(self, dns_record: DnsRecord): - c4k_build = self.repo.get_c4k(self.project) - c4k_build.update_runtime_config(dns_record) - self.repo.set_c4k(self.project, c4k_build) + devops = self.repo.get_devops(self.project) + devops.specialized_builds[BuildType.C4K].update_runtime_config(dns_record) + self.repo.set_devops(self.project, devops) def write_c4k_config(self): - build = self.repo.get_devops(self.project) - c4k_build = self.repo.get_c4k(self.project) - path = build.build_path() + "/out_c4k_config.yaml" - self.file_api.write_yaml_to_file(path, c4k_build.config()) + devops = self.repo.get_devops(self.project) + path = devops.build_path() + "/out_c4k_config.yaml" + self.file_api.write_yaml_to_file( + path, devops.specialized_builds[BuildType.C4K].config() + ) def write_c4k_auth(self): - build = self.repo.get_devops(self.project) - c4k_build = self.repo.get_c4k(self.project) - path = build.build_path() + "/out_c4k_auth.yaml" - self.file_api.write_yaml_to_file(path, c4k_build.c4k_mixin_auth) + devops = self.repo.get_devops(self.project) + path = devops.build_path() + "/out_c4k_auth.yaml" + self.file_api.write_yaml_to_file( + path, devops.specialized_builds[BuildType.C4K].auth() + ) def c4k_apply(self, dry_run=False): - build = self.repo.get_devops(self.project) - c4k_build = self.repo.get_c4k(self.project) - return self.execution_api.execute(c4k_build.command(build), dry_run) + devops = self.repo.get_devops(self.project) + return self.execution_api.execute( + devops.specialized_builds[BuildType.C4K].command(devops), dry_run + ) diff --git a/src/test/python/test_c4k_build.py b/src/test/python/test_c4k_build.py index aab2e14..b9471c3 100644 --- a/src/test/python/test_c4k_build.py +++ b/src/test/python/test_c4k_build.py @@ -5,10 +5,6 @@ from src.main.python.ddadevops.c4k_build import C4kBuild, add_c4k_mixin_config from .domain.test_helper import devops_config -class MyC4kBuild(C4kBuild): - pass - - def test_c4k_mixin(tmp_path): build_dir = "build" project_name = "testing-project" @@ -16,7 +12,7 @@ def test_c4k_mixin(tmp_path): tmp_path_str = str(tmp_path) project = Project(tmp_path_str, name=project_name) - sut = MyC4kBuild( + sut = C4kBuild( project, devops_config( { @@ -25,27 +21,18 @@ def test_c4k_mixin(tmp_path): "module": "c4k-test", "c4k_config": {"a": 1, "b": 2}, "c4k_auth": {"c": 3, "d": 4}, - "grafana_cloud_user": "user", - "grafana_cloud_password": "password", - 'grafana_cloud_url': "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", + "c4k_grafana_cloud_user": "user", + "c4k_grafana_cloud_password": "password", } ), ) - sut.initialize_build_dir() - assert ( - sut.build_path() == f"{tmp_path_str}/target/mybuild/c4k-test" - ) - - # TODO: mv this to domain test - # sut.update_runtime_config(DnsRecord("test.de", ipv6="1::")) - # assert "fqdn" in sut.specialized_builds[BuildType.C4K].config() - # assert "mon-cfg" in sut.specialized_builds[BuildType.C4K].config() - # assert "mon-auth" in sut.specialized_builds[BuildType.C4K].c4k_mixin_auth + assert sut.build_path() == f"{tmp_path_str}/target/name/c4k-test" + sut.update_runtime_config(DnsRecord("test.de", ipv6="::1")) sut.write_c4k_config() - assert os.path.exists(f"{mixin.build_path()}/out_c4k_config.yaml") + assert os.path.exists(f"{sut.build_path()}/out_c4k_config.yaml") sut.write_c4k_auth() - assert os.path.exists(f"{mixin.build_path()}/out_c4k_auth.yaml") + assert os.path.exists(f"{sut.build_path()}/out_c4k_auth.yaml") From 72339f62cf381275b36742c648b86dc125f710d7 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 20:19:07 +0200 Subject: [PATCH 046/173] unify c4k prefixes --- doc/architecture/Domain.md | 28 ++++++++++++------------- src/main/python/ddadevops/domain/c4k.py | 22 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 2e9cd5a..d65a55e 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -4,25 +4,25 @@ classDiagram class Devops { <> - stage name - project_root_path module + stage build_dir_name + project_root_path } class Image { - dockerhub_user - dockerhub_password - build_dir_name - use_package_common_files - build_commons_path - docker_build_commons_dir_name - docker_publish_tag + image_dockerhub_user + image_dockerhub_password + image_publish_tag + image_build_dir_name + image_use_package_common_files + image_build_commons_path + image_build_commons_dir_name } class C4k { - executabel_name + c4k_executabel_name c4k_mixin_config c4k_mixin_auth } @@ -34,13 +34,13 @@ classDiagram } class Release { - main_branch - config_file + release_main_branch + release_config_file } class ReleaseContext { release_type - version - current_branch + release_version + release_current_branch } Devops *-- "0..1" Image: spcialized_builds diff --git a/src/main/python/ddadevops/domain/c4k.py b/src/main/python/ddadevops/domain/c4k.py index 88e0a81..bb26686 100644 --- a/src/main/python/ddadevops/domain/c4k.py +++ b/src/main/python/ddadevops/domain/c4k.py @@ -10,15 +10,15 @@ class C4k(Validateable): def __init__(self, input: dict): self.module = input.get("module") self.stage = input.get("stage") - self.executabel_name = input.get("c4k_executabel_name", input.get("module")) + self.c4k_executabel_name = input.get("c4k_executabel_name", input.get("module")) self.c4k_config = input.get("c4k_config", {}) - self.grafana_cloud_url = input.get( + self.c4k_grafana_cloud_url = input.get( "c4k_grafana_cloud_url", "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", ) self.c4k_auth = input.get("c4k_auth", {}) - self.grafana_cloud_user = input.get('c4k_grafana_cloud_user') - self.grafana_cloud_password = input.get('c4k_grafana_cloud_password') + self.c4k_grafana_cloud_user = input.get('c4k_grafana_cloud_user') + self.c4k_grafana_cloud_password = input.get('c4k_grafana_cloud_password') self.dns_record: Optional[DnsRecord] = None # TODO: these functions should be located at TerraformBuild later on. @@ -29,9 +29,9 @@ class C4k(Validateable): result = [] result += self.__validate_is_not_empty__("module") result += self.__validate_is_not_empty__("stage") - result += self.__validate_is_not_empty__("executabel_name") - result += self.__validate_is_not_empty__("grafana_cloud_user") - result += self.__validate_is_not_empty__("grafana_cloud_password") + result += self.__validate_is_not_empty__("c4k_executabel_name") + result += self.__validate_is_not_empty__("c4k_grafana_cloud_user") + result += self.__validate_is_not_empty__("c4k_grafana_cloud_password") if self.dns_record: result += self.dns_record.validate() return result @@ -44,15 +44,15 @@ class C4k(Validateable): result["mon-cfg"] = { "cluster-name": self.module, "cluster-stage": self.stage, - "grafana-cloud-url": self.grafana_cloud_url, + "grafana-cloud-url": self.c4k_grafana_cloud_url, } return result def auth(self): result = self.c4k_auth.copy() result["mon-auth"] = { - "grafana-cloud-user": self.grafana_cloud_user, - "grafana-cloud-password": self.grafana_cloud_password, + "grafana-cloud-user": self.c4k_grafana_cloud_user, + "grafana-cloud-password": self.c4k_grafana_cloud_password, } return result @@ -62,4 +62,4 @@ class C4k(Validateable): config_path = f"{build_path}/out_c4k_config.yaml" auth_path = f"{build_path}/out_c4k_auth.yaml" output_path = f"{build_path}/out_{module}.yaml" - return f"c4k-{self.executabel_name}-standalone.jar {config_path} {auth_path} > {output_path}" + return f"c4k-{self.c4k_executabel_name}-standalone.jar {config_path} {auth_path} > {output_path}" From 26b76a045ab413286fd2459102417fb1592ab5d2 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 20:55:25 +0200 Subject: [PATCH 047/173] unify image prefixes --- .../application/image_build_service.py | 39 +++++------ src/main/python/ddadevops/domain/image.py | 28 ++++---- src/test/python/domain/test_common.py | 58 ---------------- src/test/python/domain/test_image.py | 14 ++++ src/test/python/domain/test_release.py | 69 +++++++++++++++++++ 5 files changed, 117 insertions(+), 91 deletions(-) create mode 100644 src/test/python/domain/test_image.py create mode 100644 src/test/python/domain/test_release.py diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index b210699..9c43c31 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -1,4 +1,4 @@ -from src.main.python.ddadevops.domain import Image, Devops +from src.main.python.ddadevops.domain import Image, Devops, BuildType from src.main.python.ddadevops.infrastructure import FileApi, ResourceApi, ImageApi @@ -8,27 +8,27 @@ class ImageBuildService: self.resource_api = ResourceApi() self.image_api = ImageApi() - def __copy_build_resource_file_from_package__(self, resource_name, image: Image): + def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}") self.file_api.write_data_to_file( - f"{image.devops.build_path()}/{resource_name}", data + f"{devops.build_path()}/{resource_name}", data ) - def __copy_build_resources_from_package__(self, image: Image): + def __copy_build_resources_from_package__(self, devops: Devops): self.__copy_build_resource_file_from_package__( - "image/resources/install_functions.sh", image + "image/resources/install_functions.sh", devops.specialized_builds[BuildType.C4K] ) - def __copy_build_resources_from_dir__(self, image: Image): + def __copy_build_resources_from_dir__(self, devops: Devops): self.file_api.cp_force( - image.docker_build_commons_path(), image.devops.build_path() + image.build_commons_path(), devops.build_path() ) def initialize_build_dir(self, devops: Devops): - image = devops.specialized_build - build_path = image.devops.build_path() + image = devops.specialized_builds[BuildType.C4K] + build_path = devops.build_path() self.file_api.clean_dir(f"{build_path}/image/resources") - if image.use_package_common_files: + if image.image_use_package_common_files: self.__copy_build_resources_from_package__(image) else: self.__copy_build_resources_from_dir__(image) @@ -36,25 +36,24 @@ class ImageBuildService: self.file_api.cp_recursive("test", build_path) def image(self, devops: Devops): - image = devops.specialized_build - self.image_api.image(image.devops.name, image.devops.build_path()) + self.image_api.image(devops.name, devops.build_path()) def drun(self, devops: Devops): - image = devops.specialized_build - self.image_api.drun(image.devops.name) + self.image_api.drun(devops.name) def dockerhub_login(self, devops: Devops): - image = devops.specialized_build + image = devops.specialized_builds[BuildType.C4K] self.image_api.dockerhub_login( - image.dockerhub_user, image.dockerhub_password + image.image_dockerhub_user, image.image_dockerhub_password ) def dockerhub_publish(self, devops: Devops): - image = devops.specialized_build + image = devops.specialized_builds[BuildType.C4K] + if image.image_tag is None or image.image_tag == "": + raise ValueError(f"image_tag must not be empty.") self.image_api.dockerhub_publish( - image.devops.name, image.dockerhub_user, image.docker_publish_tag + devops.name, image.image_dockerhub_user, image.image_tag ) def test(self, devops: Devops): - image = devops.specialized_build - self.image_api.test(image.devops.name, image.devops.build_path()) + self.image_api.test(devops.name, devops.build_path()) diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index d5f9cc2..165585d 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -2,7 +2,6 @@ from typing import Optional, List from .common import ( filter_none, Validateable, - Devops, ) @@ -11,23 +10,26 @@ class Image(Validateable): self, input: dict, ): - self.dockerhub_user = input.get("image_dockerhub_user") - self.dockerhub_password = input.get("image_dockerhub_password") - # TODO: rename to image_tag - self.docker_publish_tag = input.get("image_tag") - self.build_commons_path = input.get("image_build_commons_path") - self.use_package_common_files = input.get("image_use_package_common_files", True) - # TODO: rename to image_build_commons_dir_name - self.docker_build_commons_dir_name = input.get( + self.image_dockerhub_user = input.get("image_dockerhub_user") + self.image_dockerhub_password = input.get("image_dockerhub_password") + self.image_tag = input.get("image_tag") + self.image_build_commons_path = input.get("image_build_commons_path") + self.image_use_package_common_files = input.get( + "image_use_package_common_files", True + ) + self.image_build_commons_dir_name = input.get( "image_build_commons_dir_name", "docker" ) def validate(self) -> List[str]: result = [] - result += self.__validate_is_not_empty__("dockerhub_user") - result += self.__validate_is_not_empty__("dockerhub_password") + result += self.__validate_is_not_empty__("image_dockerhub_user") + result += self.__validate_is_not_empty__("image_dockerhub_password") return result - def docker_build_commons_path(self): - commons_path = [self.build_commons_path, self.docker_build_commons_dir_name] + def build_commons_path(self): + commons_path = [ + self.image_build_commons_path, + self.image_build_commons_dir_name, + ] return "/".join(filter_none(commons_path)) + "/" diff --git a/src/test/python/domain/test_common.py b/src/test/python/domain/test_common.py index 5c2bd6d..9b4e21f 100644 --- a/src/test/python/domain/test_common.py +++ b/src/test/python/domain/test_common.py @@ -66,61 +66,3 @@ def test_should_validate_DnsRecord(): sut = DnsRecord("name", ipv6="1::") assert sut.is_valid() - -def test_version(tmp_path: Path): - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.SNAPSHOT) - assert version.get_version_string() == "1.2.3-SNAPSHOT" - assert version.version_list == [1, 2, 3] - assert version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.BUMP) - assert version.get_version_string() == "1.2.4-SNAPSHOT" - assert version.version_list == [1, 2, 4] - assert version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.PATCH) - assert version.get_version_string() == "1.2.4" - assert version.version_list == [1, 2, 4] - assert not version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.MINOR) - assert version.get_version_string() == "1.3.0" - assert version.version_list == [1, 3, 0] - assert not version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.MAJOR) - assert version.get_version_string() == "2.0.0" - assert version.version_list == [2, 0, 0] - assert not version.is_snapshot - - -def test_release_context(tmp_path): - version = Version(tmp_path, [1, 2, 3]) - release = ReleaseContext(ReleaseType.MINOR, version, "main") - - release_version = release.release_version() - assert release_version.get_version_string() in "1.3.0" - - bump_version = release.bump_version() - assert bump_version.get_version_string() in "1.3.1-SNAPSHOT" - - -def test_release(tmp_path): - devops = build_devops({}) - sut = Release(devops, "main", "config_file.json") - assert not sut.is_valid() - - sut.set_release_context( - ReleaseContext(ReleaseType.MINOR, Version("id", [1, 2, 3]), "main") - ) - assert sut.is_valid() - - -def test_devops_build_commons_path(): - sut = build_devops({}) - assert "docker/" == sut.specialized_builds[BuildType.IMAGE].docker_build_commons_path() diff --git a/src/test/python/domain/test_image.py b/src/test/python/domain/test_image.py new file mode 100644 index 0000000..61d3350 --- /dev/null +++ b/src/test/python/domain/test_image.py @@ -0,0 +1,14 @@ +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain.common import ( + BuildType, +) +from .test_helper import build_devops + + +def test_devops_build_commons_path(): + sut = build_devops({}) + image = sut.specialized_builds[BuildType.IMAGE] + assert image is not None + assert image.is_valid() + assert "docker/" == image.build_commons_path() diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py new file mode 100644 index 0000000..5af241b --- /dev/null +++ b/src/test/python/domain/test_release.py @@ -0,0 +1,69 @@ +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain.common import ( + Validateable, + DnsRecord, + Devops, + BuildType, +) +from src.main.python.ddadevops.domain import ( + Version, + ReleaseType, + Release, + ReleaseContext, +) +from src.main.python.ddadevops.domain.image import Image +from .test_helper import build_devops + +def test_version(tmp_path: Path): + version = Version(tmp_path, [1, 2, 3]) + version.increment(ReleaseType.SNAPSHOT) + assert version.get_version_string() == "1.2.3-SNAPSHOT" + assert version.version_list == [1, 2, 3] + assert version.is_snapshot + + version = Version(tmp_path, [1, 2, 3]) + version.increment(ReleaseType.BUMP) + assert version.get_version_string() == "1.2.4-SNAPSHOT" + assert version.version_list == [1, 2, 4] + assert version.is_snapshot + + version = Version(tmp_path, [1, 2, 3]) + version.increment(ReleaseType.PATCH) + assert version.get_version_string() == "1.2.4" + assert version.version_list == [1, 2, 4] + assert not version.is_snapshot + + version = Version(tmp_path, [1, 2, 3]) + version.increment(ReleaseType.MINOR) + assert version.get_version_string() == "1.3.0" + assert version.version_list == [1, 3, 0] + assert not version.is_snapshot + + version = Version(tmp_path, [1, 2, 3]) + version.increment(ReleaseType.MAJOR) + assert version.get_version_string() == "2.0.0" + assert version.version_list == [2, 0, 0] + assert not version.is_snapshot + + +def test_release_context(tmp_path): + version = Version(tmp_path, [1, 2, 3]) + release = ReleaseContext(ReleaseType.MINOR, version, "main") + + release_version = release.release_version() + assert release_version.get_version_string() in "1.3.0" + + bump_version = release.bump_version() + assert bump_version.get_version_string() in "1.3.1-SNAPSHOT" + + +def test_release(tmp_path): + devops = build_devops({}) + sut = Release(devops, "main", "config_file.json") + assert not sut.is_valid() + + sut.set_release_context( + ReleaseContext(ReleaseType.MINOR, Version("id", [1, 2, 3]), "main") + ) + assert sut.is_valid() From d670605d370b46927aa0cd8fd5498617f7833d96 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 29 Apr 2023 22:03:44 +0200 Subject: [PATCH 048/173] integrate release as mixin --- doc/architecture/Domain.md | 2 +- src/main/python/ddadevops/domain/common.py | 33 +++++++--- .../python/ddadevops/domain/devops_factory.py | 19 +++++- src/main/python/ddadevops/domain/release.py | 63 ++++++++++++------- src/test/python/domain/test_devops_factory.py | 23 ++++++- src/test/python/domain/test_helper.py | 10 ++- src/test/python/domain/test_release.py | 36 ++++++----- 7 files changed, 133 insertions(+), 53 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index d65a55e..ce9f80b 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -45,7 +45,7 @@ classDiagram Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds - Devops *-- Release: release + Devops *-- "0..1" Release: mixins C4k *-- DnsRecord Release *-- "0..1" ReleaseContext diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 1a3594c..0a599cf 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -4,13 +4,20 @@ from typing import List, TypedDict import logging import deprecation + def filter_none(list_to_filter): return [x for x in list_to_filter if x is not None] + class BuildType(Enum): IMAGE = 0 C4K = 1 + +class MixinType(Enum): + RELEASE = 0 + + class Validateable: def __validate_is_not_none__(self, field_name: str) -> List[str]: value = self.__dict__[field_name] @@ -32,9 +39,10 @@ class Validateable: def throw_if_invalid(self): if not self.is_valid(): - issues = '\n'.join(self.validate()) + issues = "\n".join(self.validate()) raise ValueError(f"Invalid Validateable: {issues}") + class DnsRecord(Validateable): def __init__(self, fqdn, ipv4=None, ipv6=None): self.fqdn = fqdn @@ -50,13 +58,19 @@ class DnsRecord(Validateable): class Devops(Validateable): - def __init__(self, input: dict, specialized_builds: dict[BuildType, Validateable]): - self.stage = input.get('stage') - self.project_root_path = input.get('project_root_path') - self.module = input.get('module') - self.name = input.get('name', self.module) - self.build_dir_name = input.get('build_dir_name', 'target') - self.specialized_builds=specialized_builds + def __init__( + self, + input: dict, + specialized_builds: dict[BuildType, Validateable], + mixins: dict[MixinType, Validateable], + ): + self.stage = input.get("stage") + self.project_root_path = input.get("project_root_path") + self.module = input.get("module") + self.name = input.get("name", self.module) + self.build_dir_name = input.get("build_dir_name", "target") + self.specialized_builds = specialized_builds + self.mixins = mixins def build_path(self): path = [self.project_root_path, self.build_dir_name, self.name, self.module] @@ -71,6 +85,9 @@ class Devops(Validateable): if self.specialized_builds: for build in self.specialized_builds: result += self.specialized_builds[build].validate() + if self.mixins: + for mixin in self.mixins: + result += self.mixins[mixin].validate() return result def __put__(self, key, value): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index a16a0b9..caac032 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -1,9 +1,11 @@ import deprecation from enum import Enum from typing import List -from .common import Devops, BuildType +from .common import Devops, BuildType, MixinType from .image import Image from .c4k import C4k +from .release import Release, ReleaseContext + class DevopsFactory: def __init__(self): @@ -11,13 +13,20 @@ class DevopsFactory: def build_devops(self, input) -> Devops: build_types = self.__parse_build_types__(input["build_types"]) + mixin_types = self.__parse_mixin_types__(input["mixin_types"]) + specialized_builds = {} if BuildType.IMAGE in build_types: specialized_builds[BuildType.IMAGE] = Image(input) if BuildType.C4K in build_types: specialized_builds[BuildType.C4K] = C4k(input) - devops = Devops(input, specialized_builds=specialized_builds) + mixins = {} + if MixinType.RELEASE in mixin_types: + release_context = ReleaseContext(input) + mixins[MixinType.RELEASE] = Release(input, release_context=release_context) + + devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) devops.throw_if_invalid() @@ -31,3 +40,9 @@ class DevopsFactory: for build_type in build_types: result += [BuildType[build_type]] return result + + def __parse_mixin_types__(self, mixin_types: List[str]) -> List[MixinType]: + result = [] + for mixin_type in mixin_types: + result += [MixinType[mixin_type]] + return result diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 2c27b4a..b10a085 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -6,18 +6,21 @@ from .common import ( Devops, ) + class ReleaseType(Enum): MAJOR = 0 MINOR = 1 PATCH = 2 SNAPSHOT = 3 BUMP = None + NONE = 15 + class EnvironmentKeys(Enum): DDADEVOPS_RELEASE_TYPE = 0 -class Version(): +class Version(Validateable): def __init__(self, path: Path, version_list: list): self.path = path self.version_list = version_list @@ -62,11 +65,25 @@ class Version(): bump_version.increment(ReleaseType.BUMP) return bump_version + class ReleaseContext(Validateable): - def __init__(self, release_type: ReleaseType | None, version: Version, current_branch: str): - self.release_type = release_type - self.version = version - self.current_branch = current_branch + def __init__( + self, + input: dict, + ): + self.release_type = ReleaseType[input.get("release_type", "SNAPSHOT")] + self.release_current_version = input.get("release_current_version") + self.release_current_branch = input.get("release_current_branch") + self.version = self.__version_from_str__() + + # TODO: mv version parsing to version + def __version_from_str__(self): + if not self.release_current_version: + return + version_parsed = [] + for x in self.release_current_version.split("."): + version_parsed += [int(x)] + return Version("unused", version_parsed) def release_version(self) -> Version: return self.version.create_release_version(self.release_type) @@ -77,30 +94,32 @@ class ReleaseContext(Validateable): def validate(self): result = [] result += self.__validate_is_not_empty__("release_type") - result += self.__validate_is_not_empty__("version") - result += self.__validate_is_not_empty__("current_branch") + result += self.__validate_is_not_empty__("release_current_version") + result += self.__validate_is_not_empty__("release_current_branch") + if self.version: + result += self.version.validate() return result def validate_branch(self, main_branch: str): result = [] - if self.release_type is not None and main_branch != self.current_branch: + if ( + self.release_type is not None + and self.release_type != ReleaseType.NONE + and main_branch != self.release_current_branch + ): result.append(f"Releases are allowed only on {main_branch}") return result + class Release(Validateable): def __init__( self, - devops: Devops, - main_branch: str, - config_file: str, + input: dict, + release_context: ReleaseContext, ): - self.devops = devops - self.main_branch = main_branch - self.config_file = config_file - self.release_context: ReleaseContext | None = None - - def set_release_context(self, set_release_context: ReleaseContext): - self.release_context = set_release_context + self.release_main_branch = input.get("release_main_branch", "main") + self.release_config_file = input.get("release_config_file", "project.clj") + self.release_context = release_context def release_version(self): return self.release_context.release_version() @@ -108,14 +127,12 @@ class Release(Validateable): def bump_version(self): return self.release_context.bump_version() - def validate(self): result = [] - result += self.__validate_is_not_empty__("main_branch") - result += self.__validate_is_not_empty__("config_file") + result += self.__validate_is_not_empty__("release_main_branch") + result += self.__validate_is_not_empty__("release_config_file") result += self.__validate_is_not_empty__("release_context") if self.release_context is not None: result += self.release_context.validate() - result += self.release_context.validate_branch(self.main_branch) + result += self.release_context.validate_branch(self.release_main_branch) return result - \ No newline at end of file diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index 056e75c..ab4e414 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -22,12 +22,13 @@ def test_devops_factory(): "module": "test_image", "project_root_path": "../../..", "build_types": ["IMAGE"], + "mixin_types": [], "image_dockerhub_user": "dockerhub_user", "image_dockerhub_password": "dockerhub_password", "image_tag": "docker_image_tag", } ) - assert sut != None + assert sut is not None sut = DevopsFactory().build_devops( { @@ -36,8 +37,26 @@ def test_devops_factory(): "module": "test_image", "project_root_path": "../../..", "build_types": ["C4K"], + "mixin_types": [], "c4k_grafana_cloud_user": "user", "c4k_grafana_cloud_password": "password", } ) - assert sut != None + assert sut is not None + + sut = DevopsFactory().build_devops( + { + "stage": "test", + "name": "mybuild", + "module": "test_image", + "project_root_path": "../../..", + "build_types": [], + "mixin_types": ["RELEASE"], + "release_main_branch": "main", + "release_config_file": "project.clj", + "release_type": "NONE", + "release_current_version": "1.0.0", + "release_current_branch": "my_feature", + } + ) + assert sut is not None diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index 4d96b4e..391f729 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -9,14 +9,20 @@ def devops_config(overrides: dict) -> dict: "project_root_path": "../../..", "build_dir_name": "target", "build_types": ["IMAGE", "C4K"], + "mixin_types": ["RELEASE"], "image_dockerhub_user": "dockerhub_user", "image_dockerhub_password": "dockerhub_password", "image_tag": "image_tag", - 'c4k_config': {}, + "c4k_config": {}, "c4k_grafana_cloud_user": "user", "c4k_grafana_cloud_password": "password", "c4k_grafana_cloud_url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", - 'c4k_auth': {}, + "c4k_auth": {}, + "release_main_branch": "main", + "release_config_file": "project.clj", + "release_type": "NONE", + "release_current_version": "1.0.0", + "release_current_branch": "my_feature", } input = default.copy() input.update(overrides) diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index 5af241b..b81d29a 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -5,6 +5,7 @@ from src.main.python.ddadevops.domain.common import ( DnsRecord, Devops, BuildType, + MixinType, ) from src.main.python.ddadevops.domain import ( Version, @@ -13,7 +14,8 @@ from src.main.python.ddadevops.domain import ( ReleaseContext, ) from src.main.python.ddadevops.domain.image import Image -from .test_helper import build_devops +from .test_helper import build_devops, devops_config + def test_version(tmp_path: Path): version = Version(tmp_path, [1, 2, 3]) @@ -48,22 +50,26 @@ def test_version(tmp_path: Path): def test_release_context(tmp_path): - version = Version(tmp_path, [1, 2, 3]) - release = ReleaseContext(ReleaseType.MINOR, version, "main") - - release_version = release.release_version() - assert release_version.get_version_string() in "1.3.0" - - bump_version = release.bump_version() - assert bump_version.get_version_string() in "1.3.1-SNAPSHOT" + sut = ReleaseContext( + devops_config( + { + "release_type": "MINOR", + "release_current_version": "1.2.3", + "release_current_branch": "main", + } + ) + ) + assert sut.release_version().get_version_string() == "1.3.0" + assert sut.bump_version().get_version_string() == "1.3.1-SNAPSHOT" def test_release(tmp_path): - devops = build_devops({}) - sut = Release(devops, "main", "config_file.json") - assert not sut.is_valid() - - sut.set_release_context( - ReleaseContext(ReleaseType.MINOR, Version("id", [1, 2, 3]), "main") + sut = build_devops( + { + "release_type": "MINOR", + "release_current_version": "1.2.3", + "release_current_branch": "main", + } ) + assert sut.mixins[MixinType.RELEASE] assert sut.is_valid() From 039a5da8f6ded5a77b33d43c77b72f5fc7a51294 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sun, 30 Apr 2023 11:29:17 +0200 Subject: [PATCH 049/173] mv test to appr. place --- src/main/python/ddadevops/domain/__init__.py | 2 +- .../release_mixin/test_release_mixin.py | 80 -------------- src/test/python/resource_helper.py | 12 ++ src/test/python/test_devops_build.py | 19 ++-- src/test/python/test_release_mixin.py | 103 ++++++++++++++++++ 5 files changed, 125 insertions(+), 91 deletions(-) delete mode 100644 src/test/python/release_mixin/test_release_mixin.py create mode 100644 src/test/python/resource_helper.py create mode 100644 src/test/python/test_release_mixin.py diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 4bd528a..8121e0d 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,4 +1,4 @@ -from .common import Validateable, DnsRecord, Devops, BuildType +from .common import Validateable, DnsRecord, Devops, BuildType, MixinType from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k diff --git a/src/test/python/release_mixin/test_release_mixin.py b/src/test/python/release_mixin/test_release_mixin.py deleted file mode 100644 index 62a890d..0000000 --- a/src/test/python/release_mixin/test_release_mixin.py +++ /dev/null @@ -1,80 +0,0 @@ -import pytest as pt -from pathlib import Path -from pybuilder.core import Project - -from src.main.python.ddadevops.release_mixin import ReleaseMixin -from src.main.python.ddadevops.infrastructure.release_mixin import GitApi, EnvironmentApi -from src.main.python.ddadevops.domain import Devops, ReleaseContext, Release - -from .helper import Helper - -MAIN_BRANCH = 'main' -STAGE = 'test' -PROJECT_ROOT_PATH = '.' -MODULE = 'test' -BUILD_DIR_NAME = "build_dir" - -def change_test_dir( tmp_path: Path, monkeypatch: pt.MonkeyPatch): - monkeypatch.chdir(tmp_path) - -class MyBuild(ReleaseMixin): - pass - -def initialize_with_object(project, CONFIG_FILE): - project.build_depends_on('ddadevops>=3.1.2') - devops = Devops(STAGE, PROJECT_ROOT_PATH, MODULE, "release_test", BUILD_DIR_NAME) - release = Release(devops, MAIN_BRANCH, CONFIG_FILE) - build = MyBuild(project, release) - return build - -def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch): - - # init - th = Helper() - th.copy_files(th.TEST_FILE_PATH, tmp_path) - th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - - change_test_dir(tmp_path, monkeypatch) - project = Project(tmp_path) - - git_api = GitApi() - git_api.init() - git_api.set_user_config("ex.ample@mail.com", "Ex Ample") - git_api.add_file(th.TEST_FILE_NAME) - git_api.commit("MAJOR release") - - build = initialize_with_object(project, th.TEST_FILE_PATH) - build.prepare_release() - release_version = build.release_repo.version_repository.get_version() - - # test - assert "124.0.1-SNAPSHOT" in release_version.get_version_string() - -def test_release_mixin_environment(tmp_path: Path, monkeypatch: pt.MonkeyPatch): - - # init - th = Helper() - th.copy_files(th.TEST_FILE_PATH, tmp_path) - th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - - change_test_dir(tmp_path, monkeypatch) - project = Project(tmp_path) - - git_api = GitApi() - git_api.init() - git_api.set_user_config("ex.ample@mail.com", "Ex Ample") - git_api.add_file(th.TEST_FILE_NAME) - git_api.commit("Commit Message") - - environment_api = EnvironmentApi() - environment_api.set("DDADEVOPS_RELEASE_TYPE", "MAJOR") - - build = initialize_with_object(project, th.TEST_FILE_PATH) - build.prepare_release() - release_version = build.release_repo.version_repository.get_version() - - # test - assert "124.0.1-SNAPSHOT" in release_version.get_version_string() - - # tear down - environment_api.set("DDADEVOPS_RELEASE_TYPE", "") diff --git a/src/test/python/resource_helper.py b/src/test/python/resource_helper.py new file mode 100644 index 0000000..7de20b5 --- /dev/null +++ b/src/test/python/resource_helper.py @@ -0,0 +1,12 @@ +from pathlib import Path +from src.main.python.ddadevops.infrastructure import ExecutionApi + +class ResourceHelper(): + def __init__(self, file_name = 'config.json'): + self.TEST_FILE_NAME = file_name + self.TEST_FILE_ROOT = Path('src/test/resources/') + self.TEST_FILE_PATH = self.TEST_FILE_ROOT / self.TEST_FILE_NAME + + def copy_files(self, source: Path, target: Path): + api = ExecutionApi() + api.execute(f"cp {source} {target}") diff --git a/src/test/python/test_devops_build.py b/src/test/python/test_devops_build.py index 006d46b..c107058 100644 --- a/src/test/python/test_devops_build.py +++ b/src/test/python/test_devops_build.py @@ -1,21 +1,20 @@ import os from pybuilder.core import Project -from src.main.python.ddadevops.domain.common import Devops from src.main.python.ddadevops.devops_build import DevopsBuild from .domain.test_helper import devops_config -class MyDevopsBuild(DevopsBuild): - pass - - def test_devops_build(tmp_path): - build_dir = "build" - project_name = "testing-project" - module_name = "c4k-test" tmp_path_str = str(tmp_path) - project = Project(tmp_path_str, name=project_name) - devops_build = DevopsBuild(project, devops_config({})) + project = Project(tmp_path_str, name="name") + devops_build = DevopsBuild( + project, + devops_config( + { + "project_root_path": tmp_path_str, + } + ), + ) devops_build.initialize_build_dir() assert os.path.exists(f"{devops_build.build_path()}") diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py new file mode 100644 index 0000000..b0772a3 --- /dev/null +++ b/src/test/python/test_release_mixin.py @@ -0,0 +1,103 @@ +import pytest as pt +from pathlib import Path +from pybuilder.core import Project + +from src.main.python.ddadevops.release_mixin import ReleaseMixin +from src.main.python.ddadevops.infrastructure.release_mixin import ( + GitApi, + EnvironmentApi, +) +from src.main.python.ddadevops.domain import Devops, ReleaseContext, Release + +from .resource_helper import ResourceHelper +from .domain.test_helper import devops_config + +MAIN_BRANCH = "main" +STAGE = "test" +PROJECT_ROOT_PATH = "." +MODULE = "test" +BUILD_DIR_NAME = "build_dir" + +# def change_test_dir( tmp_path: Path, monkeypatch: pt.MonkeyPatch): +# monkeypatch.chdir(tmp_path) + +# class MyBuild(ReleaseMixin): +# pass + + +def test_release_mixin(tmp_path): + tmp_path_str = str(tmp_path) + + project = Project(tmp_path_str, name="name") + sut = ReleaseMixin( + project, + devops_config( + { + "project_root_path": tmp_path_str, + "build_types": [], + "mixin_types": ["RELEASE"], + } + ), + ) + sut.initialize_build_dir() + assert os.path.exists(f"{devops_build.build_path()}") + + +# def initialize_with_object(project, CONFIG_FILE): +# project.build_depends_on('ddadevops>=3.1.2') +# devops = Devops(STAGE, PROJECT_ROOT_PATH, MODULE, "release_test", BUILD_DIR_NAME) +# release = Release(devops, MAIN_BRANCH, CONFIG_FILE) +# build = MyBuild(project, release) +# return build + +# def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch): + +# # init +# th = Helper() +# th.copy_files(th.TEST_FILE_PATH, tmp_path) +# th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME + +# change_test_dir(tmp_path, monkeypatch) +# project = Project(tmp_path) + +# git_api = GitApi() +# git_api.init() +# git_api.set_user_config("ex.ample@mail.com", "Ex Ample") +# git_api.add_file(th.TEST_FILE_NAME) +# git_api.commit("MAJOR release") + +# build = initialize_with_object(project, th.TEST_FILE_PATH) +# build.prepare_release() +# release_version = build.release_repo.version_repository.get_version() + +# # test +# assert "124.0.1-SNAPSHOT" in release_version.get_version_string() + +# def test_release_mixin_environment(tmp_path: Path, monkeypatch: pt.MonkeyPatch): + +# # init +# th = Helper() +# th.copy_files(th.TEST_FILE_PATH, tmp_path) +# th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME + +# change_test_dir(tmp_path, monkeypatch) +# project = Project(tmp_path) + +# git_api = GitApi() +# git_api.init() +# git_api.set_user_config("ex.ample@mail.com", "Ex Ample") +# git_api.add_file(th.TEST_FILE_NAME) +# git_api.commit("Commit Message") + +# environment_api = EnvironmentApi() +# environment_api.set("DDADEVOPS_RELEASE_TYPE", "MAJOR") + +# build = initialize_with_object(project, th.TEST_FILE_PATH) +# build.prepare_release() +# release_version = build.release_repo.version_repository.get_version() + +# # test +# assert "124.0.1-SNAPSHOT" in release_version.get_version_string() + +# # tear down +# environment_api.set("DDADEVOPS_RELEASE_TYPE", "") From 8a864153d90e42faf70b7baec94041e29f93dae6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 1 May 2023 12:08:41 +0200 Subject: [PATCH 050/173] mv test to appr. place --- src/main/python/ddadevops/release_mixin.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 597ee6a..22446c9 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -2,29 +2,29 @@ from pybuilder.core import Project from src.main.python.ddadevops.devops_build import DevopsBuild from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, ReleaseTypeRepository, VersionRepository, GitApi, EnvironmentApi from src.main.python.ddadevops.application import PrepareReleaseService, TagAndPushReleaseService -from src.main.python.ddadevops.domain import Release, EnvironmentKeys +from src.main.python.ddadevops.domain import Release, EnvironmentKeys, MixinType class ReleaseMixin(DevopsBuild): - def __init__(self, project: Project, release: Release): - super().__init__(project, devops=release.devops) - self.repo.set_release(self.project, release) - self.main_branch = release.main_branch - + def __init__(self, project: Project, input: dict): + super().__init__(project, input) + devops = self.repo.get_devops(self.project) git_api = GitApi() + self.tag_and_push_release_service = TagAndPushReleaseService(git_api) environment_api = EnvironmentApi() + if MixinType.RELEASE not in devops.mixins: + raise ValueError(f"ReleaseMixin requires MixinType.RELEASE") + + # TODO: move this to service env_key = EnvironmentKeys.DDADEVOPS_RELEASE_TYPE.name environment_val_set = environment_api.get(env_key) != "" and environment_api.get(env_key) is not None - if environment_val_set: release_type_repo = ReleaseTypeRepository.from_environment(environment_api) else: release_type_repo = ReleaseTypeRepository.from_git(git_api) - version_repo = VersionRepository(release.config_file) self.release_repo = ReleaseContextRepository(version_repo, release_type_repo) - + # Here the initialization can happen self.prepare_release_service = PrepareReleaseService() - self.tag_and_push_release_service = TagAndPushReleaseService(git_api) def prepare_release(self): release = self.release_repo.get_release(self.main_branch) From df1200de4a5a77babe090e36cc73e8a448f23845 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 2 May 2023 08:55:09 +0200 Subject: [PATCH 051/173] bring release_mixin partially back to work --- src/main/python/ddadevops/release_mixin.py | 6 +- src/test/python/test_release_mixin.py | 71 +++++++++++++--------- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 22446c9..c2d93c6 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -15,19 +15,21 @@ class ReleaseMixin(DevopsBuild): raise ValueError(f"ReleaseMixin requires MixinType.RELEASE") # TODO: move this to service + release = devops.mixins[MixinType.RELEASE] env_key = EnvironmentKeys.DDADEVOPS_RELEASE_TYPE.name environment_val_set = environment_api.get(env_key) != "" and environment_api.get(env_key) is not None if environment_val_set: release_type_repo = ReleaseTypeRepository.from_environment(environment_api) else: release_type_repo = ReleaseTypeRepository.from_git(git_api) - version_repo = VersionRepository(release.config_file) + version_repo = VersionRepository(release.release_config_file) self.release_repo = ReleaseContextRepository(version_repo, release_type_repo) # Here the initialization can happen self.prepare_release_service = PrepareReleaseService() def prepare_release(self): - release = self.release_repo.get_release(self.main_branch) + devops = self.repo.get_devops(self.project) + release = devops.mixins[MixinType.RELEASE] self.prepare_release_service.write_and_commit_release(release, self.release_repo.version_repository) self.prepare_release_service.write_and_commit_bump(release, self.release_repo.version_repository) diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py index b0772a3..2e13ea6 100644 --- a/src/test/python/test_release_mixin.py +++ b/src/test/python/test_release_mixin.py @@ -18,11 +18,30 @@ PROJECT_ROOT_PATH = "." MODULE = "test" BUILD_DIR_NAME = "build_dir" -# def change_test_dir( tmp_path: Path, monkeypatch: pt.MonkeyPatch): -# monkeypatch.chdir(tmp_path) -# class MyBuild(ReleaseMixin): -# pass +def change_test_dir(tmp_path: Path, monkeypatch: pt.MonkeyPatch): + monkeypatch.chdir(tmp_path) + + +def initialize_with_object(project, CONFIG_FILE): + project.build_depends_on("ddadevops>=3.1.2") + build = ReleaseMixin( + project, + devops_config( + { + "name": "release_test", + "stage": "test", + "module": MODULE, + "project_root_path": PROJECT_ROOT_PATH, + "build_types": [], + "mixin_types": ["RELEASE"], + "build_dir_name": BUILD_DIR_NAME, + "release_main_branch": MAIN_BRANCH, + "release_config_file": CONFIG_FILE, + } + ), + ) + return build def test_release_mixin(tmp_path): @@ -39,39 +58,31 @@ def test_release_mixin(tmp_path): } ), ) - sut.initialize_build_dir() - assert os.path.exists(f"{devops_build.build_path()}") + assert sut is not None -# def initialize_with_object(project, CONFIG_FILE): -# project.build_depends_on('ddadevops>=3.1.2') -# devops = Devops(STAGE, PROJECT_ROOT_PATH, MODULE, "release_test", BUILD_DIR_NAME) -# release = Release(devops, MAIN_BRANCH, CONFIG_FILE) -# build = MyBuild(project, release) -# return build +def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch): + # init + th = ResourceHelper() + th.copy_files(th.TEST_FILE_PATH, tmp_path) + th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME -# def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch): + change_test_dir(tmp_path, monkeypatch) + project = Project(tmp_path) -# # init -# th = Helper() -# th.copy_files(th.TEST_FILE_PATH, tmp_path) -# th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME + git_api = GitApi() + git_api.init() + git_api.set_user_config("ex.ample@mail.com", "Ex Ample") + git_api.add_file(th.TEST_FILE_NAME) + git_api.commit("MAJOR release") -# change_test_dir(tmp_path, monkeypatch) -# project = Project(tmp_path) + build = initialize_with_object(project, th.TEST_FILE_PATH) + build.prepare_release() + release_version = build.release_repo.version_repository.get_version() -# git_api = GitApi() -# git_api.init() -# git_api.set_user_config("ex.ample@mail.com", "Ex Ample") -# git_api.add_file(th.TEST_FILE_NAME) -# git_api.commit("MAJOR release") + # test + assert "124.0.1-SNAPSHOT" in release_version.get_version_string() -# build = initialize_with_object(project, th.TEST_FILE_PATH) -# build.prepare_release() -# release_version = build.release_repo.version_repository.get_version() - -# # test -# assert "124.0.1-SNAPSHOT" in release_version.get_version_string() # def test_release_mixin_environment(tmp_path: Path, monkeypatch: pt.MonkeyPatch): From 97c5c4eb91b327eaed918f0ba3f626199f841fbd Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 2 May 2023 12:54:31 +0200 Subject: [PATCH 052/173] Add missing arguments --- .../python/ddadevops/application/release_mixin_services.py | 5 +++-- src/main/python/ddadevops/release_mixin.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 22c904c..6b33e52 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -22,11 +22,12 @@ class PrepareReleaseService(): class TagAndPushReleaseService(): - def __init__(self, git_api: GitApi): + def __init__(self, git_api: GitApi, main_branch: str): self.git_api = git_api + self.main_branch = main_branch def tag_release(self, release_repo: ReleaseContextRepository): - annotation = 'v' + release_repo.get_release().version.get_version_string() + annotation = 'v' + release_repo.get_release(self.main_branch).version.get_version_string() message = 'Release ' + annotation self.git_api.tag_annotated_second_last(annotation, message) diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 597ee6a..d30e6d5 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -24,7 +24,7 @@ class ReleaseMixin(DevopsBuild): self.release_repo = ReleaseContextRepository(version_repo, release_type_repo) self.prepare_release_service = PrepareReleaseService() - self.tag_and_push_release_service = TagAndPushReleaseService(git_api) + self.tag_and_push_release_service = TagAndPushReleaseService(git_api, self.main_branch) def prepare_release(self): release = self.release_repo.get_release(self.main_branch) From 8fd243a60d199e7cd97183a151cfdeac708a0218 Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 2 May 2023 13:10:27 +0200 Subject: [PATCH 053/173] WIP fixing tests --- src/test/python/release_mixin/mock_infrastructure.py | 7 +++---- src/test/python/release_mixin/test_services.py | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/test/python/release_mixin/mock_infrastructure.py b/src/test/python/release_mixin/mock_infrastructure.py index 81b09b5..78c731b 100644 --- a/src/test/python/release_mixin/mock_infrastructure.py +++ b/src/test/python/release_mixin/mock_infrastructure.py @@ -32,12 +32,11 @@ class MockReleaseTypeRepository(): return ReleaseType.MINOR class MockReleaseRepository(): - def __init__(self, version_repository: MockVersionRepository, release_type_repository: MockReleaseTypeRepository, main_branch: str): + def __init__(self, version_repository: MockVersionRepository, release_type_repository: MockReleaseTypeRepository): self.version_repository = version_repository self.release_type_repository = release_type_repository - self.main_branch = main_branch self.get_release_count = 0 - def get_release(self) -> ReleaseContext: + def get_release(self, main_branch) -> ReleaseContext: self.get_release_count += 1 - return ReleaseContext(self.release_type_repository.get_release_type(), self.version_repository.get_version(), self.main_branch) + return ReleaseContext(self.release_type_repository.get_release_type(), self.version_repository.get_version(), main_branch) diff --git a/src/test/python/release_mixin/test_services.py b/src/test/python/release_mixin/test_services.py index 35063d8..c7dbf80 100644 --- a/src/test/python/release_mixin/test_services.py +++ b/src/test/python/release_mixin/test_services.py @@ -4,17 +4,17 @@ from src.test.python.release_mixin import MockGitApi def test_prepare_release_service(): # init - mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi()), 'main') + mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi())) prepare_release_service = PrepareReleaseService() prepare_release_service.git_api = MockGitApi() - prepare_release_service.write_and_commit_release(mock_release_repo.get_release(), mock_release_repo.version_repository) + prepare_release_service.write_and_commit_release(mock_release_repo.get_release("main"), mock_release_repo.version_repository) #test assert prepare_release_service.git_api.add_file_count == 1 assert prepare_release_service.git_api.commit_count == 1 # init - prepare_release_service.write_and_commit_bump(mock_release_repo.get_release(), mock_release_repo.version_repository) + prepare_release_service.write_and_commit_bump(mock_release_repo.get_release("main"), mock_release_repo.version_repository) # test assert prepare_release_service.git_api.add_file_count == 2 @@ -22,8 +22,8 @@ def test_prepare_release_service(): def test_tag_and_push_release_service(): # init - mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi()), 'main') - tag_and_push_release_service = TagAndPushReleaseService(MockGitApi()) + mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi())) + tag_and_push_release_service = TagAndPushReleaseService(MockGitApi(), "main") tag_and_push_release_service.tag_release(mock_release_repo) tag_and_push_release_service.push_release() From e27db8647777f61b63b6a972b8af18907ca1d0ab Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 2 May 2023 13:23:42 +0200 Subject: [PATCH 054/173] Fix logical error Using match would always produce None results, as we are matching a string exactly. Reintroducing more fuzzy "in" logic fixed tests. --- .../infrastructure/release_mixin/repo.py | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 588118a..dc27457 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -63,41 +63,36 @@ class ReleaseTypeRepository: return release_type_repo def __get_release_type_git(self) -> ReleaseType | None: - latest_commit = self.git_api.get_latest_commit() - - match latest_commit.upper(): - case ReleaseType.MAJOR.name: - return ReleaseType.MAJOR - case ReleaseType.MINOR.name: - return ReleaseType.MINOR - case ReleaseType.PATCH.name: - return ReleaseType.PATCH - case ReleaseType.SNAPSHOT.name: - return ReleaseType.SNAPSHOT - case _: - return None + latest_commit = self.git_api.get_latest_commit().upper() + if ReleaseType.MAJOR.name in latest_commit: + return ReleaseType.MAJOR + if ReleaseType.MINOR.name in latest_commit: + return ReleaseType.MINOR + if ReleaseType.PATCH.name in latest_commit: + return ReleaseType.PATCH + if ReleaseType.SNAPSHOT.name in latest_commit: + return ReleaseType.SNAPSHOT + return None def __get_release_type_environment(self) -> ReleaseType | None: release_name = self.environment_api.get( EnvironmentKeys.DDADEVOPS_RELEASE_TYPE.name - ) + ).upper() if release_name is None: raise ValueError( "Release Name not found. Is the Environment correctly configured?" ) - match release_name.upper(): - case ReleaseType.MAJOR.name: - return ReleaseType.MAJOR - case ReleaseType.MINOR.name: - return ReleaseType.MINOR - case ReleaseType.PATCH.name: - return ReleaseType.PATCH - case ReleaseType.SNAPSHOT.name: - return ReleaseType.SNAPSHOT - case _: - return None + if ReleaseType.MAJOR.name in release_name: + return ReleaseType.MAJOR + if ReleaseType.MINOR.name in release_name: + return ReleaseType.MINOR + if ReleaseType.PATCH.name in release_name: + return ReleaseType.PATCH + if ReleaseType.SNAPSHOT.name in release_name: + return ReleaseType.SNAPSHOT + return None def get_release_type(self) -> ReleaseType | None: if self.get_from_git: From 6171d15241486e92ea8e81bc9df72c44acc7189e Mon Sep 17 00:00:00 2001 From: erik Date: Tue, 2 May 2023 13:31:14 +0200 Subject: [PATCH 055/173] [Skip-CI] Remove logging from domain --- src/main/python/ddadevops/domain/common.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index fd15e27..076bfd3 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,5 +1,4 @@ from typing import List -import logging import deprecation def filter_none(list_to_filter): @@ -45,7 +44,6 @@ class Devops(Validateable): self.stage = stage self.name = name self.project_root_path = project_root_path - logging.warning(f"Set project root in DevOps {self.project_root_path}") self.module = module if not name: self.name = module @@ -61,7 +59,6 @@ class Devops(Validateable): def build_path(self): path = [self.project_root_path, self.build_dir_name, self.name, self.module] - logging.warning(f"Set project build_path in Devops {path}") return "/".join(filter_none(path)) def __put__(self, key, value): From 1d6bffcc975bb0c6d004ecae9f461ee61957abcd Mon Sep 17 00:00:00 2001 From: Clemens Date: Fri, 5 May 2023 08:54:21 +0200 Subject: [PATCH 056/173] Remove invalid pylint option --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1abb7b6..5c1518f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0719,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ pytest: stage: lint&test From 1e9f8ff077b07e72d019bd6ddfba4ea03e26bbff Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 5 May 2023 11:57:36 +0200 Subject: [PATCH 057/173] merge release & ReleaseContext --- doc/architecture/Domain.md | 22 ++++++-- src/main/python/ddadevops/__init__.py | 2 +- src/main/python/ddadevops/domain/__init__.py | 2 +- .../python/ddadevops/domain/devops_factory.py | 5 +- src/main/python/ddadevops/domain/release.py | 56 ++++--------------- .../infrastructure/release_mixin/repo.py | 17 +++--- src/test/python/domain/test_common.py | 1 - src/test/python/domain/test_release.py | 3 +- .../release_mixin/mock_infrastructure.py | 6 +- src/test/python/test_release_mixin.py | 2 +- 10 files changed, 45 insertions(+), 71 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index ce9f80b..484433f 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -35,19 +35,29 @@ classDiagram class Release { release_main_branch - release_config_file - } - class ReleaseContext { - release_type - release_version release_current_branch + release_type + } + + class BuildFile { + filename + file_type + content + } + + class Version { + get_version_string(release_type: ReleaseType) + create_release_version(release_type: ReleaseType): + create_bump_version(release_type: ReleaseType): } Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" Release: mixins + Release *-- "0..1" BuildFile: primary_release_file + Release *-- "0..n" BuildFile: secondary_release_files + BuildFile *-- "1" Version C4k *-- DnsRecord - Release *-- "0..1" ReleaseContext ``` diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index 30ae280..37bb5b1 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -20,6 +20,6 @@ from .devops_build import DevopsBuild, create_devops_build_config, get_devops_bu from .credential import gopass_password_from_path, gopass_field_from_path from .release_mixin import ReleaseMixin -from .domain import Validateable, DnsRecord, Devops, Image, Release, ReleaseContext +from .domain import Validateable, DnsRecord, Devops, Image, Release __version__ = "${version}" diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 8121e0d..93fe834 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -2,4 +2,4 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k -from .release import Release, ReleaseContext, ReleaseType, Version, EnvironmentKeys +from .release import Release, ReleaseType, Version, EnvironmentKeys diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index caac032..7e2a3bb 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -4,7 +4,7 @@ from typing import List from .common import Devops, BuildType, MixinType from .image import Image from .c4k import C4k -from .release import Release, ReleaseContext +from .release import Release class DevopsFactory: @@ -23,8 +23,7 @@ class DevopsFactory: mixins = {} if MixinType.RELEASE in mixin_types: - release_context = ReleaseContext(input) - mixins[MixinType.RELEASE] = Release(input, release_context=release_context) + mixins[MixinType.RELEASE] = Release(input) devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index b10a085..b87cc62 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -8,12 +8,10 @@ from .common import ( class ReleaseType(Enum): - MAJOR = 0 - MINOR = 1 - PATCH = 2 - SNAPSHOT = 3 - BUMP = None - NONE = 15 + MAJOR = 3 + MINOR = 2 + PATCH = 1 + NONE = None class EnvironmentKeys(Enum): @@ -66,14 +64,13 @@ class Version(Validateable): return bump_version -class ReleaseContext(Validateable): - def __init__( - self, - input: dict, - ): - self.release_type = ReleaseType[input.get("release_type", "SNAPSHOT")] - self.release_current_version = input.get("release_current_version") +class Release(Validateable): + def __init__(self, input: dict): + self.release_type = ReleaseType[input.get("release_type", "NONE")] + self.release_main_branch = input.get("release_main_branch", "main") self.release_current_branch = input.get("release_current_branch") + self.release_current_version = input.get("release_current_version") + self.release_config_file = input.get("release_config_file", "project.clj") self.version = self.__version_from_str__() # TODO: mv version parsing to version @@ -96,12 +93,10 @@ class ReleaseContext(Validateable): result += self.__validate_is_not_empty__("release_type") result += self.__validate_is_not_empty__("release_current_version") result += self.__validate_is_not_empty__("release_current_branch") + result += self.__validate_is_not_empty__("release_main_branch") + result += self.__validate_is_not_empty__("release_config_file") if self.version: result += self.version.validate() - return result - - def validate_branch(self, main_branch: str): - result = [] if ( self.release_type is not None and self.release_type != ReleaseType.NONE @@ -109,30 +104,3 @@ class ReleaseContext(Validateable): ): result.append(f"Releases are allowed only on {main_branch}") return result - - -class Release(Validateable): - def __init__( - self, - input: dict, - release_context: ReleaseContext, - ): - self.release_main_branch = input.get("release_main_branch", "main") - self.release_config_file = input.get("release_config_file", "project.clj") - self.release_context = release_context - - def release_version(self): - return self.release_context.release_version() - - def bump_version(self): - return self.release_context.bump_version() - - def validate(self): - result = [] - result += self.__validate_is_not_empty__("release_main_branch") - result += self.__validate_is_not_empty__("release_config_file") - result += self.__validate_is_not_empty__("release_context") - if self.release_context is not None: - result += self.release_context.validate() - result += self.release_context.validate_branch(self.release_main_branch) - return result diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 588118a..2557042 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -1,6 +1,5 @@ from src.main.python.ddadevops.domain import ( - ReleaseContext, Version, ReleaseType, EnvironmentKeys, @@ -116,11 +115,11 @@ class ReleaseContextRepository: self.version_repository = version_repository self.release_type_repository = release_type_repository - def get_release(self, main_branch: str) -> ReleaseContext: - result = ReleaseContext( - self.release_type_repository.get_release_type(), - self.version_repository.get_version(), - main_branch, - ) - result.throw_if_invalid() - return result + # def get_release(self, main_branch: str) -> ReleaseContext: + # result = ReleaseContext( + # self.release_type_repository.get_release_type(), + # self.version_repository.get_version(), + # main_branch, + # ) + # result.throw_if_invalid() + # return result diff --git a/src/test/python/domain/test_common.py b/src/test/python/domain/test_common.py index 9b4e21f..ef4769f 100644 --- a/src/test/python/domain/test_common.py +++ b/src/test/python/domain/test_common.py @@ -10,7 +10,6 @@ from src.main.python.ddadevops.domain import ( Version, ReleaseType, Release, - ReleaseContext, ) from src.main.python.ddadevops.domain.image import Image from .test_helper import build_devops diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index b81d29a..673ada0 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -11,7 +11,6 @@ from src.main.python.ddadevops.domain import ( Version, ReleaseType, Release, - ReleaseContext, ) from src.main.python.ddadevops.domain.image import Image from .test_helper import build_devops, devops_config @@ -50,7 +49,7 @@ def test_version(tmp_path: Path): def test_release_context(tmp_path): - sut = ReleaseContext( + sut = Release( devops_config( { "release_type": "MINOR", diff --git a/src/test/python/release_mixin/mock_infrastructure.py b/src/test/python/release_mixin/mock_infrastructure.py index 81b09b5..74bd1a5 100644 --- a/src/test/python/release_mixin/mock_infrastructure.py +++ b/src/test/python/release_mixin/mock_infrastructure.py @@ -1,6 +1,6 @@ from pathlib import Path -from src.main.python.ddadevops.domain import ReleaseType, Version, ReleaseContext +from src.main.python.ddadevops.domain import ReleaseType, Version from .mock_infrastructure_api import MockGitApi @@ -38,6 +38,6 @@ class MockReleaseRepository(): self.main_branch = main_branch self.get_release_count = 0 - def get_release(self) -> ReleaseContext: + def get_release(self) -> : self.get_release_count += 1 - return ReleaseContext(self.release_type_repository.get_release_type(), self.version_repository.get_version(), self.main_branch) + return Release(self.release_type_repository.get_release_type(), self.version_repository.get_version(), self.main_branch) diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py index 2e13ea6..ed5c417 100644 --- a/src/test/python/test_release_mixin.py +++ b/src/test/python/test_release_mixin.py @@ -7,7 +7,7 @@ from src.main.python.ddadevops.infrastructure.release_mixin import ( GitApi, EnvironmentApi, ) -from src.main.python.ddadevops.domain import Devops, ReleaseContext, Release +from src.main.python.ddadevops.domain import Devops, Release from .resource_helper import ResourceHelper from .domain.test_helper import devops_config From f569d07a841d43d8d2a1f8a2c221a5136bf788b8 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 9 May 2023 09:19:26 +0200 Subject: [PATCH 058/173] introduce version parsed from string --- src/main/python/ddadevops/__init__.py | 3 +- src/main/python/ddadevops/domain/__init__.py | 5 +- src/main/python/ddadevops/domain/common.py | 16 +++- src/main/python/ddadevops/domain/release.py | 78 +++---------------- src/main/python/ddadevops/domain/version.py | 64 ++++++++++++++++ src/test/python/domain/test_common.py | 2 +- src/test/python/domain/test_release.py | 33 -------- src/test/python/domain/test_version.py | 79 ++++++++++++++++++++ 8 files changed, 171 insertions(+), 109 deletions(-) create mode 100644 src/main/python/ddadevops/domain/version.py create mode 100644 src/test/python/domain/test_version.py diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index 37bb5b1..201a228 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -19,7 +19,6 @@ from .devops_terraform_build import DevopsTerraformBuild, create_devops_terrafor from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build from .credential import gopass_password_from_path, gopass_field_from_path from .release_mixin import ReleaseMixin - -from .domain import Validateable, DnsRecord, Devops, Image, Release +from .domain import Validateable, DnsRecord, Devops, Image, Release, Version __version__ = "${version}" diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 93fe834..739acdf 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,5 +1,6 @@ -from .common import Validateable, DnsRecord, Devops, BuildType, MixinType +from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, ReleaseType from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k -from .release import Release, ReleaseType, Version, EnvironmentKeys +from .release import Release, EnvironmentKeys +from .version import Version diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 0a599cf..ee8c3cd 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -18,6 +18,13 @@ class MixinType(Enum): RELEASE = 0 +class ReleaseType(Enum): + MAJOR = 3 + MINOR = 2 + PATCH = 1 + NONE = None + + class Validateable: def __validate_is_not_none__(self, field_name: str) -> List[str]: value = self.__dict__[field_name] @@ -26,10 +33,13 @@ class Validateable: return [] def __validate_is_not_empty__(self, field_name: str) -> List[str]: + result = self.__validate_is_not_none__(field_name) value = self.__dict__[field_name] - if value is None or value == "": - return [f"Field '{field_name}' must not be empty."] - return [] + if type(value) is str and value == "": + result += [f"Field '{field_name}' must not be empty."] + elif type(value) is list and len(value) == 0: + result += [f"Field '{field_name}' must not be empty."] + return result def validate(self) -> List[str]: return [] diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index b87cc62..0051452 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -4,83 +4,25 @@ from pathlib import Path from .common import ( Validateable, Devops, + ReleaseType, +) +from .version import ( + Version, ) - - -class ReleaseType(Enum): - MAJOR = 3 - MINOR = 2 - PATCH = 1 - NONE = None class EnvironmentKeys(Enum): DDADEVOPS_RELEASE_TYPE = 0 -class Version(Validateable): - def __init__(self, path: Path, version_list: list): - self.path = path - self.version_list = version_list - self.version_string: Optional[str | None] = None - self.is_snapshot: Optional[bool | None] = None - - def increment(self, release_type: ReleaseType | None): - self.is_snapshot = False - match release_type: - case ReleaseType.BUMP: - self.is_snapshot = True - self.version_list[ReleaseType.PATCH.value] += 1 - case ReleaseType.SNAPSHOT: - self.is_snapshot = True - case ReleaseType.PATCH: - self.version_list[ReleaseType.PATCH.value] += 1 - case ReleaseType.MINOR: - self.version_list[ReleaseType.PATCH.value] = 0 - self.version_list[ReleaseType.MINOR.value] += 1 - case ReleaseType.MAJOR: - self.version_list[ReleaseType.PATCH.value] = 0 - self.version_list[ReleaseType.MINOR.value] = 0 - self.version_list[ReleaseType.MAJOR.value] += 1 - case None: - raise RuntimeError("Release Type was not set!") - - def get_version_string(self) -> str: - self.version_string = ".".join([str(x) for x in self.version_list]) - if self.is_snapshot: - self.version_string += "-SNAPSHOT" - return self.version_string - - def create_release_version(self, release_type: ReleaseType | None): - release_version = Version(self.path, self.version_list.copy()) - release_version.is_snapshot = self.is_snapshot - release_version.increment(release_type) - return release_version - - def create_bump_version(self): - bump_version = Version(self.path, self.version_list.copy()) - bump_version.is_snapshot = self.is_snapshot - bump_version.increment(ReleaseType.BUMP) - return bump_version - - class Release(Validateable): def __init__(self, input: dict): self.release_type = ReleaseType[input.get("release_type", "NONE")] self.release_main_branch = input.get("release_main_branch", "main") self.release_current_branch = input.get("release_current_branch") - self.release_current_version = input.get("release_current_version") self.release_config_file = input.get("release_config_file", "project.clj") - self.version = self.__version_from_str__() - - # TODO: mv version parsing to version - def __version_from_str__(self): - if not self.release_current_version: - return - version_parsed = [] - for x in self.release_current_version.split("."): - version_parsed += [int(x)] - return Version("unused", version_parsed) + self.release_current_version = Version.from_str(input.get("release_current_version")) + def release_version(self) -> Version: return self.version.create_release_version(self.release_type) @@ -91,16 +33,16 @@ class Release(Validateable): def validate(self): result = [] result += self.__validate_is_not_empty__("release_type") - result += self.__validate_is_not_empty__("release_current_version") result += self.__validate_is_not_empty__("release_current_branch") result += self.__validate_is_not_empty__("release_main_branch") result += self.__validate_is_not_empty__("release_config_file") - if self.version: - result += self.version.validate() + result += self.__validate_is_not_empty__("release_current_version") + if self.release_current_version: + result += self.release_current_version.validate() if ( self.release_type is not None and self.release_type != ReleaseType.NONE - and main_branch != self.release_current_branch + and self.release_main_branch != self.release_current_branch ): result.append(f"Releases are allowed only on {main_branch}") return result diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py new file mode 100644 index 0000000..adb67b7 --- /dev/null +++ b/src/main/python/ddadevops/domain/version.py @@ -0,0 +1,64 @@ +from enum import Enum +from typing import Optional +from .common import ( + Validateable, +) + + +class Version(Validateable): + @classmethod + def from_str(cls, input_str: str): + snapshot_parsed = input_str.split("-") + version_str = snapshot_parsed[0] + suffix_str = None + if len(snapshot_parsed) > 1: + suffix_str = snapshot_parsed[1] + version_no_parsed = [] + for x in version_str.split("."): + version_no_parsed += [int(x)] + return cls( + version_no_parsed, + suffix_str, + input_str, + ) + + def __init__( + self, + version_list: list, + snapshot_suffix: Optional[str] = None, + version_str: Optional[str] = None, + ): + self.version_list = version_list + self.snapshot_suffix = snapshot_suffix + self.version_string = version_str + + def is_snapshot(self): + return not self.snapshot_suffix == None + + def to_string(self) -> str: + version_no = ".".join([str(x) for x in self.version_list]) + if self.is_snapshot(): + return f"{version_no}-{self.snapshot_suffix}" + return version_no + + def validate(self): + result = [] + result += self.__validate_is_not_empty__("version_list") + if self.version_list and len(self.version_list) < 3: + result += [f"version_list must have at least 3 levels."] + if self.version_list and self.version_string and self.to_string() != self.version_string: + result += [f"version_string not parsed correct. Input was {self.version_string} parsed was {self.to_string()}"] + + return result + + # def create_release_version(self, release_type: ReleaseType | None): + # release_version = Version(self.path, self.version_list.copy()) + # release_version.is_snapshot = self.is_snapshot + # release_version.increment(release_type) + # return release_version + + # def create_bump_version(self): + # bump_version = Version(self.path, self.version_list.copy()) + # bump_version.is_snapshot = self.is_snapshot + # bump_version.increment(ReleaseType.BUMP) + # return bump_version diff --git a/src/test/python/domain/test_common.py b/src/test/python/domain/test_common.py index ef4769f..5c9bc8a 100644 --- a/src/test/python/domain/test_common.py +++ b/src/test/python/domain/test_common.py @@ -50,7 +50,7 @@ def test_should_validate_non_empty_others(): def test_validate_with_reason(): sut = MockValidateable(None) - assert sut.validate()[0] == "Field 'field' must not be empty." + assert sut.validate()[0] == "Field 'field' must not be None." def test_should_validate_DnsRecord(): diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index 673ada0..af2f3d7 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -15,39 +15,6 @@ from src.main.python.ddadevops.domain import ( from src.main.python.ddadevops.domain.image import Image from .test_helper import build_devops, devops_config - -def test_version(tmp_path: Path): - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.SNAPSHOT) - assert version.get_version_string() == "1.2.3-SNAPSHOT" - assert version.version_list == [1, 2, 3] - assert version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.BUMP) - assert version.get_version_string() == "1.2.4-SNAPSHOT" - assert version.version_list == [1, 2, 4] - assert version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.PATCH) - assert version.get_version_string() == "1.2.4" - assert version.version_list == [1, 2, 4] - assert not version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.MINOR) - assert version.get_version_string() == "1.3.0" - assert version.version_list == [1, 3, 0] - assert not version.is_snapshot - - version = Version(tmp_path, [1, 2, 3]) - version.increment(ReleaseType.MAJOR) - assert version.get_version_string() == "2.0.0" - assert version.version_list == [2, 0, 0] - assert not version.is_snapshot - - def test_release_context(tmp_path): sut = Release( devops_config( diff --git a/src/test/python/domain/test_version.py b/src/test/python/domain/test_version.py new file mode 100644 index 0000000..3e2be00 --- /dev/null +++ b/src/test/python/domain/test_version.py @@ -0,0 +1,79 @@ +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain import ( + Version, + ReleaseType, +) +from src.main.python.ddadevops.domain.image import Image +from .test_helper import build_devops, devops_config + + +def test_version_creation(): + sut = Version.from_str("1.2.3") + assert sut.to_string() == "1.2.3" + assert sut.version_list == [1, 2, 3] + assert sut.is_snapshot() == False + + sut = Version.from_str("1.2.3-SNAPSHOT") + assert sut.to_string() == "1.2.3-SNAPSHOT" + assert sut.version_list == [1, 2, 3] + assert sut.is_snapshot() == True + + +def test_should_validate_version_list(): + sut = Version(None) + assert not sut.is_valid() + + sut = Version([]) + assert not sut.is_valid() + + sut = Version([1, 2]) + assert not sut.is_valid() + + sut = Version([1, 2, 3]) + assert sut.is_valid() + + +def test_should_validate_parsing(): + sut = Version.from_str("1.2") + assert not sut.is_valid() + + sut = Version.from_str("1.2.3") + sut.version_list = [2, 2, 2] + assert not sut.is_valid() + + sut = Version.from_str("1.2.3") + assert sut.is_valid() + + sut = Version.from_str("1.2.3-SNAPSHOT") + assert sut.is_valid() + + sut = Version.from_str("1.2.3-dev") + assert sut.is_valid() + + +def test_version(): + pass + # version = Version.version_from_str("1.2.3-SNAPSHOT") + # version.increment(ReleaseType.NONE) + # assert version.get_version_string() == "1.2.3-SNAPSHOT" + # assert version.version_list == [1, 2, 3] + # assert version.is_snapshot + + # version = Version.version_from_str("1.2.3") + # version.increment(ReleaseType.PATCH) + # assert version.get_version_string() == "1.2.4" + # assert version.version_list == [1, 2, 4] + # assert not version.is_snapshot + + # version = Version.version_from_str("1.2.3") + # version.increment(ReleaseType.MINOR) + # assert version.get_version_string() == "1.3.0" + # assert version.version_list == [1, 3, 0] + # assert not version.is_snapshot + + # version = Version.version_from_str("1.2.3") + # version.increment(ReleaseType.MAJOR) + # assert version.get_version_string() == "2.0.0" + # assert version.version_list == [2, 0, 0] + # assert not version.is_snapshot From 9ae6679bfd218e4b5f05f12fbdd8bf6b40f5c096 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 10 May 2023 08:53:05 +0200 Subject: [PATCH 059/173] realize repository --- .../infrastructure/infrastructure.py | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 300180f..dbbd98f 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -8,34 +8,21 @@ import deprecation from ..domain import Devops, Image, C4k, Release from ..python_util import execute +class DevopsRepository: + def get_devops(self, project) -> Devops: + devops = project.get_property("devops") + devops.throw_if_invalid() + return devops + + def set_devops(self, project, devops: Devops): + devops.throw_if_invalid() + project.set_property("devops", devops) + class ProjectRepository: def set_build(self, project, build): project.set_property("devops_build", build) - def get_devops(self, project) -> Devops: - return project.get_property("build") - - def set_devops(self, project, build: Devops): - project.set_property("build", build) - - @deprecation.deprecated(deprecated_in="3.2") - def get_c4k(self, project) -> C4k: - return project.get_property("c4k_build") - - @deprecation.deprecated(deprecated_in="3.2") - def set_c4k(self, project, build: C4k): - project.set_property("c4k_build", build) - - @deprecation.deprecated(deprecated_in="3.2") - def get_release(self, project) -> Release: - return project.get_property("release_build") - - @deprecation.deprecated(deprecated_in="3.2") - def set_release(self, project, build: Release): - project.set_property("release_build", build) - - class ResourceApi: def read_resource(self, path: str) -> bytes: From d38c2baaa11d4f7ce0b2c63bb15d7078488fb527 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 10 May 2023 09:28:43 +0200 Subject: [PATCH 060/173] add some version funct. --- doc/architecture/Domain.md | 16 ++++--- src/main/python/ddadevops/domain/version.py | 26 +++++++----- src/test/python/domain/test_version.py | 46 +++++++++++---------- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 484433f..bce25f8 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -37,25 +37,29 @@ classDiagram release_main_branch release_current_branch release_type + version } class BuildFile { - filename + <> + file_path [id] file_type content } class Version { - get_version_string(release_type: ReleaseType) - create_release_version(release_type: ReleaseType): - create_bump_version(release_type: ReleaseType): + get_version_string() + create_major() + create_minor() + create_patchn() + create_bump() } Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" Release: mixins - Release *-- "0..1" BuildFile: primary_release_file - Release *-- "0..n" BuildFile: secondary_release_files + Release o-- "0..1" BuildFile: primary_release_file + Release o-- "0..n" BuildFile: secondary_release_files BuildFile *-- "1" Version C4k *-- DnsRecord diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index adb67b7..94b07be 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -48,17 +48,21 @@ class Version(Validateable): result += [f"version_list must have at least 3 levels."] if self.version_list and self.version_string and self.to_string() != self.version_string: result += [f"version_string not parsed correct. Input was {self.version_string} parsed was {self.to_string()}"] - return result - # def create_release_version(self, release_type: ReleaseType | None): - # release_version = Version(self.path, self.version_list.copy()) - # release_version.is_snapshot = self.is_snapshot - # release_version.increment(release_type) - # return release_version + def create_patch(self): + new_version_list = self.version_list.copy() + if self.is_snapshot(): + return Version(new_version_list, snapshot_suffix=None, version_str=None) + else: + new_version_list[2] += 1 + return Version(new_version_list, snapshot_suffix=None, version_str=None) - # def create_bump_version(self): - # bump_version = Version(self.path, self.version_list.copy()) - # bump_version.is_snapshot = self.is_snapshot - # bump_version.increment(ReleaseType.BUMP) - # return bump_version + def create_minor(self): + new_version_list = self.version_list.copy() + if self.is_snapshot() and new_version_list[2] == 0: + return Version(new_version_list, snapshot_suffix=None, version_str=None) + else: + new_version_list[2] = 0 + new_version_list[1] += 1 + return Version(new_version_list, snapshot_suffix=None, version_str=None) diff --git a/src/test/python/domain/test_version.py b/src/test/python/domain/test_version.py index 3e2be00..c13e277 100644 --- a/src/test/python/domain/test_version.py +++ b/src/test/python/domain/test_version.py @@ -52,28 +52,30 @@ def test_should_validate_parsing(): assert sut.is_valid() -def test_version(): - pass - # version = Version.version_from_str("1.2.3-SNAPSHOT") - # version.increment(ReleaseType.NONE) - # assert version.get_version_string() == "1.2.3-SNAPSHOT" - # assert version.version_list == [1, 2, 3] - # assert version.is_snapshot +def test_should_create_patch(): + version = Version.from_str("1.2.3-SNAPSHOT") + sut = version.create_patch() + assert sut.to_string() == "1.2.3" + assert version.to_string() == "1.2.3-SNAPSHOT" - # version = Version.version_from_str("1.2.3") - # version.increment(ReleaseType.PATCH) - # assert version.get_version_string() == "1.2.4" - # assert version.version_list == [1, 2, 4] - # assert not version.is_snapshot + version = Version.from_str("1.2.3") + sut = version.create_patch() + assert sut.to_string() == "1.2.4" + assert version.to_string() == "1.2.3" - # version = Version.version_from_str("1.2.3") - # version.increment(ReleaseType.MINOR) - # assert version.get_version_string() == "1.3.0" - # assert version.version_list == [1, 3, 0] - # assert not version.is_snapshot +def test_should_create_minor(): + version = Version.from_str("1.2.3-SNAPSHOT") + sut = version.create_minor() + assert sut.to_string() == "1.3.0" - # version = Version.version_from_str("1.2.3") - # version.increment(ReleaseType.MAJOR) - # assert version.get_version_string() == "2.0.0" - # assert version.version_list == [2, 0, 0] - # assert not version.is_snapshot + version = Version.from_str("1.2.3") + sut = version.create_minor() + assert sut.to_string() == "1.3.0" + + version = Version.from_str("1.3.0-SNAPSHOT") + sut = version.create_minor() + assert sut.to_string() == "1.3.0" + + version = Version.from_str("1.3.0") + sut = version.create_minor() + assert sut.to_string() == "1.4.0" \ No newline at end of file From 2e2c04c8ba163b355c5d1c7fd8b0f80e79cf994b Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 10 May 2023 19:37:15 +0200 Subject: [PATCH 061/173] finish version --- doc/architecture/Domain.md | 4 +-- src/main/python/ddadevops/domain/version.py | 19 ++++++++++++ src/test/python/domain/test_version.py | 33 ++++++++++++++++++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index bce25f8..3c9014b 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -51,8 +51,8 @@ classDiagram get_version_string() create_major() create_minor() - create_patchn() - create_bump() + create_patch() + create_bump(snapshot_suffix) } Devops *-- "0..1" Image: spcialized_builds diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 94b07be..1b01b29 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -50,6 +50,15 @@ class Version(Validateable): result += [f"version_string not parsed correct. Input was {self.version_string} parsed was {self.to_string()}"] return result + def create_bump(self, snapshot_suffix: str = None): + new_version_list = self.version_list.copy() + if self.is_snapshot(): + return Version(new_version_list, snapshot_suffix=self.snapshot_suffix, version_str=None) + else: + new_version_list[2] = 0 + new_version_list[1] += 1 + return Version(new_version_list, snapshot_suffix=snapshot_suffix, version_str=None) + def create_patch(self): new_version_list = self.version_list.copy() if self.is_snapshot(): @@ -66,3 +75,13 @@ class Version(Validateable): new_version_list[2] = 0 new_version_list[1] += 1 return Version(new_version_list, snapshot_suffix=None, version_str=None) + + def create_major(self): + new_version_list = self.version_list.copy() + if self.is_snapshot() and new_version_list[2] == 0 and new_version_list[1] == 0 : + return Version(new_version_list, snapshot_suffix=None, version_str=None) + else: + new_version_list[2] = 0 + new_version_list[1] = 0 + new_version_list[0] += 1 + return Version(new_version_list, snapshot_suffix=None, version_str=None) diff --git a/src/test/python/domain/test_version.py b/src/test/python/domain/test_version.py index c13e277..2caee0d 100644 --- a/src/test/python/domain/test_version.py +++ b/src/test/python/domain/test_version.py @@ -78,4 +78,35 @@ def test_should_create_minor(): version = Version.from_str("1.3.0") sut = version.create_minor() - assert sut.to_string() == "1.4.0" \ No newline at end of file + assert sut.to_string() == "1.4.0" + + +def test_should_create_major(): + version = Version.from_str("1.2.3-SNAPSHOT") + sut = version.create_major() + assert sut.to_string() == "2.0.0" + + version = Version.from_str("1.2.3") + sut = version.create_major() + assert sut.to_string() == "2.0.0" + + version = Version.from_str("1.0.0-SNAPSHOT") + sut = version.create_major() + assert sut.to_string() == "1.0.0" + + version = Version.from_str("1.0.0") + sut = version.create_major() + assert sut.to_string() == "2.0.0" + +def test_should_create_bump(): + version = Version.from_str("1.2.3-SNAPSHOT") + sut = version.create_bump() + assert sut.to_string() == "1.2.3-SNAPSHOT" + + version = Version.from_str("1.2.3") + sut = version.create_bump("SNAPSHOT") + assert sut.to_string() == "1.3.0-SNAPSHOT" + + version = Version.from_str("1.0.0") + sut = version.create_bump("SNAPSHOT") + assert sut.to_string() == "1.1.0-SNAPSHOT" \ No newline at end of file From 8359406330088ea0f563a1174e5c40daa72cdb0e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 10 May 2023 19:52:53 +0200 Subject: [PATCH 062/173] fix release --- doc/architecture/Domain.md | 2 +- .../python/ddadevops/domain/devops_factory.py | 4 ++- src/main/python/ddadevops/domain/release.py | 23 +++++--------- src/test/python/domain/test_devops_factory.py | 6 ++-- src/test/python/domain/test_helper.py | 6 ++-- src/test/python/domain/test_release.py | 31 +++++++++---------- 6 files changed, 33 insertions(+), 39 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 3c9014b..e99055d 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -34,9 +34,9 @@ classDiagram } class Release { + release_type release_main_branch release_current_branch - release_type version } diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 7e2a3bb..d60bd1b 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -5,6 +5,7 @@ from .common import Devops, BuildType, MixinType from .image import Image from .c4k import C4k from .release import Release +from .version import Version class DevopsFactory: @@ -23,7 +24,8 @@ class DevopsFactory: mixins = {} if MixinType.RELEASE in mixin_types: - mixins[MixinType.RELEASE] = Release(input) + version = Version.from_str(input["version"]) + mixins[MixinType.RELEASE] = Release(input, version) devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 0051452..7aaf162 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -16,33 +16,26 @@ class EnvironmentKeys(Enum): class Release(Validateable): - def __init__(self, input: dict): + def __init__(self, input: dict, version: Version): self.release_type = ReleaseType[input.get("release_type", "NONE")] self.release_main_branch = input.get("release_main_branch", "main") self.release_current_branch = input.get("release_current_branch") - self.release_config_file = input.get("release_config_file", "project.clj") - self.release_current_version = Version.from_str(input.get("release_current_version")) + self.release_config_file = input.get("release_config_file", "./project.clj") + self.version = version - - def release_version(self) -> Version: - return self.version.create_release_version(self.release_type) - - def bump_version(self) -> Version: - return self.release_version().create_bump_version() - def validate(self): result = [] result += self.__validate_is_not_empty__("release_type") - result += self.__validate_is_not_empty__("release_current_branch") result += self.__validate_is_not_empty__("release_main_branch") + result += self.__validate_is_not_empty__("release_current_branch") result += self.__validate_is_not_empty__("release_config_file") - result += self.__validate_is_not_empty__("release_current_version") - if self.release_current_version: - result += self.release_current_version.validate() + result += self.__validate_is_not_empty__("version") + if self.version: + result += self.version.validate() if ( self.release_type is not None and self.release_type != ReleaseType.NONE and self.release_main_branch != self.release_current_branch ): - result.append(f"Releases are allowed only on {main_branch}") + result.append(f"Releases are allowed only on {self.release_main_branch}") return result diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index ab4e414..e1da188 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -52,11 +52,11 @@ def test_devops_factory(): "project_root_path": "../../..", "build_types": [], "mixin_types": ["RELEASE"], - "release_main_branch": "main", - "release_config_file": "project.clj", + "version": "1.0.0", "release_type": "NONE", - "release_current_version": "1.0.0", + "release_main_branch": "main", "release_current_branch": "my_feature", + "release_config_file": "project.clj", } ) assert sut is not None diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index 391f729..381c489 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -18,11 +18,11 @@ def devops_config(overrides: dict) -> dict: "c4k_grafana_cloud_password": "password", "c4k_grafana_cloud_url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", "c4k_auth": {}, - "release_main_branch": "main", - "release_config_file": "project.clj", + "version": "1.0.0", "release_type": "NONE", - "release_current_version": "1.0.0", + "release_main_branch": "main", "release_current_branch": "my_feature", + "release_config_file": "./project.clj", } input = default.copy() input.update(overrides) diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index af2f3d7..6c80915 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -15,27 +15,26 @@ from src.main.python.ddadevops.domain import ( from src.main.python.ddadevops.domain.image import Image from .test_helper import build_devops, devops_config -def test_release_context(tmp_path): + +def test_sould_validate_release(): sut = Release( devops_config( { "release_type": "MINOR", - "release_current_version": "1.2.3", "release_current_branch": "main", } - ) + ), + Version.from_str("1.3.1-SNAPSHOT"), ) - assert sut.release_version().get_version_string() == "1.3.0" - assert sut.bump_version().get_version_string() == "1.3.1-SNAPSHOT" - - -def test_release(tmp_path): - sut = build_devops( - { - "release_type": "MINOR", - "release_current_version": "1.2.3", - "release_current_branch": "main", - } - ) - assert sut.mixins[MixinType.RELEASE] assert sut.is_valid() + + sut = Release( + devops_config( + { + "release_type": "MINOR", + "release_current_branch": "some-feature-branch", + } + ), + Version.from_str("1.3.1-SNAPSHOT"), + ) + assert not sut.is_valid() From ac7f5b8dd1b06cded85b98c521fc2e64e6b25bfc Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 11 May 2023 09:20:59 +0200 Subject: [PATCH 063/173] First steps for BuildFile --- doc/architecture/Domain.md | 4 +- src/main/python/ddadevops/domain/__init__.py | 1 + .../python/ddadevops/domain/build_file.py | 47 +++++++++++++++++++ src/main/python/ddadevops/domain/common.py | 11 +++-- .../infrastructure/infrastructure.py | 25 ++++++++-- src/test/python/domain/test_build_file.py | 33 +++++++++++++ 6 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 src/main/python/ddadevops/domain/build_file.py create mode 100644 src/test/python/domain/test_build_file.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index e99055d..f1d3e33 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -43,8 +43,10 @@ classDiagram class BuildFile { <> file_path [id] - file_type content + build_file_type() + getVersion() + setVersion(version) } class Version { diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 739acdf..3662bc8 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -4,3 +4,4 @@ from .image import Image from .c4k import C4k from .release import Release, EnvironmentKeys from .version import Version +from .build_file import BuildFileType, BuildFile diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py new file mode 100644 index 0000000..2e70893 --- /dev/null +++ b/src/main/python/ddadevops/domain/build_file.py @@ -0,0 +1,47 @@ +from enum import Enum +from typing import Optional +from pathlib import Path +from .common import ( + Validateable, + Devops, + ReleaseType, +) +from .version import ( + Version, +) + +class BuildFileType(Enum): + JS = '.json' + JAVA_GRADLE = '.gradle' + JAVA_CLOJURE = ".clj" + PYTHON = '.py' + +class BuildFile(Validateable): + def __init__(self, file_path: Path, content: str): + self.file_path = file_path + self.content = content + + def validate(self): + result = [] + result += self.__validate_is_not_empty__("file_path") + result += self.__validate_is_not_empty__("content") + if not self.build_file_type(): + result += [f"Suffix {self.file_path} is unknown."] + return result + + def build_file_type(self): + if not self.file_path: + return None + config_file_type = self.file_path.suffix + match config_file_type: + case '.json': + result = BuildFileType.JS + case '.gradle': + result = BuildFileType.JAVA_GRADLE + case '.clj': + result = BuildFileType.JAVA_CLOJURE + case '.py': + result = BuildFileType.PYTHON + case _: + result = None + return result diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index ee8c3cd..ced80c6 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -34,11 +34,12 @@ class Validateable: def __validate_is_not_empty__(self, field_name: str) -> List[str]: result = self.__validate_is_not_none__(field_name) - value = self.__dict__[field_name] - if type(value) is str and value == "": - result += [f"Field '{field_name}' must not be empty."] - elif type(value) is list and len(value) == 0: - result += [f"Field '{field_name}' must not be empty."] + if len(result) == 0: + value = self.__dict__[field_name] + if type(value) is str and value == "": + result += [f"Field '{field_name}' must not be empty."] + elif type(value) is list and len(value) == 0: + result += [f"Field '{field_name}' must not be empty."] return result def validate(self) -> List[str]: diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index dbbd98f..c165ae9 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -5,9 +5,10 @@ from subprocess import run from pkg_resources import resource_string import yaml import deprecation -from ..domain import Devops, Image, C4k, Release +from ..domain import Devops, Image, C4k, Release, BuildFile from ..python_util import execute + class DevopsRepository: def get_devops(self, project) -> Devops: devops = project.get_property("devops") @@ -19,10 +20,24 @@ class DevopsRepository: project.set_property("devops", devops) -class ProjectRepository: - def set_build(self, project, build): - project.set_property("devops_build", build) +class BuildFileRepository: + def get(self, path: Path) -> BuildFile: + with open(path, "r", encoding="utf-8") as file: + content = file.read() + result = BuildFile(path, content) + result.throw_if_invalid() + return result + def write(self, build_file: BuildFile): + build_file.throw_if_invalid() + with open(build_file.file_path, "r+", encoding="utf-8") as file: + file.seek(0) + file.write(build_file.content) + + + +class ProjectRepository: + pass class ResourceApi: def read_resource(self, path: str) -> bytes: @@ -60,7 +75,7 @@ class ImageApi: def drun(self, name: str): run( - f"docker run -it --entrypoint=\"\" {name} /bin/bash", + f'docker run -it --entrypoint="" {name} /bin/bash', shell=True, check=True, ) diff --git a/src/test/python/domain/test_build_file.py b/src/test/python/domain/test_build_file.py new file mode 100644 index 0000000..c505438 --- /dev/null +++ b/src/test/python/domain/test_build_file.py @@ -0,0 +1,33 @@ +from pathlib import Path +from src.main.python.ddadevops.domain import ( + BuildFileType, + BuildFile, +) + + +def test_sould_validate_build_file(): + sut = BuildFile( + Path("./project.clj"), + "content" + ) + assert sut.is_valid() + + sut = BuildFile( + None, + "" + ) + assert not sut.is_valid() + + sut = BuildFile( + Path("./unknown.extension"), + "content" + ) + assert not sut.is_valid() + + +def test_sould_calculate_build_type(): + sut = BuildFile( + Path("./project.clj"), + "content" + ) + assert sut.build_file_type() == BuildFileType.JAVA_CLOJURE From eafa4a4069b4ce243b9d66b04540ff57342c5c9f Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 11 May 2023 10:03:09 +0200 Subject: [PATCH 064/173] first try to implement get_version --- .../python/ddadevops/domain/build_file.py | 5 +++++ src/test/python/domain/test_build_file.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py index 2e70893..285c6d1 100644 --- a/src/main/python/ddadevops/domain/build_file.py +++ b/src/main/python/ddadevops/domain/build_file.py @@ -1,6 +1,7 @@ from enum import Enum from typing import Optional from pathlib import Path +import json from .common import ( Validateable, Devops, @@ -45,3 +46,7 @@ class BuildFile(Validateable): case _: result = None return result + + def get_version(self) -> Version: + version = json.loads(self.content)["version"] + return Version.from_str(version) diff --git a/src/test/python/domain/test_build_file.py b/src/test/python/domain/test_build_file.py index c505438..833f0c6 100644 --- a/src/test/python/domain/test_build_file.py +++ b/src/test/python/domain/test_build_file.py @@ -2,6 +2,7 @@ from pathlib import Path from src.main.python.ddadevops.domain import ( BuildFileType, BuildFile, + Version, ) @@ -31,3 +32,21 @@ def test_sould_calculate_build_type(): "content" ) assert sut.build_file_type() == BuildFileType.JAVA_CLOJURE + +def test_sould_parse_version(): + sut = BuildFile( + Path("./package.js"), + """{ + "name": "c4k-jira", + "description": "Generate c4k yaml for a jira deployment.", + "author": "meissa GmbH", + "version": "1.1.5-SNAPSHOT", + "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-jira#readme", + "bin": { + "c4k-jira": "./c4k-jira.js" + }, +} +""" + ) + assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") + From 599f654de8f37824b3e58b4658635926ce3e4594 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 11 May 2023 15:00:15 +0200 Subject: [PATCH 065/173] Fix syntax error --- src/test/python/release_mixin/mock_infrastructure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/python/release_mixin/mock_infrastructure.py b/src/test/python/release_mixin/mock_infrastructure.py index 74bd1a5..9cf6b06 100644 --- a/src/test/python/release_mixin/mock_infrastructure.py +++ b/src/test/python/release_mixin/mock_infrastructure.py @@ -1,6 +1,6 @@ from pathlib import Path -from src.main.python.ddadevops.domain import ReleaseType, Version +from src.main.python.ddadevops.domain import ReleaseType, Version, Release from .mock_infrastructure_api import MockGitApi @@ -38,6 +38,6 @@ class MockReleaseRepository(): self.main_branch = main_branch self.get_release_count = 0 - def get_release(self) -> : + def get_release(self) -> Release: self.get_release_count += 1 return Release(self.release_type_repository.get_release_type(), self.version_repository.get_version(), self.main_branch) From 9e7bef2066158d789b53d71607510542dc702d85 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 11 May 2023 15:00:37 +0200 Subject: [PATCH 066/173] Fix Typo: executabel -> executable --- src/main/python/ddadevops/c4k_build.py | 4 ++-- src/main/python/ddadevops/domain/c4k.py | 6 +++--- src/test/python/domain/test_c4k.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/python/ddadevops/c4k_build.py b/src/main/python/ddadevops/c4k_build.py index 9cd0f99..2cb7b15 100644 --- a/src/main/python/ddadevops/c4k_build.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -10,7 +10,7 @@ def add_c4k_mixin_config( config, c4k_config_dict, c4k_auth_dict, - executabel_name=None, + executable_name=None, grafana_cloud_user=None, grafana_cloud_password=None, grafana_cloud_url="https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", @@ -35,7 +35,7 @@ def add_c4k_mixin_config( config.update( { "C4kMixin": { - "executabel_name": executabel_name, + "executable_name": executable_name, "config": c4k_config_dict, "auth": c4k_auth_dict, } diff --git a/src/main/python/ddadevops/domain/c4k.py b/src/main/python/ddadevops/domain/c4k.py index bb26686..ca086a6 100644 --- a/src/main/python/ddadevops/domain/c4k.py +++ b/src/main/python/ddadevops/domain/c4k.py @@ -10,7 +10,7 @@ class C4k(Validateable): def __init__(self, input: dict): self.module = input.get("module") self.stage = input.get("stage") - self.c4k_executabel_name = input.get("c4k_executabel_name", input.get("module")) + self.c4k_executable_name = input.get("c4k_executable_name", input.get("module")) self.c4k_config = input.get("c4k_config", {}) self.c4k_grafana_cloud_url = input.get( "c4k_grafana_cloud_url", @@ -29,7 +29,7 @@ class C4k(Validateable): result = [] result += self.__validate_is_not_empty__("module") result += self.__validate_is_not_empty__("stage") - result += self.__validate_is_not_empty__("c4k_executabel_name") + result += self.__validate_is_not_empty__("c4k_executable_name") result += self.__validate_is_not_empty__("c4k_grafana_cloud_user") result += self.__validate_is_not_empty__("c4k_grafana_cloud_password") if self.dns_record: @@ -62,4 +62,4 @@ class C4k(Validateable): config_path = f"{build_path}/out_c4k_config.yaml" auth_path = f"{build_path}/out_c4k_auth.yaml" output_path = f"{build_path}/out_{module}.yaml" - return f"c4k-{self.c4k_executabel_name}-standalone.jar {config_path} {auth_path} > {output_path}" + return f"c4k-{self.c4k_executable_name}-standalone.jar {config_path} {auth_path} > {output_path}" diff --git a/src/test/python/domain/test_c4k.py b/src/test/python/domain/test_c4k.py index 175980e..e9d14d9 100644 --- a/src/test/python/domain/test_c4k.py +++ b/src/test/python/domain/test_c4k.py @@ -87,11 +87,11 @@ def test_c4k_build_should_calculate_command(): sut = build_devops( { "project_root_path": ".", - "c4k_executabel_name": "executabel_name", + "c4k_executable_name": "executable_name", } ) assert ( - "c4k-executabel_name-standalone.jar " + "c4k-executable_name-standalone.jar " + "./target/name/module/out_c4k_config.yaml " + "./target/name/module/out_c4k_auth.yaml > " + "./target/name/module/out_module.yaml" From 64116edcb47b07c1f3d0ca856bcaca5c7ee2a6d3 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 11 May 2023 15:03:32 +0200 Subject: [PATCH 067/173] Use list comprehension instead of for --- src/main/python/ddadevops/domain/version.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 1b01b29..5907bfa 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -13,9 +13,7 @@ class Version(Validateable): suffix_str = None if len(snapshot_parsed) > 1: suffix_str = snapshot_parsed[1] - version_no_parsed = [] - for x in version_str.split("."): - version_no_parsed += [int(x)] + version_no_parsed = [int(x) for x in version_str.split(".")] return cls( version_no_parsed, suffix_str, From 6766691ee76e6cc016908e1d200139b24428fc4d Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 11 May 2023 15:20:49 +0200 Subject: [PATCH 068/173] Add review comments for version --- src/test/python/domain/test_version.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/python/domain/test_version.py b/src/test/python/domain/test_version.py index 2caee0d..6289d30 100644 --- a/src/test/python/domain/test_version.py +++ b/src/test/python/domain/test_version.py @@ -99,10 +99,12 @@ def test_should_create_major(): assert sut.to_string() == "2.0.0" def test_should_create_bump(): + # TODO: bom - 2023_05_11: Why does this not actually bump the version? version = Version.from_str("1.2.3-SNAPSHOT") sut = version.create_bump() assert sut.to_string() == "1.2.3-SNAPSHOT" + # TODO: bom - 2023_05_11: Why do we bump the minor version and not the patch? version = Version.from_str("1.2.3") sut = version.create_bump("SNAPSHOT") assert sut.to_string() == "1.3.0-SNAPSHOT" From f2c6f7787e2d332b5f3c505972b56664d7e2ea3f Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 11 May 2023 18:13:40 +0200 Subject: [PATCH 069/173] js & gradle works --- .../python/ddadevops/domain/build_file.py | 14 ++++++++- src/main/python/ddadevops/domain/version.py | 3 ++ src/test/python/domain/test_build_file.py | 29 ++++++++++++++++--- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py index 285c6d1..321ab33 100644 --- a/src/main/python/ddadevops/domain/build_file.py +++ b/src/main/python/ddadevops/domain/build_file.py @@ -1,6 +1,7 @@ from enum import Enum from typing import Optional from pathlib import Path +import re import json from .common import ( Validateable, @@ -48,5 +49,16 @@ class BuildFile(Validateable): return result def get_version(self) -> Version: - version = json.loads(self.content)["version"] + version = None + match self.build_file_type(): + case BuildFileType.JS: + print(json.loads(self.content)) + version = json.loads(self.content)["version"] + print(version) + case BuildFileType.JAVA_GRADLE: + version_line = re.search("\nversion = .*", self.content) + version_line_group = version_line.group() + version_string = re.search( + '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) + version = version_string.group() return Version.from_str(version) diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 5907bfa..41f0dc1 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -30,6 +30,9 @@ class Version(Validateable): self.snapshot_suffix = snapshot_suffix self.version_string = version_str + def __eq__(self, other): + return other and self.to_string() == other.to_string() + def is_snapshot(self): return not self.snapshot_suffix == None diff --git a/src/test/python/domain/test_build_file.py b/src/test/python/domain/test_build_file.py index 833f0c6..33a5f19 100644 --- a/src/test/python/domain/test_build_file.py +++ b/src/test/python/domain/test_build_file.py @@ -33,19 +33,40 @@ def test_sould_calculate_build_type(): ) assert sut.build_file_type() == BuildFileType.JAVA_CLOJURE + sut = BuildFile( + Path("./build.gradle"), + "content" + ) + assert sut.build_file_type() == BuildFileType.JAVA_GRADLE + + sut = BuildFile( + Path("./package.json"), + "content" + ) + assert sut.build_file_type() == BuildFileType.JS + def test_sould_parse_version(): sut = BuildFile( - Path("./package.js"), + Path("./package.json"), """{ - "name": "c4k-jira", + "name":"c4k-jira", "description": "Generate c4k yaml for a jira deployment.", "author": "meissa GmbH", "version": "1.1.5-SNAPSHOT", "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-jira#readme", - "bin": { + "bin":{ "c4k-jira": "./c4k-jira.js" - }, + } } +""" + ) + assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") + + sut = BuildFile( + Path("./build.gradle"), + """ +version = "1.1.5-SNAPSHOT" + """ ) assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") From 6a0ed7deefe15b0392a581f6d2233085c71221d6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 11 May 2023 18:37:47 +0200 Subject: [PATCH 070/173] get_version now works --- .../python/ddadevops/domain/build_file.py | 39 ++++++--- src/test/python/domain/test_build_file.py | 86 +++++++++++++------ 2 files changed, 85 insertions(+), 40 deletions(-) diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py index 321ab33..d15f038 100644 --- a/src/main/python/ddadevops/domain/build_file.py +++ b/src/main/python/ddadevops/domain/build_file.py @@ -49,16 +49,31 @@ class BuildFile(Validateable): return result def get_version(self) -> Version: - version = None - match self.build_file_type(): - case BuildFileType.JS: - print(json.loads(self.content)) - version = json.loads(self.content)["version"] - print(version) - case BuildFileType.JAVA_GRADLE: - version_line = re.search("\nversion = .*", self.content) - version_line_group = version_line.group() - version_string = re.search( - '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) - version = version_string.group() + try: + match self.build_file_type(): + case BuildFileType.JS: + print(json.loads(self.content)) + version = json.loads(self.content)["version"] + print(version) + case BuildFileType.JAVA_GRADLE: + version_line = re.search("\nversion = .*", self.content) + version_line_group = version_line.group() + version_string = re.search( + '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) + version = version_string.group() + case BuildFileType.PYTHON: + version_line = re.search("\nversion = .*\n", self.content) + version_line_group = version_line.group() + version_string = re.search( + '[0-9]*\\.[0-9]*\\.[0-9]*(-dev)?[0-9]*', version_line_group) + version = version_string.group() + case BuildFileType.JAVA_CLOJURE: + version_line = re.search("\\(defproject .*\n", self.content) + version_line_group = version_line.group() + version_string = re.search( + '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) + version = version_string.group() + except: + raise Exception(f"Version not found in file {self.file_path}") + return Version.from_str(version) diff --git a/src/test/python/domain/test_build_file.py b/src/test/python/domain/test_build_file.py index 33a5f19..4db3076 100644 --- a/src/test/python/domain/test_build_file.py +++ b/src/test/python/domain/test_build_file.py @@ -1,3 +1,4 @@ +import pytest from pathlib import Path from src.main.python.ddadevops.domain import ( BuildFileType, @@ -7,48 +8,32 @@ from src.main.python.ddadevops.domain import ( def test_sould_validate_build_file(): - sut = BuildFile( - Path("./project.clj"), - "content" - ) + sut = BuildFile(Path("./project.clj"), "content") assert sut.is_valid() - sut = BuildFile( - None, - "" - ) + sut = BuildFile(None, "") assert not sut.is_valid() - sut = BuildFile( - Path("./unknown.extension"), - "content" - ) + sut = BuildFile(Path("./unknown.extension"), "content") assert not sut.is_valid() def test_sould_calculate_build_type(): - sut = BuildFile( - Path("./project.clj"), - "content" - ) + sut = BuildFile(Path("./project.clj"), "content") assert sut.build_file_type() == BuildFileType.JAVA_CLOJURE - sut = BuildFile( - Path("./build.gradle"), - "content" - ) + sut = BuildFile(Path("./build.gradle"), "content") assert sut.build_file_type() == BuildFileType.JAVA_GRADLE - sut = BuildFile( - Path("./package.json"), - "content" - ) + sut = BuildFile(Path("./package.json"), "content") assert sut.build_file_type() == BuildFileType.JS -def test_sould_parse_version(): + +def test_sould_parse_js(): sut = BuildFile( Path("./package.json"), - """{ + """ +{ "name":"c4k-jira", "description": "Generate c4k yaml for a jira deployment.", "author": "meissa GmbH", @@ -58,16 +43,61 @@ def test_sould_parse_version(): "c4k-jira": "./c4k-jira.js" } } -""" +""", ) assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") + sut = BuildFile( + Path("./package.json"), + """ +{ + "name":"c4k-jira", + "description": "Generate c4k yaml for a jira deployment.", + "author": "meissa GmbH", + "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-jira#readme", + "bin":{ + "c4k-jira": "./c4k-jira.js" + } +} +""", + ) + with pytest.raises(Exception): + sut.get_version() + + +def test_sould_parse_gradle(): sut = BuildFile( Path("./build.gradle"), """ version = "1.1.5-SNAPSHOT" -""" +""", ) assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") + +def test_sould_parse_py(): + sut = BuildFile( + Path("./build.py"), + """ +from pybuilder.core import init, use_plugin, Author +use_plugin("python.core") + +name = "ddadevops" +version = "1.1.5-dev" +""", + ) + assert sut.get_version() == Version.from_str("1.1.5-dev") + + +def test_sould_parse_clj(): + sut = BuildFile( + Path("./project.clj"), + """ +(defproject org.domaindrivenarchitecture/c4k-jira "1.1.5-SNAPSHOT" + :description "jira c4k-installation package" + :url "https://domaindrivenarchitecture.org" + ) +""", + ) + assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") From 15bf7e58872715a64bec3e0eaa119e720c8d3869 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 11 May 2023 19:21:02 +0200 Subject: [PATCH 071/173] parse & set now works --- .../python/ddadevops/domain/build_file.py | 84 +++++++++++++++---- src/test/python/domain/test_build_file.py | 59 ++++++++++--- 2 files changed, 114 insertions(+), 29 deletions(-) diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py index d15f038..7fc941d 100644 --- a/src/main/python/ddadevops/domain/build_file.py +++ b/src/main/python/ddadevops/domain/build_file.py @@ -12,17 +12,19 @@ from .version import ( Version, ) + class BuildFileType(Enum): - JS = '.json' - JAVA_GRADLE = '.gradle' + JS = ".json" + JAVA_GRADLE = ".gradle" JAVA_CLOJURE = ".clj" - PYTHON = '.py' + PYTHON = ".py" + class BuildFile(Validateable): def __init__(self, file_path: Path, content: str): self.file_path = file_path self.content = content - + def validate(self): result = [] result += self.__validate_is_not_empty__("file_path") @@ -36,13 +38,13 @@ class BuildFile(Validateable): return None config_file_type = self.file_path.suffix match config_file_type: - case '.json': + case ".json": result = BuildFileType.JS - case '.gradle': + case ".gradle": result = BuildFileType.JAVA_GRADLE - case '.clj': + case ".clj": result = BuildFileType.JAVA_CLOJURE - case '.py': + case ".py": result = BuildFileType.PYTHON case _: result = None @@ -52,28 +54,74 @@ class BuildFile(Validateable): try: match self.build_file_type(): case BuildFileType.JS: - print(json.loads(self.content)) - version = json.loads(self.content)["version"] - print(version) + version_str = json.loads(self.content)["version"] case BuildFileType.JAVA_GRADLE: + # TODO: '\nversion = ' will not parse all ?! version_line = re.search("\nversion = .*", self.content) version_line_group = version_line.group() version_string = re.search( - '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) - version = version_string.group() + "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?", version_line_group + ) + version_str = version_string.group() case BuildFileType.PYTHON: + # TODO: '\nversion = ' will not parse all ?! version_line = re.search("\nversion = .*\n", self.content) version_line_group = version_line.group() version_string = re.search( - '[0-9]*\\.[0-9]*\\.[0-9]*(-dev)?[0-9]*', version_line_group) - version = version_string.group() + "[0-9]*\\.[0-9]*\\.[0-9]*(-dev)?[0-9]*", version_line_group + ) + version_str = version_string.group() case BuildFileType.JAVA_CLOJURE: + # TODO: unsure about the trailing '\n' ! version_line = re.search("\\(defproject .*\n", self.content) version_line_group = version_line.group() version_string = re.search( - '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) - version = version_string.group() + "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?", version_line_group + ) + version_str = version_string.group() except: raise Exception(f"Version not found in file {self.file_path}") - return Version.from_str(version) + result = Version.from_str(version_str) + result.throw_if_invalid() + + return result + + def set_version(self, new_version: Version): + # TODO: How can we create regex-pattern constants to use them at both places? + try: + match self.build_file_type(): + case BuildFileType.JS: + json_data = json.loads(self.content) + json_data["version"] = new_version.to_string() + self.content = json.dumps(json_data, indent=4) + case BuildFileType.JAVA_GRADLE: + substitute = re.sub( + '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', + f'\nversion = "{new_version.to_string()}"', + self.content, + ) + self.content = substitute + case BuildFileType.PYTHON: + substitute = re.sub( + '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-dev)?[0-9]*"', + f'\nversion = "{new_version.to_string()}"', + self.content, + ) + self.content = substitute + case BuildFileType.JAVA_CLOJURE: + version_line = re.search("\\(defproject .*\n", self.content) + version_line_group = version_line.group() + # TODO: we should stick here on defproject instead of first line! + version_string = re.search( + "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?", version_line_group + ) + version_str = version_string.group() + substitute = re.sub( + '"[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', + f'"{new_version.to_string()}"', + self.content, + ) + self.content = substitute + except: + raise Exception(f"Version not found in file {self.file_path}") diff --git a/src/test/python/domain/test_build_file.py b/src/test/python/domain/test_build_file.py index 4db3076..cfc0c74 100644 --- a/src/test/python/domain/test_build_file.py +++ b/src/test/python/domain/test_build_file.py @@ -29,7 +29,7 @@ def test_sould_calculate_build_type(): assert sut.build_file_type() == BuildFileType.JS -def test_sould_parse_js(): +def test_sould_parse_and_set_js(): sut = BuildFile( Path("./package.json"), """ @@ -52,20 +52,29 @@ def test_sould_parse_js(): """ { "name":"c4k-jira", - "description": "Generate c4k yaml for a jira deployment.", - "author": "meissa GmbH", - "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-jira#readme", - "bin":{ - "c4k-jira": "./c4k-jira.js" - } } """, ) with pytest.raises(Exception): sut.get_version() + sut = BuildFile( + Path("./package.json"), + """ +{ + "name":"c4k-jira", + "version": "1.1.5-SNAPSHOT" +} +""", + ) + sut.set_version(Version.from_str("1.1.5-SNAPSHOT").create_major()) + assert """{ + "name": "c4k-jira", + "version": "2.0.0" +}""" == sut.content -def test_sould_parse_gradle(): + +def test_sould_parse_and_set_version_for_gradle(): sut = BuildFile( Path("./build.gradle"), """ @@ -75,8 +84,16 @@ version = "1.1.5-SNAPSHOT" ) assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") + sut = BuildFile( + Path("./build.gradle"), + """ +version = "1.1.5-SNAPSHOT" +""", + ) + sut.set_version(Version.from_str("1.1.5-SNAPSHOT").create_major()) + assert '\nversion = "2.0.0"\n' == sut.content -def test_sould_parse_py(): +def test_sould_parse_and_set_version_for_py(): sut = BuildFile( Path("./build.py"), """ @@ -89,15 +106,35 @@ version = "1.1.5-dev" ) assert sut.get_version() == Version.from_str("1.1.5-dev") + sut = BuildFile( + Path("./build.py"), + """ +version = "1.1.5-dev1" +""", + ) + sut.set_version(Version.from_str("1.1.5-dev1").create_major()) + assert '\nversion = "2.0.0"\n' == sut.content -def test_sould_parse_clj(): + +def test_sould_parse_and_set_version_for_clj(): sut = BuildFile( Path("./project.clj"), """ (defproject org.domaindrivenarchitecture/c4k-jira "1.1.5-SNAPSHOT" :description "jira c4k-installation package" :url "https://domaindrivenarchitecture.org" - ) +) """, ) assert sut.get_version() == Version.from_str("1.1.5-SNAPSHOT") + + sut = BuildFile( + Path("./project.clj"), + """ +(defproject org.domaindrivenarchitecture/c4k-jira "1.1.5-SNAPSHOT" + :description "jira c4k-installation package" +) +""", + ) + sut.set_version(Version.from_str("1.1.5-SNAPSHOT").create_major()) + assert '\n(defproject org.domaindrivenarchitecture/c4k-jira "2.0.0"\n :description "jira c4k-installation package"\n)\n' == sut.content From 634c8ff7256d1c56257af67adf72ab0fdbdf91e9 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 12 May 2023 14:32:05 +0200 Subject: [PATCH 072/173] adjust names --- doc/architecture/Domain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index f1d3e33..070b6ae 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -50,7 +50,7 @@ classDiagram } class Version { - get_version_string() + to_string() create_major() create_minor() create_patch() @@ -60,8 +60,8 @@ classDiagram Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" Release: mixins - Release o-- "0..1" BuildFile: primary_release_file - Release o-- "0..n" BuildFile: secondary_release_files + Release o-- "0..1" BuildFile: primary_build_file + Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version C4k *-- DnsRecord From ce69b5970d7cb4dccb9d4b5fd8ac1f7d3b2da487 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 12 May 2023 14:32:31 +0200 Subject: [PATCH 073/173] adjust version to bump only patch level --- src/main/python/ddadevops/domain/version.py | 3 +-- src/test/python/domain/test_version.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 41f0dc1..4083839 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -56,8 +56,7 @@ class Version(Validateable): if self.is_snapshot(): return Version(new_version_list, snapshot_suffix=self.snapshot_suffix, version_str=None) else: - new_version_list[2] = 0 - new_version_list[1] += 1 + new_version_list[2] += 1 return Version(new_version_list, snapshot_suffix=snapshot_suffix, version_str=None) def create_patch(self): diff --git a/src/test/python/domain/test_version.py b/src/test/python/domain/test_version.py index 6289d30..850fd7a 100644 --- a/src/test/python/domain/test_version.py +++ b/src/test/python/domain/test_version.py @@ -99,16 +99,14 @@ def test_should_create_major(): assert sut.to_string() == "2.0.0" def test_should_create_bump(): - # TODO: bom - 2023_05_11: Why does this not actually bump the version? version = Version.from_str("1.2.3-SNAPSHOT") sut = version.create_bump() assert sut.to_string() == "1.2.3-SNAPSHOT" - # TODO: bom - 2023_05_11: Why do we bump the minor version and not the patch? version = Version.from_str("1.2.3") sut = version.create_bump("SNAPSHOT") - assert sut.to_string() == "1.3.0-SNAPSHOT" + assert sut.to_string() == "1.2.4-SNAPSHOT" version = Version.from_str("1.0.0") sut = version.create_bump("SNAPSHOT") - assert sut.to_string() == "1.1.0-SNAPSHOT" \ No newline at end of file + assert sut.to_string() == "1.0.1-SNAPSHOT" \ No newline at end of file From 1e2fcce452e481e28bc5f12bbb2adcb40fbbf654 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 12 May 2023 14:38:46 +0200 Subject: [PATCH 074/173] create version outside of devops aggregate --- src/main/python/ddadevops/domain/devops_factory.py | 3 +-- src/test/python/domain/test_devops_factory.py | 14 ++++++++------ src/test/python/domain/test_helper.py | 9 +++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index d60bd1b..899ee48 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -12,7 +12,7 @@ class DevopsFactory: def __init__(self): pass - def build_devops(self, input) -> Devops: + def build_devops(self, input: dict, version: Version) -> Devops: build_types = self.__parse_build_types__(input["build_types"]) mixin_types = self.__parse_mixin_types__(input["mixin_types"]) @@ -24,7 +24,6 @@ class DevopsFactory: mixins = {} if MixinType.RELEASE in mixin_types: - version = Version.from_str(input["version"]) mixins[MixinType.RELEASE] = Release(input, version) devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index e1da188..b81d734 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -1,6 +1,6 @@ import pytest -from src.main.python.ddadevops.domain.devops_factory import ( - DevopsFactory, +from src.main.python.ddadevops.domain import ( + DevopsFactory, Version ) @@ -26,7 +26,8 @@ def test_devops_factory(): "image_dockerhub_user": "dockerhub_user", "image_dockerhub_password": "dockerhub_password", "image_tag": "docker_image_tag", - } + }, + Version.from_str("1.0.0") ) assert sut is not None @@ -40,7 +41,8 @@ def test_devops_factory(): "mixin_types": [], "c4k_grafana_cloud_user": "user", "c4k_grafana_cloud_password": "password", - } + }, + Version.from_str("1.0.0") ) assert sut is not None @@ -52,11 +54,11 @@ def test_devops_factory(): "project_root_path": "../../..", "build_types": [], "mixin_types": ["RELEASE"], - "version": "1.0.0", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", "release_config_file": "project.clj", - } + }, + Version.from_str("1.0.0") ) assert sut is not None diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index 381c489..3ed0afb 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -1,4 +1,4 @@ -from src.main.python.ddadevops.domain import DevopsFactory, Devops +from src.main.python.ddadevops.domain import DevopsFactory, Devops, Version def devops_config(overrides: dict) -> dict: @@ -18,7 +18,6 @@ def devops_config(overrides: dict) -> dict: "c4k_grafana_cloud_password": "password", "c4k_grafana_cloud_url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", "c4k_auth": {}, - "version": "1.0.0", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", @@ -29,5 +28,7 @@ def devops_config(overrides: dict) -> dict: return input -def build_devops(overrides: dict) -> Devops: - return DevopsFactory().build_devops(devops_config(overrides)) +def build_devops( + overrides: dict, version: Version = Version.from_str("1.0.0-SNAPSHOT") +) -> Devops: + return DevopsFactory().build_devops(devops_config(overrides), version=version) From 17f41dbd7ae74ee2f241ffa5065b7dcc5fa4b1d6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 12 May 2023 14:57:23 +0200 Subject: [PATCH 075/173] introduce build_file ids --- src/main/python/ddadevops/domain/release.py | 14 ++++++++++++-- src/test/python/domain/test_helper.py | 3 ++- src/test/python/domain/test_release.py | 10 ++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 7aaf162..b5b126b 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -20,7 +20,8 @@ class Release(Validateable): self.release_type = ReleaseType[input.get("release_type", "NONE")] self.release_main_branch = input.get("release_main_branch", "main") self.release_current_branch = input.get("release_current_branch") - self.release_config_file = input.get("release_config_file", "./project.clj") + self.release_primary_build_file = input.get("release_primary_build_file", "./project.clj") + self.release_secondary_build_files = input.get("release_secondary_build_files", []) self.version = version def validate(self): @@ -28,8 +29,17 @@ class Release(Validateable): result += self.__validate_is_not_empty__("release_type") result += self.__validate_is_not_empty__("release_main_branch") result += self.__validate_is_not_empty__("release_current_branch") - result += self.__validate_is_not_empty__("release_config_file") + result += self.__validate_is_not_empty__("release_primary_build_file") result += self.__validate_is_not_empty__("version") + try: + Path(self.release_primary_build_file) + except Exception as e: + result.append(f"release_primary_build_file must be a valid path but was {e}") + for path in self.release_secondary_build_files: + try: + Path(path) + except Exception as e: + result.append(f"release_secondary_build_file must be contain valid paths but was {e}") if self.version: result += self.version.validate() if ( diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index 3ed0afb..b02cfd2 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -21,7 +21,8 @@ def devops_config(overrides: dict) -> dict: "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", - "release_config_file": "./project.clj", + "release_primary_build_file": "./project.clj", + "release_secondary_build_file": [], } input = default.copy() input.update(overrides) diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index 6c80915..3fd4cf9 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -38,3 +38,13 @@ def test_sould_validate_release(): Version.from_str("1.3.1-SNAPSHOT"), ) assert not sut.is_valid() + + sut = Release( + devops_config( + { + "release_primary_build_file": 1, + } + ), + Version.from_str("1.3.1-SNAPSHOT"), + ) + assert not sut.is_valid() From c13b70150b0d9b92a93b2a9f1634600a30a799a4 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 13 May 2023 16:01:35 +0200 Subject: [PATCH 076/173] introduce init service --- doc/architecture/Domain.md | 4 +-- src/main/python/ddadevops/domain/__init__.py | 1 + .../python/ddadevops/domain/devops_factory.py | 2 +- .../python/ddadevops/domain/init_service.py | 29 +++++++++++++++ .../ddadevops/infrastructure/__init__.py | 1 + .../infrastructure/infrastructure.py | 28 --------------- .../ddadevops/infrastructure/repository.py | 35 +++++++++++++++++++ src/test/python/domain/test_devops_factory.py | 3 +- src/test/python/domain/test_helper.py | 20 +++++++++-- src/test/python/domain/test_init_service.py | 19 ++++++++++ 10 files changed, 107 insertions(+), 35 deletions(-) create mode 100644 src/main/python/ddadevops/domain/init_service.py create mode 100644 src/main/python/ddadevops/infrastructure/repository.py create mode 100644 src/test/python/domain/test_init_service.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 070b6ae..7f608b5 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -45,8 +45,8 @@ classDiagram file_path [id] content build_file_type() - getVersion() - setVersion(version) + get_version() + set_version(version) } class Version { diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 3662bc8..dd6af5a 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -5,3 +5,4 @@ from .c4k import C4k from .release import Release, EnvironmentKeys from .version import Version from .build_file import BuildFileType, BuildFile +from .init_service import InitService \ No newline at end of file diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 899ee48..6cb4a6d 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -12,7 +12,7 @@ class DevopsFactory: def __init__(self): pass - def build_devops(self, input: dict, version: Version) -> Devops: + def build_devops(self, input: dict, version: Version = None) -> Devops: build_types = self.__parse_build_types__(input["build_types"]) mixin_types = self.__parse_mixin_types__(input["mixin_types"]) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py new file mode 100644 index 0000000..00927c5 --- /dev/null +++ b/src/main/python/ddadevops/domain/init_service.py @@ -0,0 +1,29 @@ +from pathlib import Path +from .common import Devops, MixinType +from .devops_factory import DevopsFactory +from .version import Version +from src.main.python.ddadevops.infrastructure import ( + BuildFileRepository +) + +class InitService: + def __init__(self, devops_factory, build_file_repository): + self.devops_factory = devops_factory + self.build_file_repository = build_file_repository + + @classmethod + def prod(cls): + return cls( + DevopsFactory(), + BuildFileRepository(), + ) + + def initialize(self, input: dict) -> Devops: + mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) + + if MixinType.RELEASE in mixin_types: + primary_build_file_id = input.get("release_primary_build_file", "./project.clj") + primary_build_file = self.build_file_repository.get(Path(primary_build_file_id)) + version = primary_build_file.get_version() + + return self.devops_factory.build_devops(input, version=version) diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index 10deeea..baba117 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -1 +1,2 @@ from .infrastructure import FileApi, ImageApi, ResourceApi, ExecutionApi, ProjectRepository +from .repository import DevopsRepository, BuildFileRepository \ No newline at end of file diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index c165ae9..1b50682 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -8,34 +8,6 @@ import deprecation from ..domain import Devops, Image, C4k, Release, BuildFile from ..python_util import execute - -class DevopsRepository: - def get_devops(self, project) -> Devops: - devops = project.get_property("devops") - devops.throw_if_invalid() - return devops - - def set_devops(self, project, devops: Devops): - devops.throw_if_invalid() - project.set_property("devops", devops) - - -class BuildFileRepository: - def get(self, path: Path) -> BuildFile: - with open(path, "r", encoding="utf-8") as file: - content = file.read() - result = BuildFile(path, content) - result.throw_if_invalid() - return result - - def write(self, build_file: BuildFile): - build_file.throw_if_invalid() - with open(build_file.file_path, "r+", encoding="utf-8") as file: - file.seek(0) - file.write(build_file.content) - - - class ProjectRepository: pass diff --git a/src/main/python/ddadevops/infrastructure/repository.py b/src/main/python/ddadevops/infrastructure/repository.py new file mode 100644 index 0000000..b694947 --- /dev/null +++ b/src/main/python/ddadevops/infrastructure/repository.py @@ -0,0 +1,35 @@ +from pathlib import Path +from sys import stdout +from os import chmod +from subprocess import run +from pkg_resources import resource_string +import yaml +import deprecation +from ..domain import Devops, Image, C4k, Release, BuildFile +from ..python_util import execute + + +class DevopsRepository: + def get_devops(self, project) -> Devops: + devops = project.get_property("devops") + devops.throw_if_invalid() + return devops + + def set_devops(self, project, devops: Devops): + devops.throw_if_invalid() + project.set_property("devops", devops) + + +class BuildFileRepository: + def get(self, path: Path) -> BuildFile: + with open(path, "r", encoding="utf-8") as file: + content = file.read() + result = BuildFile(path, content) + result.throw_if_invalid() + return result + + def write(self, build_file: BuildFile): + build_file.throw_if_invalid() + with open(build_file.file_path, "r+", encoding="utf-8") as file: + file.seek(0) + file.write(build_file.content) diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index b81d734..7287869 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -26,8 +26,7 @@ def test_devops_factory(): "image_dockerhub_user": "dockerhub_user", "image_dockerhub_password": "dockerhub_password", "image_tag": "docker_image_tag", - }, - Version.from_str("1.0.0") + } ) assert sut is not None diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/test_helper.py index b02cfd2..52f18ba 100644 --- a/src/test/python/domain/test_helper.py +++ b/src/test/python/domain/test_helper.py @@ -1,4 +1,5 @@ -from src.main.python.ddadevops.domain import DevopsFactory, Devops, Version +from pathlib import Path +from src.main.python.ddadevops.domain import DevopsFactory, Devops, Version, BuildFile def devops_config(overrides: dict) -> dict: @@ -21,7 +22,7 @@ def devops_config(overrides: dict) -> dict: "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", - "release_primary_build_file": "./project.clj", + "release_primary_build_file": "./package.json", "release_secondary_build_file": [], } input = default.copy() @@ -33,3 +34,18 @@ def build_devops( overrides: dict, version: Version = Version.from_str("1.0.0-SNAPSHOT") ) -> Devops: return DevopsFactory().build_devops(devops_config(overrides), version=version) + + +class BuildFileRepositoryMock: + def get(self, path: Path) -> BuildFile: + return BuildFile( + Path("./package.json"), + """ +{ + "version": "1.1.5-SNAPSHOT" +} +""", + ) + + def write(self, build_file: BuildFile): + pass diff --git a/src/test/python/domain/test_init_service.py b/src/test/python/domain/test_init_service.py new file mode 100644 index 0000000..d7e8ba0 --- /dev/null +++ b/src/test/python/domain/test_init_service.py @@ -0,0 +1,19 @@ +import pytest +from src.main.python.ddadevops.domain import ( + InitService, + DevopsFactory, + Version, + MixinType, +) +from .test_helper import BuildFileRepositoryMock, devops_config + + +def test_sould_load_build_file(): + sut = InitService( + DevopsFactory(), + BuildFileRepositoryMock(), + ) + assert ( + Version.from_str("1.1.5-SNAPSHOT") + == sut.initialize(devops_config({})).mixins[MixinType.RELEASE].version + ) From e31d86ab69bcd3e9d35ed899d124a0c8abdee571 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 13 May 2023 16:09:03 +0200 Subject: [PATCH 077/173] refactor: test_helper -> helper --- src/test/python/domain/{test_helper.py => helper.py} | 0 src/test/python/domain/test_c4k.py | 6 +++--- src/test/python/domain/test_common.py | 6 ++---- src/test/python/domain/test_devops.py | 4 ++-- src/test/python/domain/test_image.py | 4 ++-- src/test/python/domain/test_init_service.py | 2 +- src/test/python/domain/test_release.py | 8 +++----- src/test/python/domain/test_version.py | 8 +++++--- 8 files changed, 18 insertions(+), 20 deletions(-) rename src/test/python/domain/{test_helper.py => helper.py} (100%) diff --git a/src/test/python/domain/test_helper.py b/src/test/python/domain/helper.py similarity index 100% rename from src/test/python/domain/test_helper.py rename to src/test/python/domain/helper.py diff --git a/src/test/python/domain/test_c4k.py b/src/test/python/domain/test_c4k.py index e9d14d9..bc69f1c 100644 --- a/src/test/python/domain/test_c4k.py +++ b/src/test/python/domain/test_c4k.py @@ -1,11 +1,11 @@ import pytest from pathlib import Path -from src.main.python.ddadevops.domain.common import ( +from src.main.python.ddadevops.domain import ( DnsRecord, BuildType, + C4k ) -from src.main.python.ddadevops.domain.c4k import C4k -from .test_helper import build_devops +from .helper import build_devops def test_creation(): diff --git a/src/test/python/domain/test_common.py b/src/test/python/domain/test_common.py index 5c9bc8a..ff606f1 100644 --- a/src/test/python/domain/test_common.py +++ b/src/test/python/domain/test_common.py @@ -1,18 +1,16 @@ from pybuilder.core import Project from pathlib import Path -from src.main.python.ddadevops.domain.common import ( +from src.main.python.ddadevops.domain import ( Validateable, DnsRecord, Devops, BuildType, -) -from src.main.python.ddadevops.domain import ( Version, ReleaseType, Release, ) from src.main.python.ddadevops.domain.image import Image -from .test_helper import build_devops +from .helper import build_devops class MockValidateable(Validateable): diff --git a/src/test/python/domain/test_devops.py b/src/test/python/domain/test_devops.py index 0a881ea..a7c081a 100644 --- a/src/test/python/domain/test_devops.py +++ b/src/test/python/domain/test_devops.py @@ -1,8 +1,8 @@ import pytest -from .test_helper import build_devops -from src.main.python.ddadevops.domain.common import ( +from src.main.python.ddadevops.domain import ( Devops, ) +from .helper import build_devops def test_devops_buildpath(): sut = build_devops({'module': "cloud", 'name': "meissa"}) diff --git a/src/test/python/domain/test_image.py b/src/test/python/domain/test_image.py index 61d3350..e26c98d 100644 --- a/src/test/python/domain/test_image.py +++ b/src/test/python/domain/test_image.py @@ -1,9 +1,9 @@ from pybuilder.core import Project from pathlib import Path -from src.main.python.ddadevops.domain.common import ( +from src.main.python.ddadevops.domain import ( BuildType, ) -from .test_helper import build_devops +from .helper import build_devops def test_devops_build_commons_path(): diff --git a/src/test/python/domain/test_init_service.py b/src/test/python/domain/test_init_service.py index d7e8ba0..5ae5c2a 100644 --- a/src/test/python/domain/test_init_service.py +++ b/src/test/python/domain/test_init_service.py @@ -5,7 +5,7 @@ from src.main.python.ddadevops.domain import ( Version, MixinType, ) -from .test_helper import BuildFileRepositoryMock, devops_config +from .helper import BuildFileRepositoryMock, devops_config def test_sould_load_build_file(): diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index 3fd4cf9..ec1ed9b 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -1,19 +1,17 @@ from pybuilder.core import Project from pathlib import Path -from src.main.python.ddadevops.domain.common import ( +from src.main.python.ddadevops.domain import ( Validateable, DnsRecord, Devops, BuildType, MixinType, -) -from src.main.python.ddadevops.domain import ( Version, ReleaseType, Release, + Image, ) -from src.main.python.ddadevops.domain.image import Image -from .test_helper import build_devops, devops_config +from .helper import build_devops, devops_config def test_sould_validate_release(): diff --git a/src/test/python/domain/test_version.py b/src/test/python/domain/test_version.py index 850fd7a..d601e67 100644 --- a/src/test/python/domain/test_version.py +++ b/src/test/python/domain/test_version.py @@ -3,9 +3,9 @@ from pathlib import Path from src.main.python.ddadevops.domain import ( Version, ReleaseType, + Image, ) -from src.main.python.ddadevops.domain.image import Image -from .test_helper import build_devops, devops_config +from .helper import build_devops, devops_config def test_version_creation(): @@ -63,6 +63,7 @@ def test_should_create_patch(): assert sut.to_string() == "1.2.4" assert version.to_string() == "1.2.3" + def test_should_create_minor(): version = Version.from_str("1.2.3-SNAPSHOT") sut = version.create_minor() @@ -98,6 +99,7 @@ def test_should_create_major(): sut = version.create_major() assert sut.to_string() == "2.0.0" + def test_should_create_bump(): version = Version.from_str("1.2.3-SNAPSHOT") sut = version.create_bump() @@ -109,4 +111,4 @@ def test_should_create_bump(): version = Version.from_str("1.0.0") sut = version.create_bump("SNAPSHOT") - assert sut.to_string() == "1.0.1-SNAPSHOT" \ No newline at end of file + assert sut.to_string() == "1.0.1-SNAPSHOT" From a13bda5a30ce0b446f98a2f1ec85bda9e58af760 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 13 May 2023 17:05:18 +0200 Subject: [PATCH 078/173] devops build now is working again --- src/main/python/ddadevops/devops_build.py | 23 +++++---- .../python/ddadevops/domain/init_service.py | 4 +- .../ddadevops/infrastructure/repository.py | 11 ++++- src/test/python/domain/helper.py | 2 +- src/test/python/resource_helper.py | 12 ++--- src/test/python/test_devops_build.py | 13 ++--- src/test/resources/package.json | 33 +++++++++++++ src/test/resources/project.clj | 47 +++++++++++++++++++ 8 files changed, 114 insertions(+), 31 deletions(-) create mode 100644 src/test/resources/package.json create mode 100644 src/test/resources/project.clj diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index b9e83f6..4ebe93b 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,9 +1,7 @@ from typing import Optional import deprecation -from .domain import ( - Devops, DevopsFactory -) -from .infrastructure import ProjectRepository, FileApi +from .domain import Devops, InitService +from .infrastructure import DevopsRepository, FileApi @deprecation.deprecated(deprecated_in="3.2", details="create objects direct instead") @@ -17,27 +15,28 @@ def create_devops_build_config( "build_dir_name": build_dir_name, } + def get_devops_build(project): return project.get_property("devops_build") + class DevopsBuild: def __init__(self, project, input: dict): self.project = project self.file_api = FileApi() - self.devops_factory = DevopsFactory() - self.repo = ProjectRepository() - devops = self.devops_factory.build_devops(input) - self.repo.set_devops(self.project, devops) - self.repo.set_build(self.project, self) + self.init_service = InitService.prod(project.basedir) + self.devops_repo = DevopsRepository() + devops = self.init_service.initialize(input) + self.devops_repo.set_devops(self.project, devops) def name(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) return devops.name def build_path(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) return devops.build_path() def initialize_build_dir(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) self.file_api.clean_dir(devops.build_path()) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 00927c5..2f54d5a 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -12,10 +12,10 @@ class InitService: self.build_file_repository = build_file_repository @classmethod - def prod(cls): + def prod(cls, base_dir: str): return cls( DevopsFactory(), - BuildFileRepository(), + BuildFileRepository(base_dir), ) def initialize(self, input: dict) -> Devops: diff --git a/src/main/python/ddadevops/infrastructure/repository.py b/src/main/python/ddadevops/infrastructure/repository.py index b694947..b3c3546 100644 --- a/src/main/python/ddadevops/infrastructure/repository.py +++ b/src/main/python/ddadevops/infrastructure/repository.py @@ -21,8 +21,11 @@ class DevopsRepository: class BuildFileRepository: + def __init__(self, base_dir: str): + self.base_dir = Path(base_dir) + def get(self, path: Path) -> BuildFile: - with open(path, "r", encoding="utf-8") as file: + with open(self.base_dir.joinpath(path), "r", encoding="utf-8") as file: content = file.read() result = BuildFile(path, content) result.throw_if_invalid() @@ -30,6 +33,10 @@ class BuildFileRepository: def write(self, build_file: BuildFile): build_file.throw_if_invalid() - with open(build_file.file_path, "r+", encoding="utf-8") as file: + with open( + self.base_dir.joinpath(build_file.file_path), + "r+", + encoding="utf-8", + ) as file: file.seek(0) file.write(build_file.content) diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 52f18ba..a661caa 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -7,7 +7,7 @@ def devops_config(overrides: dict) -> dict: "name": "name", "module": "module", "stage": "test", - "project_root_path": "../../..", + "project_root_path": "root_path", "build_dir_name": "target", "build_types": ["IMAGE", "C4K"], "mixin_types": ["RELEASE"], diff --git a/src/test/python/resource_helper.py b/src/test/python/resource_helper.py index 7de20b5..295e7fc 100644 --- a/src/test/python/resource_helper.py +++ b/src/test/python/resource_helper.py @@ -1,12 +1,8 @@ from pathlib import Path from src.main.python.ddadevops.infrastructure import ExecutionApi -class ResourceHelper(): - def __init__(self, file_name = 'config.json'): - self.TEST_FILE_NAME = file_name - self.TEST_FILE_ROOT = Path('src/test/resources/') - self.TEST_FILE_PATH = self.TEST_FILE_ROOT / self.TEST_FILE_NAME - def copy_files(self, source: Path, target: Path): - api = ExecutionApi() - api.execute(f"cp {source} {target}") +def copy_resource(source: Path, target: Path): + api = ExecutionApi() + res_source = Path('src/test/resources/').joinpath(source) + api.execute(f"cp {str(res_source)} {str(target)}") diff --git a/src/test/python/test_devops_build.py b/src/test/python/test_devops_build.py index c107058..c4c35b5 100644 --- a/src/test/python/test_devops_build.py +++ b/src/test/python/test_devops_build.py @@ -1,18 +1,19 @@ import os +from pathlib import Path from pybuilder.core import Project -from src.main.python.ddadevops.devops_build import DevopsBuild -from .domain.test_helper import devops_config - +from src.main.python.ddadevops import DevopsBuild +from .domain.helper import devops_config +from .resource_helper import copy_resource def test_devops_build(tmp_path): - tmp_path_str = str(tmp_path) + copy_resource(Path('package.json'), tmp_path) + project = Project(str(tmp_path), name="name") - project = Project(tmp_path_str, name="name") devops_build = DevopsBuild( project, devops_config( { - "project_root_path": tmp_path_str, + "project_root_path": str(tmp_path), } ), ) diff --git a/src/test/resources/package.json b/src/test/resources/package.json new file mode 100644 index 0000000..d78ae03 --- /dev/null +++ b/src/test/resources/package.json @@ -0,0 +1,33 @@ +{ + "name": "c4k-jira", + "description": "Generate c4k yaml for a jira deployment.", + "author": "meissa GmbH", + "version": "1.1.5-SNAPSHOT", + "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-jira#readme", + "repository": "https://www.npmjs.com/package/c4k-jira", + "license": "APACHE2", + "main": "c4k-jira.js", + "bin": { + "c4k-jira": "./c4k-jira.js" + }, + "keywords": [ + "cljs", + "jira", + "k8s", + "c4k", + "deployment", + "yaml", + "convention4kubernetes" + ], + "bugs": { + "url": "https://gitlab.com/domaindrivenarchitecture/c4k-jira/issues" + }, + "dependencies": { + "js-base64": "^3.6.1", + "js-yaml": "^4.0.0" + }, + "devDependencies": { + "shadow-cljs": "^2.11.18", + "source-map-support": "^0.5.19" + } +} diff --git a/src/test/resources/project.clj b/src/test/resources/project.clj new file mode 100644 index 0000000..36ada11 --- /dev/null +++ b/src/test/resources/project.clj @@ -0,0 +1,47 @@ +(defproject org.domaindrivenarchitecture/c4k-jira "1.1.5-SNAPSHOT" + :description "jira c4k-installation package" + :url "https://domaindrivenarchitecture.org" + :license {:name "Apache License, Version 2.0" + :url "https://www.apache.org/licenses/LICENSE-2.0.html"} + :dependencies [[org.clojure/clojure "1.11.1"] + [org.clojure/tools.reader "1.3.6"] + [org.domaindrivenarchitecture/c4k-common-clj "2.0.3"] + [hickory "0.7.1"]] + :target-path "target/%s/" + :source-paths ["src/main/cljc" + "src/main/clj"] + :resource-paths ["src/main/resources"] + :repositories [["snapshots" :clojars] + ["releases" :clojars]] + :deploy-repositories [["snapshots" {:sign-releases false :url "https://clojars.org/repo"}] + ["releases" {:sign-releases false :url "https://clojars.org/repo"}]] + :profiles {:test {:test-paths ["src/test/cljc"] + :resource-paths ["src/test/resources"] + :dependencies [[dda/data-test "0.1.1"]]} + :dev {:plugins [[lein-shell "0.5.0"]]} + :uberjar {:aot :all + :main dda.c4k-jira.uberjar + :uberjar-name "c4k-jira-standalone.jar" + :dependencies [[org.clojure/tools.cli "1.0.214"] + [ch.qos.logback/logback-classic "1.4.5" + :exclusions [com.sun.mail/javax.mail]] + [org.slf4j/jcl-over-slf4j "2.0.6"]]}} + :release-tasks [["test"] + ["vcs" "assert-committed"] + ["change" "version" "leiningen.release/bump-version" "release"] + ["vcs" "commit"] + ["vcs" "tag" "v" "--no-sign"] + ["change" "version" "leiningen.release/bump-version"]] + :aliases {"native" ["shell" + "native-image" + "--report-unsupported-elements-at-runtime" + "--initialize-at-build-time" + "-jar" "target/uberjar/c4k-jira-standalone.jar" + "-H:ResourceConfigurationFiles=graalvm-resource-config.json" + "-H:Log=registerResource" + "-H:Name=target/graalvm/${:name}"] + "inst" ["shell" "sudo" + "install" + "-m=755" + "target/uberjar/c4k-jira-standalone.jar" + "/usr/local/bin/c4k-jira-standalone.jar"]}) From 73e73e8d3db7834f1401eaed9fc6f2a0f70db0cd Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 13 May 2023 17:20:30 +0200 Subject: [PATCH 079/173] devops image build now may work --- .../application/image_build_service.py | 16 +++++++++---- .../python/ddadevops/devops_image_build.py | 16 ++++++------- .../python/ddadevops/domain/init_service.py | 1 + src/test/python/test_devops_build.py | 5 ++-- src/test/python/test_image_build.py | 24 +++++++++++-------- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index 9c43c31..33a3d2d 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -3,10 +3,18 @@ from src.main.python.ddadevops.infrastructure import FileApi, ResourceApi, Image class ImageBuildService: - def __init__(self): - self.file_api = FileApi() - self.resource_api = ResourceApi() - self.image_api = ImageApi() + def __init__(self, file_api: FileApi, resource_api: ResourceApi, image_api: ImageApi): + self.file_api = file_api + self.resource_api = resource_api + self.image_api = image_api + + @classmethod + def prod(cls): + return cls( + FileApi(), + ResourceApi(), + ImageApi(), + ) def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}") diff --git a/src/main/python/ddadevops/devops_image_build.py b/src/main/python/ddadevops/devops_image_build.py index db55b7e..c5684bb 100644 --- a/src/main/python/ddadevops/devops_image_build.py +++ b/src/main/python/ddadevops/devops_image_build.py @@ -35,32 +35,32 @@ def create_devops_docker_build_config( class DevopsImageBuild(DevopsBuild): def __init__(self, project, input: dict): super().__init__(project, input) - self.image_build_service = ImageBuildService() - devops = self.repo.get_devops(self.project) + self.image_build_service = ImageBuildService.prod() + devops = self.devops_repo.get_devops(self.project) if BuildType.IMAGE not in devops.specialized_builds: raise ValueError(f"ImageBuild requires BuildType.IMAGE") def initialize_build_dir(self): super().initialize_build_dir() - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) self.image_build_service.initialize_build_dir(devops) def image(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) self.image_build_service.image(devops) def drun(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) self.image_build_service.drun(devops) def dockerhub_login(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) self.image_build_service.dockerhub_login(devops) def dockerhub_publish(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) self.image_build_service.dockerhub_publish(devops) def test(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) self.image_build_service.test(devops) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 2f54d5a..e6d690e 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -20,6 +20,7 @@ class InitService: def initialize(self, input: dict) -> Devops: mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) + version = None if MixinType.RELEASE in mixin_types: primary_build_file_id = input.get("release_primary_build_file", "./project.clj") diff --git a/src/test/python/test_devops_build.py b/src/test/python/test_devops_build.py index c4c35b5..f31f738 100644 --- a/src/test/python/test_devops_build.py +++ b/src/test/python/test_devops_build.py @@ -6,14 +6,15 @@ from .domain.helper import devops_config from .resource_helper import copy_resource def test_devops_build(tmp_path): + str_tmp_path = str(tmp_path) copy_resource(Path('package.json'), tmp_path) - project = Project(str(tmp_path), name="name") + project = Project(str_tmp_path, name="name") devops_build = DevopsBuild( project, devops_config( { - "project_root_path": str(tmp_path), + "project_root_path": str_tmp_path, } ), ) diff --git a/src/test/python/test_image_build.py b/src/test/python/test_image_build.py index acb2f3b..71e99ec 100644 --- a/src/test/python/test_image_build.py +++ b/src/test/python/test_image_build.py @@ -1,16 +1,20 @@ import os from pybuilder.core import Project -from src.main.python.ddadevops.domain import Image, Devops -from src.main.python.ddadevops.devops_image_build import DevopsImageBuild -from .domain.test_helper import devops_config +from src.main.python.ddadevops import DevopsImageBuild +from .domain.helper import devops_config def test_devops_docker_build(tmp_path): - build_dir = "build" - project_name = "testing-project" - module_name = "docker-test" - tmp_path_str = str(tmp_path) - - project = Project(tmp_path_str, name=project_name) - image_build = DevopsImageBuild(project, devops_config({})) + str_tmp_path = str(tmp_path) + project = Project(str_tmp_path, name="name") + image_build = DevopsImageBuild( + project, + devops_config( + { + "project_root_path": str_tmp_path, + "build_types": ["IMAGE"], + "mixin_types": [], + } + ), + ) assert image_build From 4baa012918fe9a972d4c953af07d789a7db3e9d6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 16 May 2023 08:47:11 +0200 Subject: [PATCH 080/173] introduce Credentials --- doc/architecture/Domain.md | 14 +++ requirements.txt | 3 +- src/main/python/ddadevops/c4k_build.py | 12 +- src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 2 +- .../python/ddadevops/domain/credentials.py | 56 +++++++++ .../python/ddadevops/domain/devops_factory.py | 13 +++ src/main/python/ddadevops/domain/version.py | 31 +++-- src/test/python/domain/test_crededntials.py | 107 ++++++++++++++++++ src/test/python/domain/test_devops.py | 2 +- src/test/python/test_c4k_build.py | 14 +-- 11 files changed, 229 insertions(+), 26 deletions(-) create mode 100644 src/main/python/ddadevops/domain/credentials.py create mode 100644 src/test/python/domain/test_crededntials.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 7f608b5..b2838b1 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -39,6 +39,18 @@ classDiagram release_current_branch version } + class Credentials { + + } + class CredentialMapping { + name + gopass_path + gopass_field + gopass_type() + name_for_input() + name_for_environment () + } + class BuildFile { <> @@ -60,10 +72,12 @@ classDiagram Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" Release: mixins + Devops *-- "0..1" Credentials: mixins Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version C4k *-- DnsRecord + Credentials *-- "0..n" CredentialMapping: mappings ``` diff --git a/requirements.txt b/requirements.txt index 7a391b0..c53d0df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ setuptools dda-python-terraform==2.0.1 packaging boto3 -pyyaml \ No newline at end of file +pyyaml +inflection \ No newline at end of file diff --git a/src/main/python/ddadevops/c4k_build.py b/src/main/python/ddadevops/c4k_build.py index 2cb7b15..76a4a16 100644 --- a/src/main/python/ddadevops/c4k_build.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -48,31 +48,31 @@ class C4kBuild(DevopsBuild): def __init__(self, project, config): super().__init__(project, config) self.execution_api = ExecutionApi() - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) if BuildType.C4K not in devops.specialized_builds: raise ValueError(f"C4kBuild requires BuildType.C4K") def update_runtime_config(self, dns_record: DnsRecord): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) devops.specialized_builds[BuildType.C4K].update_runtime_config(dns_record) - self.repo.set_devops(self.project, devops) + self.devops_repo.set_devops(self.project, devops) def write_c4k_config(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) path = devops.build_path() + "/out_c4k_config.yaml" self.file_api.write_yaml_to_file( path, devops.specialized_builds[BuildType.C4K].config() ) def write_c4k_auth(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) path = devops.build_path() + "/out_c4k_auth.yaml" self.file_api.write_yaml_to_file( path, devops.specialized_builds[BuildType.C4K].auth() ) def c4k_apply(self, dry_run=False): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) return self.execution_api.execute( devops.specialized_builds[BuildType.C4K].command(devops), dry_run ) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index dd6af5a..9b6ff1e 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -3,6 +3,7 @@ from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k from .release import Release, EnvironmentKeys +from .credentials import Credentials, CredentialMapping, GopassType from .version import Version from .build_file import BuildFileType, BuildFile from .init_service import InitService \ No newline at end of file diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index ced80c6..95121b5 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,7 +1,6 @@ import deprecation from enum import Enum from typing import List, TypedDict -import logging import deprecation @@ -16,6 +15,7 @@ class BuildType(Enum): class MixinType(Enum): RELEASE = 0 + CREDENTIALS = 1 class ReleaseType(Enum): diff --git a/src/main/python/ddadevops/domain/credentials.py b/src/main/python/ddadevops/domain/credentials.py new file mode 100644 index 0000000..c5ab6a9 --- /dev/null +++ b/src/main/python/ddadevops/domain/credentials.py @@ -0,0 +1,56 @@ +import deprecation +from enum import Enum +from typing import List, TypedDict +from inflection import underscore +import deprecation +from .common import ( + Validateable, +) + + +class GopassType(Enum): + FIELD = 0 + PASSWORD = 1 + + +class CredentialMapping(Validateable): + def __init__(self, mapping: dict): + self.name = mapping.get("name", None) + self.gopass_field = mapping.get("gopass_field", None) + self.gopass_path = mapping.get("gopass_path", None) + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("gopass_path") + if not self.name and not self.gopass_field: + result.append(f"Either name or gopass field has to be defined.") + return result + + def gopass_type(self): + if self.gopass_field: + return GopassType.FIELD + else: + return GopassType.PASSWORD + + def name_for_input(self): + if self.name: + return self.name + else: + return underscore(self.gopass_field) + + def name_for_environment(self): + return self.name_for_input().upper() + + +class Credentials(Validateable): + def __init__(self, input: dict): + input_mappings = input.get("credentials_mapping", []) + self.mappings = [] + for input_mapping in input_mappings: + self.mappings.append(CredentialMapping(input_mapping)) + + def validate(self) -> List[str]: + result = [] + for mapping in self.mappings: + result += mapping.validate() + return result diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 6cb4a6d..ae7c097 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -25,6 +25,19 @@ class DevopsFactory: mixins = {} if MixinType.RELEASE in mixin_types: mixins[MixinType.RELEASE] = Release(input, version) + if MixinType.CREDENTIALS in mixin_types: + if BuildType.C4K in build_types: + default_mappings = [ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + }, + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_password", + } + ] + mixins[MixinType.CREDENTIALS] = Credentials(input, version) devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 4083839..230c1ae 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -11,7 +11,7 @@ class Version(Validateable): snapshot_parsed = input_str.split("-") version_str = snapshot_parsed[0] suffix_str = None - if len(snapshot_parsed) > 1: + if len(snapshot_parsed) > 1: suffix_str = snapshot_parsed[1] version_no_parsed = [int(x) for x in version_str.split(".")] return cls( @@ -21,9 +21,9 @@ class Version(Validateable): ) def __init__( - self, - version_list: list, - snapshot_suffix: Optional[str] = None, + self, + version_list: list, + snapshot_suffix: Optional[str] = None, version_str: Optional[str] = None, ): self.version_list = version_list @@ -33,6 +33,9 @@ class Version(Validateable): def __eq__(self, other): return other and self.to_string() == other.to_string() + def __hash__(self) -> int: + return self.to_string().__hash__() + def is_snapshot(self): return not self.snapshot_suffix == None @@ -47,17 +50,27 @@ class Version(Validateable): result += self.__validate_is_not_empty__("version_list") if self.version_list and len(self.version_list) < 3: result += [f"version_list must have at least 3 levels."] - if self.version_list and self.version_string and self.to_string() != self.version_string: - result += [f"version_string not parsed correct. Input was {self.version_string} parsed was {self.to_string()}"] + if ( + self.version_list + and self.version_string + and self.to_string() != self.version_string + ): + result += [ + f"version_string not parsed correct. Input was {self.version_string} parsed was {self.to_string()}" + ] return result def create_bump(self, snapshot_suffix: str = None): new_version_list = self.version_list.copy() if self.is_snapshot(): - return Version(new_version_list, snapshot_suffix=self.snapshot_suffix, version_str=None) + return Version( + new_version_list, snapshot_suffix=self.snapshot_suffix, version_str=None + ) else: new_version_list[2] += 1 - return Version(new_version_list, snapshot_suffix=snapshot_suffix, version_str=None) + return Version( + new_version_list, snapshot_suffix=snapshot_suffix, version_str=None + ) def create_patch(self): new_version_list = self.version_list.copy() @@ -78,7 +91,7 @@ class Version(Validateable): def create_major(self): new_version_list = self.version_list.copy() - if self.is_snapshot() and new_version_list[2] == 0 and new_version_list[1] == 0 : + if self.is_snapshot() and new_version_list[2] == 0 and new_version_list[1] == 0: return Version(new_version_list, snapshot_suffix=None, version_str=None) else: new_version_list[2] = 0 diff --git a/src/test/python/domain/test_crededntials.py b/src/test/python/domain/test_crededntials.py new file mode 100644 index 0000000..5ca2171 --- /dev/null +++ b/src/test/python/domain/test_crededntials.py @@ -0,0 +1,107 @@ +import pytest +from pathlib import Path +from src.main.python.ddadevops.domain import ( + CredentialMapping, + Credentials, + GopassType, + MixinType, +) +from .helper import build_devops + + +def test_should_create_mapping(): + sut = CredentialMapping( + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + } + ) + assert "grafana_cloud_user" == sut.name_for_input() + assert "GRAFANA_CLOUD_USER" == sut.name_for_environment() + assert GopassType.FIELD == sut.gopass_type() + + sut = CredentialMapping( + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_password", + } + ) + assert "grafana_cloud_password" == sut.name_for_input() + assert "GRAFANA_CLOUD_PASSWORD" == sut.name_for_environment() + assert GopassType.PASSWORD == sut.gopass_type() + + +def test_should_validate_CredentialMapping(): + sut = CredentialMapping( + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + } + ) + assert sut.is_valid() + + sut = CredentialMapping( + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_user", + } + ) + assert sut.is_valid() + + sut = CredentialMapping( + { + "gopass_path": "server/meissa/grafana-cloud", + } + ) + assert not sut.is_valid() + + +def test_should_create_credentials(): + sut = Credentials( + { + "credentials_mapping": [ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + }, + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_password", + }, + ], + } + ) + + assert sut + +def test_should_validate_credentials(): + sut = Credentials( + { + "credentials_mapping": [ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + }, + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_password", + }, + ], + } + ) + assert sut.is_valid() + + sut = Credentials( + { + "credentials_mapping": [ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + }, + { + "gopass_path": "server/meissa/grafana-cloud" + }, + ], + } + ) + assert not sut.is_valid() diff --git a/src/test/python/domain/test_devops.py b/src/test/python/domain/test_devops.py index a7c081a..a4c0de0 100644 --- a/src/test/python/domain/test_devops.py +++ b/src/test/python/domain/test_devops.py @@ -6,4 +6,4 @@ from .helper import build_devops def test_devops_buildpath(): sut = build_devops({'module': "cloud", 'name': "meissa"}) - assert "../../../target/meissa/cloud" == sut.build_path() + assert "root_path/target/meissa/cloud" == sut.build_path() diff --git a/src/test/python/test_c4k_build.py b/src/test/python/test_c4k_build.py index b9471c3..a45e1d2 100644 --- a/src/test/python/test_c4k_build.py +++ b/src/test/python/test_c4k_build.py @@ -2,22 +2,20 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain import DnsRecord from src.main.python.ddadevops.c4k_build import C4kBuild, add_c4k_mixin_config -from .domain.test_helper import devops_config +from .domain.helper import devops_config def test_c4k_mixin(tmp_path): - build_dir = "build" - project_name = "testing-project" - module_name = "c4k-test" - tmp_path_str = str(tmp_path) + str_tmp_path = str(tmp_path) + project = Project(str_tmp_path, name="name") - project = Project(tmp_path_str, name=project_name) sut = C4kBuild( project, devops_config( { + "project_root_path": str_tmp_path, + "mixin_types": [], "build_types": ["C4K"], - "project_root_path": tmp_path_str, "module": "c4k-test", "c4k_config": {"a": 1, "b": 2}, "c4k_auth": {"c": 3, "d": 4}, @@ -28,7 +26,7 @@ def test_c4k_mixin(tmp_path): ) sut.initialize_build_dir() - assert sut.build_path() == f"{tmp_path_str}/target/name/c4k-test" + assert sut.build_path() == f"{str_tmp_path}/target/name/c4k-test" sut.update_runtime_config(DnsRecord("test.de", ipv6="::1")) sut.write_c4k_config() From 3cfb60a2de51117671a178c68ff8c8d2e3afe7b4 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 16 May 2023 18:13:05 +0200 Subject: [PATCH 081/173] introduce Credentials --- .../infrastructure/release_mixin/infrastructure_api.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index f65140e..9a6ea17 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -230,13 +230,4 @@ class GitApi(): def checkout(self, branch: str): return self.execution_api.execute(f'git checkout {branch}') -class EnvironmentApi(): - def __init__(self): - self.environ = environ - - def get(self, key): - return self.environ.get(key) - - def set(self, key, value): - self.environ[key] = value From b72e7e717e46802ca8e7932ed2295b1d5a211b9e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 16 May 2023 18:13:33 +0200 Subject: [PATCH 082/173] introduce Credentials --- .../python/ddadevops/domain/credentials.py | 19 ++++--- .../ddadevops/domain/credentials_service.py | 28 ++++++++++ .../python/ddadevops/domain/devops_factory.py | 2 +- .../infrastructure/infrastructure.py | 5 ++ src/test/python/domain/test_crededntials.py | 51 +++++++++++++++++-- 5 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 src/main/python/ddadevops/domain/credentials_service.py diff --git a/src/main/python/ddadevops/domain/credentials.py b/src/main/python/ddadevops/domain/credentials.py index c5ab6a9..f144394 100644 --- a/src/main/python/ddadevops/domain/credentials.py +++ b/src/main/python/ddadevops/domain/credentials.py @@ -34,23 +34,30 @@ class CredentialMapping(Validateable): def name_for_input(self): if self.name: - return self.name + result = self.name + elif self.gopass_field: + result = underscore(self.gopass_field) else: - return underscore(self.gopass_field) + result = "" + return result def name_for_environment(self): return self.name_for_input().upper() class Credentials(Validateable): - def __init__(self, input: dict): + def __init__(self, input: dict, default_mappings: list = []): input_mappings = input.get("credentials_mapping", []) - self.mappings = [] + self.mappings = {} + for input_mapping in default_mappings: + mapping = CredentialMapping(input_mapping) + self.mappings[mapping.name_for_input()] = mapping for input_mapping in input_mappings: - self.mappings.append(CredentialMapping(input_mapping)) + mapping = CredentialMapping(input_mapping) + self.mappings[mapping.name_for_input()] = mapping def validate(self) -> List[str]: result = [] - for mapping in self.mappings: + for mapping in self.mappings.values(): result += mapping.validate() return result diff --git a/src/main/python/ddadevops/domain/credentials_service.py b/src/main/python/ddadevops/domain/credentials_service.py new file mode 100644 index 0000000..67fb633 --- /dev/null +++ b/src/main/python/ddadevops/domain/credentials_service.py @@ -0,0 +1,28 @@ +from pathlib import Path +from .common import Devops, MixinType +from .devops_factory import DevopsFactory +from .version import Version +from src.main.python.ddadevops.infrastructure import ( + BuildFileRepository +) + +class CredentialsService: + def __init__(self, gopass_api, environment_api): + + @classmethod + def prod(cls): + return cls( + DevopsFactory(), + BuildFileRepository(base_dir), + ) + + def initialize(self, input: dict) -> Devops: + mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) + version = None + + if MixinType.RELEASE in mixin_types: + primary_build_file_id = input.get("release_primary_build_file", "./project.clj") + primary_build_file = self.build_file_repository.get(Path(primary_build_file_id)) + version = primary_build_file.get_version() + + return self.devops_factory.build_devops(input, version=version) diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index ae7c097..afeb58d 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -37,7 +37,7 @@ class DevopsFactory: "name": "grafana_cloud_password", } ] - mixins[MixinType.CREDENTIALS] = Credentials(input, version) + mixins[MixinType.CREDENTIALS] = Credentials(input, default_mappings) devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 1b50682..4406e8a 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -99,3 +99,8 @@ class ExecutionApi: output = execute(command, True) print(output) return output + +class EnvironmentApi(): + + def get(self, key): + return environ.get(key) diff --git a/src/test/python/domain/test_crededntials.py b/src/test/python/domain/test_crededntials.py index 5ca2171..4e817ff 100644 --- a/src/test/python/domain/test_crededntials.py +++ b/src/test/python/domain/test_crededntials.py @@ -71,8 +71,53 @@ def test_should_create_credentials(): ], } ) - assert sut + assert 2 == len(sut.mappings) + + sut = Credentials( + {}, + default_mappings=[ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + }, + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_password", + }, + ], + ) + assert sut + assert 2 == len(sut.mappings) + + sut = Credentials( + { + "credentials_mapping": [ + { + "gopass_path": "dome/path", + "gopass_field": "some-field", + }, + { + "gopass_path": "another_path", + "name": "grafana_cloud_password", + }, + ], + }, + default_mappings=[ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + }, + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_password", + }, + ], + ) + assert sut + assert 3 == len(sut.mappings) + assert sut.mappings["grafana_cloud_password"].gopass_path == "another_path" + def test_should_validate_credentials(): sut = Credentials( @@ -98,9 +143,7 @@ def test_should_validate_credentials(): "gopass_path": "server/meissa/grafana-cloud", "gopass_field": "grafana-cloud-user", }, - { - "gopass_path": "server/meissa/grafana-cloud" - }, + {"gopass_path": "server/meissa/grafana-cloud"}, ], } ) From b7bce9df6bd9eb9abc37bf59115847a2cf5c1121 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 16 May 2023 19:21:07 +0200 Subject: [PATCH 083/173] fix environmentApi refactoring --- src/main/python/ddadevops/infrastructure/__init__.py | 2 +- .../python/ddadevops/infrastructure/release_mixin/__init__.py | 2 +- src/main/python/ddadevops/infrastructure/release_mixin/repo.py | 2 +- src/main/python/ddadevops/release_mixin.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index baba117..8ba4491 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -1,2 +1,2 @@ -from .infrastructure import FileApi, ImageApi, ResourceApi, ExecutionApi, ProjectRepository +from .infrastructure import FileApi, ImageApi, ResourceApi, ExecutionApi, ProjectRepository, EnvironmentApi from .repository import DevopsRepository, BuildFileRepository \ No newline at end of file diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py b/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py index 7c50c26..b3120ff 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py @@ -1,2 +1,2 @@ -from .infrastructure_api import FileHandler, EnvironmentApi, GitApi, JsonFileHandler, GradleFileHandler, PythonFileHandler, ClojureFileHandler +from .infrastructure_api import FileHandler, GitApi, JsonFileHandler, GradleFileHandler, PythonFileHandler, ClojureFileHandler from .repo import VersionRepository, ReleaseContextRepository, ReleaseTypeRepository diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py index 2557042..011c58c 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py @@ -7,8 +7,8 @@ from src.main.python.ddadevops.domain import ( from src.main.python.ddadevops.infrastructure.release_mixin.infrastructure_api import ( FileHandler, GitApi, - EnvironmentApi, ) +from src.main.python.ddadevops.infrastructure import EnvironmentApi class VersionRepository: diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index c2d93c6..8b5a94d 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -1,6 +1,7 @@ from pybuilder.core import Project from src.main.python.ddadevops.devops_build import DevopsBuild -from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, ReleaseTypeRepository, VersionRepository, GitApi, EnvironmentApi +from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, ReleaseTypeRepository, VersionRepository, GitApi +from src.main.python.ddadevops.infrastructure import EnvironmentApi from src.main.python.ddadevops.application import PrepareReleaseService, TagAndPushReleaseService from src.main.python.ddadevops.domain import Release, EnvironmentKeys, MixinType From 1a90f2dfe2c150c6d6e80287843ce9fcc2c7852c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 16 May 2023 19:21:31 +0200 Subject: [PATCH 084/173] mv credentials to init-service because of lifecycle --- doc/architecture/Domain.md | 3 +-- src/main/python/ddadevops/domain/common.py | 1 - .../python/ddadevops/domain/devops_factory.py | 13 ------------ .../python/ddadevops/domain/init_service.py | 21 +++++++++++++++++-- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index b2838b1..9a59a4c 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -40,7 +40,7 @@ classDiagram version } class Credentials { - + <> } class CredentialMapping { name @@ -72,7 +72,6 @@ classDiagram Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" Release: mixins - Devops *-- "0..1" Credentials: mixins Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 95121b5..0ed8664 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -15,7 +15,6 @@ class BuildType(Enum): class MixinType(Enum): RELEASE = 0 - CREDENTIALS = 1 class ReleaseType(Enum): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index afeb58d..6cb4a6d 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -25,19 +25,6 @@ class DevopsFactory: mixins = {} if MixinType.RELEASE in mixin_types: mixins[MixinType.RELEASE] = Release(input, version) - if MixinType.CREDENTIALS in mixin_types: - if BuildType.C4K in build_types: - default_mappings = [ - { - "gopass_path": "server/meissa/grafana-cloud", - "gopass_field": "grafana-cloud-user", - }, - { - "gopass_path": "server/meissa/grafana-cloud", - "name": "grafana_cloud_password", - } - ] - mixins[MixinType.CREDENTIALS] = Credentials(input, default_mappings) devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index e6d690e..087641f 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -1,5 +1,6 @@ from pathlib import Path -from .common import Devops, MixinType +from .common import Devops, MixinType, BuildType +from .credentials import Credentials from .devops_factory import DevopsFactory from .version import Version from src.main.python.ddadevops.infrastructure import ( @@ -18,8 +19,10 @@ class InitService: BuildFileRepository(base_dir), ) - def initialize(self, input: dict) -> Devops: + def initialize(self, input: dict) -> Devops: + build_types = self.devops_factory.__parse_build_types__(input["build_types"]) mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) + version = None if MixinType.RELEASE in mixin_types: @@ -27,4 +30,18 @@ class InitService: primary_build_file = self.build_file_repository.get(Path(primary_build_file_id)) version = primary_build_file.get_version() + + if BuildType.C4K in build_types: + default_mappings = [ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + }, + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "grafana_cloud_password", + } + ] + credentials = Credentials(input, default_mappings) + return self.devops_factory.build_devops(input, version=version) From ca6b693a9af3ba3a0e352278a731fdb2bb10ff60 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 17 May 2023 13:43:39 +0200 Subject: [PATCH 085/173] introduce credentials api --- src/main/python/ddadevops/credential.py | 21 +++++--- .../python/ddadevops/domain/init_service.py | 54 +++++++++++++++---- .../ddadevops/infrastructure/__init__.py | 12 ++++- .../infrastructure/infrastructure.py | 20 ++++++- src/test/python/domain/helper.py | 13 +++++ src/test/python/domain/test_init_service.py | 4 +- 6 files changed, 103 insertions(+), 21 deletions(-) diff --git a/src/main/python/ddadevops/credential.py b/src/main/python/ddadevops/credential.py index 467e5f2..12c6735 100644 --- a/src/main/python/ddadevops/credential.py +++ b/src/main/python/ddadevops/credential.py @@ -1,15 +1,24 @@ +import deprecation from .python_util import execute -def gopass_field_from_path (path, field): + +@deprecation.deprecated( + deprecated_in="3.2", details="use infrastructure.CredentialsApi instead" +) +def gopass_field_from_path(path, field): credential = None if path and field: - print('get field for: ' + path + ', ' + field) - credential = execute(['gopass', 'show', path, field]) + print("get field for: " + path + ", " + field) + credential = execute(["gopass", "show", path, field]) return credential -def gopass_password_from_path (path): + +@deprecation.deprecated( + deprecated_in="3.2", details="use infrastructure.CredentialsApi instead" +) +def gopass_password_from_path(path): credential = None if path: - print('get password for: ' + path) - credential = execute(['gopass', 'show', '--password', path]) + print("get password for: " + path) + credential = execute(["gopass", "show", "--password", path]) return credential diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 087641f..5cde7a4 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -1,36 +1,38 @@ from pathlib import Path +from typing import List from .common import Devops, MixinType, BuildType -from .credentials import Credentials +from .credentials import Credentials, GopassType from .devops_factory import DevopsFactory from .version import Version from src.main.python.ddadevops.infrastructure import ( - BuildFileRepository + BuildFileRepository, + CredentialsApi, + EnvironmentApi, ) + class InitService: - def __init__(self, devops_factory, build_file_repository): + def __init__(self, devops_factory, build_file_repository, credentials_api, environment_api): self.devops_factory = devops_factory self.build_file_repository = build_file_repository + self.credentials_api = credentials_api + self.environment_api = environment_api @classmethod def prod(cls, base_dir: str): return cls( DevopsFactory(), BuildFileRepository(base_dir), + CredentialsApi(), + EnvironmentApi(), ) - + def initialize(self, input: dict) -> Devops: build_types = self.devops_factory.__parse_build_types__(input["build_types"]) mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) version = None - if MixinType.RELEASE in mixin_types: - primary_build_file_id = input.get("release_primary_build_file", "./project.clj") - primary_build_file = self.build_file_repository.get(Path(primary_build_file_id)) - version = primary_build_file.get_version() - - if BuildType.C4K in build_types: default_mappings = [ { @@ -40,8 +42,38 @@ class InitService: { "gopass_path": "server/meissa/grafana-cloud", "name": "grafana_cloud_password", - } + }, ] credentials = Credentials(input, default_mappings) + passwords = self.resolve_passwords(credentials) + + # merge passwords & input + + if MixinType.RELEASE in mixin_types: + primary_build_file_id = input.get( + "release_primary_build_file", "./project.clj" + ) + primary_build_file = self.build_file_repository.get( + Path(primary_build_file_id) + ) + version = primary_build_file.get_version() return self.devops_factory.build_devops(input, version=version) + + def resolve_passwords(self, credentials: Credentials) -> List[str]: + result = {} + for name in credentials.mappings.keys(): + mapping = credentials.mappings[name] + env_value = self.environment_api.get(mapping.name_for_environment) + if env_value: + result[name] = env_value + else: + if mapping.gopass_type == GopassType.FIELD: + result[name] = self.credentials_api.gopass_field_from_path( + mapping.gopass_path, mapping.gopass_field + ) + if mapping.gopass_type == GopassType.PASSWORD: + result[name] = self.credentials_api.gopass_password_from_path( + mapping.gopass_path + ) + return result diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index 8ba4491..037a5df 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -1,2 +1,10 @@ -from .infrastructure import FileApi, ImageApi, ResourceApi, ExecutionApi, ProjectRepository, EnvironmentApi -from .repository import DevopsRepository, BuildFileRepository \ No newline at end of file +from .infrastructure import ( + FileApi, + ImageApi, + ResourceApi, + ExecutionApi, + ProjectRepository, + EnvironmentApi, + CredentialsApi, +) +from .repository import DevopsRepository, BuildFileRepository diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 4406e8a..95773ee 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -101,6 +101,24 @@ class ExecutionApi: return output class EnvironmentApi(): - def get(self, key): return environ.get(key) + + +class CredentialsApi(): + def __init__ (self): + self.execution_api = ExecutionApi() + + def gopass_field_from_path (self, path, field): + credential = None + if path and field: + print('get field for: ' + path + ', ' + field) + credential = self.execution_api.execute(['gopass', 'show', path, field]) + return credential + + def gopass_password_from_path (elf, path): + credential = None + if path: + print('get password for: ' + path) + credential = self.execution_api.execute(['gopass', 'show', '--password', path]) + return credential diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index a661caa..55b7b86 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -49,3 +49,16 @@ class BuildFileRepositoryMock: def write(self, build_file: BuildFile): pass + +class EnvironmentApiMock(): + def get(self, key): + pass + + +class CredentialsApiMock(): + def gopass_field_from_path (self, path, field): + pass + + def gopass_password_from_path (elf, path): + pass + diff --git a/src/test/python/domain/test_init_service.py b/src/test/python/domain/test_init_service.py index 5ae5c2a..f0b1426 100644 --- a/src/test/python/domain/test_init_service.py +++ b/src/test/python/domain/test_init_service.py @@ -5,13 +5,15 @@ from src.main.python.ddadevops.domain import ( Version, MixinType, ) -from .helper import BuildFileRepositoryMock, devops_config +from .helper import BuildFileRepositoryMock, EnvironmentApiMock, CredentialsApiMock, devops_config def test_sould_load_build_file(): sut = InitService( DevopsFactory(), BuildFileRepositoryMock(), + CredentialsApiMock(), + EnvironmentApiMock(), ) assert ( Version.from_str("1.1.5-SNAPSHOT") From 57a43085c0f7c57e5b92385f300bcd974e621d95 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 18 May 2023 14:00:51 +0200 Subject: [PATCH 086/173] init service now uses password resolution --- .../python/ddadevops/domain/devops_factory.py | 4 +- .../python/ddadevops/domain/init_service.py | 19 ++++----- src/test/python/domain/helper.py | 26 +++++++++---- src/test/python/domain/test_crededntials.py | 11 ++++++ src/test/python/domain/test_init_service.py | 39 +++++++++++++++++-- 5 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 6cb4a6d..44a025a 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -32,8 +32,8 @@ class DevopsFactory: return devops - def merge(self, input, autorization, context) -> dict: - pass + def merge(self, input: dict, context: dict, authorization: dict) -> dict: + return {} | input | context | authorization def __parse_build_types__(self, build_types: List[str]) -> List[BuildType]: result = [] diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 5cde7a4..3d2efa3 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -38,19 +38,20 @@ class InitService: { "gopass_path": "server/meissa/grafana-cloud", "gopass_field": "grafana-cloud-user", + "name": "c4k_grafana_cloud_user", }, { "gopass_path": "server/meissa/grafana-cloud", - "name": "grafana_cloud_password", + "name": "c4k_grafana_cloud_password", }, ] credentials = Credentials(input, default_mappings) - passwords = self.resolve_passwords(credentials) - - # merge passwords & input + authorization = self.resolve_passwords(credentials) + merged = self.devops_factory.merge(input, {}, authorization) + if MixinType.RELEASE in mixin_types: - primary_build_file_id = input.get( + primary_build_file_id = merged.get( "release_primary_build_file", "./project.clj" ) primary_build_file = self.build_file_repository.get( @@ -58,21 +59,21 @@ class InitService: ) version = primary_build_file.get_version() - return self.devops_factory.build_devops(input, version=version) + return self.devops_factory.build_devops(merged, version=version) def resolve_passwords(self, credentials: Credentials) -> List[str]: result = {} for name in credentials.mappings.keys(): mapping = credentials.mappings[name] - env_value = self.environment_api.get(mapping.name_for_environment) + env_value = self.environment_api.get(mapping.name_for_environment()) if env_value: result[name] = env_value else: - if mapping.gopass_type == GopassType.FIELD: + if mapping.gopass_type() == GopassType.FIELD: result[name] = self.credentials_api.gopass_field_from_path( mapping.gopass_path, mapping.gopass_field ) - if mapping.gopass_type == GopassType.PASSWORD: + if mapping.gopass_type() == GopassType.PASSWORD: result[name] = self.credentials_api.gopass_password_from_path( mapping.gopass_path ) diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 55b7b86..a810036 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -24,6 +24,12 @@ def devops_config(overrides: dict) -> dict: "release_current_branch": "my_feature", "release_primary_build_file": "./package.json", "release_secondary_build_file": [], + "credentials_mappings": [ + { + "gopass_path": "a/path", + "gopass_field": "a-field", + }, + ], } input = default.copy() input.update(overrides) @@ -50,15 +56,21 @@ class BuildFileRepositoryMock: def write(self, build_file: BuildFile): pass -class EnvironmentApiMock(): + +class EnvironmentApiMock: + def __init__(self, mappings): + self.mappings = mappings + def get(self, key): - pass + return self.mappings.get(key, None) -class CredentialsApiMock(): - def gopass_field_from_path (self, path, field): - pass +class CredentialsApiMock: + def __init__(self, mappings): + self.mappings = mappings - def gopass_password_from_path (elf, path): - pass + def gopass_field_from_path(self, path, field): + return self.mappings.get(f"{path}:{field}", None) + def gopass_password_from_path(self, path): + return self.mappings.get(path, None) diff --git a/src/test/python/domain/test_crededntials.py b/src/test/python/domain/test_crededntials.py index 4e817ff..d97d3b9 100644 --- a/src/test/python/domain/test_crededntials.py +++ b/src/test/python/domain/test_crededntials.py @@ -30,6 +30,17 @@ def test_should_create_mapping(): assert "GRAFANA_CLOUD_PASSWORD" == sut.name_for_environment() assert GopassType.PASSWORD == sut.gopass_type() + sut = CredentialMapping( + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + "name": "gfc_user", + } + ) + assert "gfc_user" == sut.name_for_input() + assert "GFC_USER" == sut.name_for_environment() + assert GopassType.FIELD == sut.gopass_type() + def test_should_validate_CredentialMapping(): sut = CredentialMapping( diff --git a/src/test/python/domain/test_init_service.py b/src/test/python/domain/test_init_service.py index f0b1426..475e8fe 100644 --- a/src/test/python/domain/test_init_service.py +++ b/src/test/python/domain/test_init_service.py @@ -4,18 +4,49 @@ from src.main.python.ddadevops.domain import ( DevopsFactory, Version, MixinType, + BuildType, +) +from .helper import ( + BuildFileRepositoryMock, + EnvironmentApiMock, + CredentialsApiMock, + devops_config, ) -from .helper import BuildFileRepositoryMock, EnvironmentApiMock, CredentialsApiMock, devops_config -def test_sould_load_build_file(): +def test_should_load_build_file(): sut = InitService( DevopsFactory(), BuildFileRepositoryMock(), - CredentialsApiMock(), - EnvironmentApiMock(), + CredentialsApiMock({ + "server/meissa/grafana-cloud:grafana-cloud-user": "gopass-gfc-user", + "server/meissa/grafana-cloud": "gopass-gfc-password", + }), + EnvironmentApiMock({}), ) assert ( Version.from_str("1.1.5-SNAPSHOT") == sut.initialize(devops_config({})).mixins[MixinType.RELEASE].version ) + + +def test_should_resolve_passwords(): + sut = InitService( + DevopsFactory(), + BuildFileRepositoryMock(), + CredentialsApiMock( + { + "server/meissa/grafana-cloud:grafana-cloud-user": "gopass-gfc-user", + "server/meissa/grafana-cloud": "gopass-gfc-password", + } + ), + EnvironmentApiMock({"C4K_GRAFANA_CLOUD_USER": "env-gfc-user"}), + ) + devops = sut.initialize(devops_config({})) + c4k = devops.specialized_builds[BuildType.C4K] + assert { + "mon-auth": { + "grafana-cloud-password": "gopass-gfc-password", + "grafana-cloud-user": "env-gfc-user", + } + } == c4k.auth() From d16a022728ec120bf245e4d469eb8632c5fc2dc6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 18 May 2023 14:12:01 +0200 Subject: [PATCH 087/173] minor fixes for test image, devops, c4k green --- .../python/ddadevops/domain/init_service.py | 1 + .../infrastructure/infrastructure.py | 24 ++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 3d2efa3..d15ec34 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -32,6 +32,7 @@ class InitService: mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) version = None + default_mappings = [] if BuildType.C4K in build_types: default_mappings = [ diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 95773ee..7f69e56 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -4,13 +4,16 @@ from os import chmod from subprocess import run from pkg_resources import resource_string import yaml +from os import environ import deprecation from ..domain import Devops, Image, C4k, Release, BuildFile from ..python_util import execute + class ProjectRepository: pass + class ResourceApi: def read_resource(self, path: str) -> bytes: return resource_string(__name__, path) @@ -100,25 +103,28 @@ class ExecutionApi: print(output) return output -class EnvironmentApi(): + +class EnvironmentApi: def get(self, key): return environ.get(key) -class CredentialsApi(): - def __init__ (self): +class CredentialsApi: + def __init__(self): self.execution_api = ExecutionApi() - def gopass_field_from_path (self, path, field): + def gopass_field_from_path(self, path, field): credential = None if path and field: - print('get field for: ' + path + ', ' + field) - credential = self.execution_api.execute(['gopass', 'show', path, field]) + print("get field for: " + path + ", " + field) + credential = self.execution_api.execute(["gopass", "show", path, field]) return credential - def gopass_password_from_path (elf, path): + def gopass_password_from_path(self, path): credential = None if path: - print('get password for: ' + path) - credential = self.execution_api.execute(['gopass', 'show', '--password', path]) + print("get password for: " + path) + credential = self.execution_api.execute( + ["gopass", "show", "--password", path] + ) return credential From 9e91491542ea619d17678a7d2a3b09f44eebb6d6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 18 May 2023 14:23:08 +0200 Subject: [PATCH 088/173] clean up usage of python_util --- .../ddadevops/infrastructure/__init__.py | 1 - .../infrastructure/infrastructure.py | 31 +++++++++++-------- src/main/python/ddadevops/python_util.py | 13 ++++++-- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index 037a5df..ab663ee 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -3,7 +3,6 @@ from .infrastructure import ( ImageApi, ResourceApi, ExecutionApi, - ProjectRepository, EnvironmentApi, CredentialsApi, ) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 7f69e56..888afc4 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -1,17 +1,11 @@ from pathlib import Path from sys import stdout -from os import chmod -from subprocess import run +from os import chmod, environ from pkg_resources import resource_string import yaml -from os import environ import deprecation +from subprocess import check_output, Popen, PIPE, run from ..domain import Devops, Image, C4k, Release, BuildFile -from ..python_util import execute - - -class ProjectRepository: - pass class ResourceApi: @@ -20,15 +14,18 @@ class ResourceApi: class FileApi: + def __init__(self): + self.execution_api = ExecutionApi() + def clean_dir(self, directory: str): - execute("rm -rf " + directory, shell=True) - execute("mkdir -p " + directory, shell=True) + self.execution_api.execute("rm -rf " + directory) + self.execution_api.execute("mkdir -p " + directory) def cp_force(self, src: str, target_dir: str): - execute("cp -f " + src + "* " + target_dir, shell=True) + self.execution_api.execute("cp -f " + src + "* " + target_dir) def cp_recursive(self, src: str, target_dir: str): - execute("cp -r " + src + " " + target_dir, shell=True) + self.execution_api.execute("cp -r " + src + " " + target_dir) def write_data_to_file(self, path: Path, data: bytes): with open(path, "w", encoding="utf-8") as output_file: @@ -99,10 +96,18 @@ class ExecutionApi: if dry_run: print(command) else: - output = execute(command, True) + output = check_output(command, encoding="UTF-8", shell=True) + output = output.rstrip() print(output) return output + def execute_live(command): + process = Popen(command, stdout=PIPE) + for line in iter(process.stdout.readline, b""): + print(line.decode("utf-8"), end="") + process.stdout.close() + process.wait() + class EnvironmentApi: def get(self, key): diff --git a/src/main/python/ddadevops/python_util.py b/src/main/python/ddadevops/python_util.py index 204ee4d..e872ad6 100644 --- a/src/main/python/ddadevops/python_util.py +++ b/src/main/python/ddadevops/python_util.py @@ -1,15 +1,22 @@ from subprocess import check_output, Popen, PIPE +import deprecation + +@deprecation.deprecated(deprecated_in="3.2", details="use ExecutionApi instead") def execute(cmd, shell=False): - output = check_output(cmd, encoding='UTF-8', shell=shell) + output = check_output(cmd, encoding="UTF-8", shell=shell) return output.rstrip() + +@deprecation.deprecated(deprecated_in="3.2", details="use ExecutionApi instead") def execute_live(cmd): process = Popen(cmd, stdout=PIPE) - for line in iter(process.stdout.readline, b''): - print(line.decode('utf-8'), end='') + for line in iter(process.stdout.readline, b""): + print(line.decode("utf-8"), end="") process.stdout.close() process.wait() + +@deprecation.deprecated(deprecated_in="3.2", details="use domain.filter_none instead") def filter_none(list_to_filter): return [x for x in list_to_filter if x is not None] From 7f7878fe3656c6e92b695d4d9847b4f0ae03a406 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 18 May 2023 17:24:05 +0200 Subject: [PATCH 089/173] release_mixin now might work --- .../python/ddadevops/application/__init__.py | 2 +- .../application/release_mixin_services.py | 80 ++++++++++++------- src/main/python/ddadevops/domain/release.py | 7 +- .../ddadevops/infrastructure/__init__.py | 1 + .../infrastructure/infrastructure.py | 51 ++++++++++++ .../infrastructure/release_mixin/__init__.py | 2 +- .../release_mixin/infrastructure_api.py | 49 ------------ src/main/python/ddadevops/release_mixin.py | 36 +++------ src/test/python/test_release_mixin.py | 4 - 9 files changed, 123 insertions(+), 109 deletions(-) diff --git a/src/main/python/ddadevops/application/__init__.py b/src/main/python/ddadevops/application/__init__.py index 6159750..9f46f1a 100644 --- a/src/main/python/ddadevops/application/__init__.py +++ b/src/main/python/ddadevops/application/__init__.py @@ -1,2 +1,2 @@ from .image_build_service import ImageBuildService -from .release_mixin_services import TagAndPushReleaseService, PrepareReleaseService +from .release_mixin_services import ReleaseService diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 22c904c..655c230 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,34 +1,60 @@ -from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, VersionRepository, GitApi +from typing import Optional, List +from src.main.python.ddadevops.infrastructure import GitApi, BuildFileRepository from src.main.python.ddadevops.domain import Version, Release -class PrepareReleaseService(): - - def __init__(self): - self.git_api = GitApi() - - def __write_and_commit_version(self, release: Release, version_repository: VersionRepository, version: Version, commit_message: str): - release.is_valid() - - version_repository.write_file(version.get_version_string()) - self.git_api.add_file(version_repository.file) - self.git_api.commit(commit_message) - - def write_and_commit_release(self, release: Release, version_repository: VersionRepository): - self.__write_and_commit_version(release, version_repository, release.release_version(), commit_message=f'Release v{release.release_version().get_version_string()}') - - def write_and_commit_bump(self, release: Release, version_repository: VersionRepository): - self.__write_and_commit_version(release, version_repository, release.bump_version(), commit_message='Version bump') - -class TagAndPushReleaseService(): - - def __init__(self, git_api: GitApi): +class ReleaseService: + def __init__(self, git_api: GitApi, build_file_repository: BuildFileRepository): self.git_api = git_api + self.build_file_repository = build_file_repository - def tag_release(self, release_repo: ReleaseContextRepository): - annotation = 'v' + release_repo.get_release().version.get_version_string() - message = 'Release ' + annotation - self.git_api.tag_annotated_second_last(annotation, message) + @classmethod + def prod(cls): + return cls( + GitApi(), + BuildFileRepository(), + ) - def push_release(self): + def prepare_release(self, release: Release): + match release.release_type: + case ReleaseType.MAJOR: + version = release.version.create_major() + case ReleaseType.MINOR: + version = release.version.create_minor() + case ReleaseType.PATCH: + version = release.version.create_patch() + case ReleaseType.NONE: + return + message = f"release: {version.to_string()}" + self.__set_version_and_commit__(version, release.build_files(), message) + + def tag_bump_and_push_release(self, release: Release): + match release.release_type: + case ReleaseType.MAJOR: + release_version = release.version.create_major() + case ReleaseType.MINOR: + release_version = release.version.create_minor() + case ReleaseType.PATCH: + release_version = release.version.create_patch() + case ReleaseType.NONE: + return + bump_version = release_version.create_bump() + release_message = f"release: {release_version.to_string()}" + bump_message = f"bump version to: {bump_version.to_string()}" + self.git_api.tag_annotated(release_version.to_string(), release_message, 0) + self.__set_version_and_commit__( + bump_version, + release.build_files(), + bump_message, + ) self.git_api.push() + + def __set_version_and_commit__( + self, version: Version, build_file_ids: List[str], message: str + ): + for id in build_file_ids: + build_file = self.build_file_repository.get(id) + build_file.set_version(release_version) + self.build_file_repository.write(build_file) + self.git_api.add_file(build_file.file_path) + self.git_api.commit(message) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index b5b126b..5c36fdc 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional +from typing import Optional, List from pathlib import Path from .common import ( Validateable, @@ -49,3 +49,8 @@ class Release(Validateable): ): result.append(f"Releases are allowed only on {self.release_main_branch}") return result + + def build_files(self) -> List[str]: + result = [self.release_primary_build_file] + result += self.release_secondary_build_files + return result diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index ab663ee..a0e975e 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -5,5 +5,6 @@ from .infrastructure import ( ExecutionApi, EnvironmentApi, CredentialsApi, + GitApi, ) from .repository import DevopsRepository, BuildFileRepository diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 888afc4..9146e3c 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -133,3 +133,54 @@ class CredentialsApi: ["gopass", "show", "--password", path] ) return credential + + +class GitApi(): + + def __init__(self): + self.execution_api = ExecutionApi() + + # pylint: disable=invalid-name + def get_latest_n_commits(self, n: int): + return self.execution_api.execute( + f'git log --oneline --format="%s %b" -n {n}') + + def get_latest_commit(self): + return self.get_latest_n_commits(1) + + def tag_annotated(self, annotation: str, message: str, count: int): + return self.execution_api.execute( + f'git tag -a {annotation} -m {message} HEAD~{count}') + + def tag_annotated_second_last(self, annotation: str, message:str): + return self.tag_annotated(annotation, message, 1) + + def get_latest_tag(self): + return self.execution_api.execute('git describe --tags --abbrev=0') + + def get_current_branch(self): + return ''.join(self.execution_api.execute('git branch --show-current')).rstrip() + + def init(self, default_branch: str = "main"): + self.execution_api.execute('git init') + self.execution_api.execute(f'git checkout -b {default_branch}') + + def set_user_config(self, email: str, name: str): + self.execution_api.execute(f'git config user.email {email}') + self.execution_api.execute(f'git config user.name {name}') + + def add_file(self, file_path: Path): + return self.execution_api.execute(f'git add {file_path}') + + def add_remote(self, origin: str, url: str): + return self.execution_api.execute(f'git remote add {origin} {url}') + + def commit(self, commit_message: str): + return self.execution_api.execute( + f'git commit -m "{commit_message}"') + + def push(self): + return self.execution_api.execute('git push') + + def checkout(self, branch: str): + return self.execution_api.execute(f'git checkout {branch}') diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py b/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py index b3120ff..9d311aa 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py @@ -1,2 +1,2 @@ -from .infrastructure_api import FileHandler, GitApi, JsonFileHandler, GradleFileHandler, PythonFileHandler, ClojureFileHandler +from .infrastructure_api import FileHandler, JsonFileHandler, GradleFileHandler, PythonFileHandler, ClojureFileHandler from .repo import VersionRepository, ReleaseContextRepository, ReleaseTypeRepository diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py index 9a6ea17..0babdc1 100644 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py @@ -180,54 +180,5 @@ class ClojureFileHandler(FileHandler): clj_file.truncate() -class GitApi(): - - def __init__(self): - self.execution_api = ExecutionApi() - - # pylint: disable=invalid-name - def get_latest_n_commits(self, n: int): - return self.execution_api.execute( - f'git log --oneline --format="%s %b" -n {n}') - - def get_latest_commit(self): - return self.get_latest_n_commits(1) - - def tag_annotated(self, annotation: str, message: str, count: int): - return self.execution_api.execute( - f'git tag -a {annotation} -m {message} HEAD~{count}') - - def tag_annotated_second_last(self, annotation: str, message:str): - return self.tag_annotated(annotation, message, 1) - - def get_latest_tag(self): - return self.execution_api.execute('git describe --tags --abbrev=0') - - def get_current_branch(self): - return ''.join(self.execution_api.execute('git branch --show-current')).rstrip() - - def init(self, default_branch: str = "main"): - self.execution_api.execute('git init') - self.execution_api.execute(f'git checkout -b {default_branch}') - - def set_user_config(self, email: str, name: str): - self.execution_api.execute(f'git config user.email {email}') - self.execution_api.execute(f'git config user.name {name}') - - def add_file(self, file_path: Path): - return self.execution_api.execute(f'git add {file_path}') - - def add_remote(self, origin: str, url: str): - return self.execution_api.execute(f'git remote add {origin} {url}') - - def commit(self, commit_message: str): - return self.execution_api.execute( - f'git commit -m "{commit_message}"') - - def push(self): - return self.execution_api.execute('git push') - - def checkout(self, branch: str): - return self.execution_api.execute(f'git checkout {branch}') diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 8b5a94d..8488f61 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -1,39 +1,23 @@ from pybuilder.core import Project from src.main.python.ddadevops.devops_build import DevopsBuild -from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, ReleaseTypeRepository, VersionRepository, GitApi -from src.main.python.ddadevops.infrastructure import EnvironmentApi -from src.main.python.ddadevops.application import PrepareReleaseService, TagAndPushReleaseService -from src.main.python.ddadevops.domain import Release, EnvironmentKeys, MixinType +from src.main.python.ddadevops.application import ReleaseService +from src.main.python.ddadevops.domain import MixinType + class ReleaseMixin(DevopsBuild): def __init__(self, project: Project, input: dict): super().__init__(project, input) - devops = self.repo.get_devops(self.project) - git_api = GitApi() - self.tag_and_push_release_service = TagAndPushReleaseService(git_api) - environment_api = EnvironmentApi() + self.release_service = ReleaseService.prod() + devops = self.devops_repo.get_devops(self.project) if MixinType.RELEASE not in devops.mixins: raise ValueError(f"ReleaseMixin requires MixinType.RELEASE") - # TODO: move this to service - release = devops.mixins[MixinType.RELEASE] - env_key = EnvironmentKeys.DDADEVOPS_RELEASE_TYPE.name - environment_val_set = environment_api.get(env_key) != "" and environment_api.get(env_key) is not None - if environment_val_set: - release_type_repo = ReleaseTypeRepository.from_environment(environment_api) - else: - release_type_repo = ReleaseTypeRepository.from_git(git_api) - version_repo = VersionRepository(release.release_config_file) - self.release_repo = ReleaseContextRepository(version_repo, release_type_repo) - # Here the initialization can happen - self.prepare_release_service = PrepareReleaseService() - def prepare_release(self): devops = self.repo.get_devops(self.project) release = devops.mixins[MixinType.RELEASE] - self.prepare_release_service.write_and_commit_release(release, self.release_repo.version_repository) - self.prepare_release_service.write_and_commit_bump(release, self.release_repo.version_repository) + self.release_service.prepare_release(release) - def tag_and_push_release(self): - self.tag_and_push_release_service.tag_release(self.release_repo) - self.tag_and_push_release_service.push_release() + def tag_bump_and_push_release(self): + devops = self.repo.get_devops(self.project) + release = devops.mixins[MixinType.RELEASE] + self.release_service.tag_bump_and_push_release(release) diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py index ed5c417..8a579a7 100644 --- a/src/test/python/test_release_mixin.py +++ b/src/test/python/test_release_mixin.py @@ -3,10 +3,6 @@ from pathlib import Path from pybuilder.core import Project from src.main.python.ddadevops.release_mixin import ReleaseMixin -from src.main.python.ddadevops.infrastructure.release_mixin import ( - GitApi, - EnvironmentApi, -) from src.main.python.ddadevops.domain import Devops, Release from .resource_helper import ResourceHelper From 639815388e4b40569078dfd920ebeba17c69d2ef Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 18 May 2023 18:23:51 +0200 Subject: [PATCH 090/173] minor fixes & initialize context for release --- .../application/release_mixin_services.py | 4 +- src/main/python/ddadevops/domain/__init__.py | 2 +- .../python/ddadevops/domain/devops_factory.py | 2 +- .../python/ddadevops/domain/init_service.py | 30 ++++++- src/main/python/ddadevops/domain/release.py | 22 +++-- .../infrastructure/infrastructure.py | 35 ++++---- src/main/python/ddadevops/release_mixin.py | 2 +- src/test/python/domain/helper.py | 41 +++++++++ src/test/python/domain/test_init_service.py | 8 +- src/test/python/test_c4k_build.py | 2 +- src/test/python/test_release_mixin.py | 86 +++++++------------ 11 files changed, 138 insertions(+), 96 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 655c230..c1744d2 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -9,10 +9,10 @@ class ReleaseService: self.build_file_repository = build_file_repository @classmethod - def prod(cls): + def prod(cls, base_dir: str): return cls( GitApi(), - BuildFileRepository(), + BuildFileRepository(base_dir), ) def prepare_release(self, release: Release): diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 9b6ff1e..f55faa3 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -2,7 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k -from .release import Release, EnvironmentKeys +from .release import Release from .credentials import Credentials, CredentialMapping, GopassType from .version import Version from .build_file import BuildFileType, BuildFile diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 44a025a..f631c31 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -33,7 +33,7 @@ class DevopsFactory: return devops def merge(self, input: dict, context: dict, authorization: dict) -> dict: - return {} | input | context | authorization + return {} | context | authorization | input def __parse_build_types__(self, build_types: List[str]) -> List[BuildType]: result = [] diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index d15ec34..488c783 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -4,19 +4,22 @@ from .common import Devops, MixinType, BuildType from .credentials import Credentials, GopassType from .devops_factory import DevopsFactory from .version import Version +from .release import ReleaseType from src.main.python.ddadevops.infrastructure import ( BuildFileRepository, CredentialsApi, EnvironmentApi, + GitApi ) class InitService: - def __init__(self, devops_factory, build_file_repository, credentials_api, environment_api): + def __init__(self, devops_factory, build_file_repository, credentials_api, environment_api, git_api): self.devops_factory = devops_factory self.build_file_repository = build_file_repository self.credentials_api = credentials_api self.environment_api = environment_api + self.git_api = git_api @classmethod def prod(cls, base_dir: str): @@ -25,6 +28,7 @@ class InitService: BuildFileRepository(base_dir), CredentialsApi(), EnvironmentApi(), + GitApi(), ) def initialize(self, input: dict) -> Devops: @@ -47,9 +51,11 @@ class InitService: }, ] credentials = Credentials(input, default_mappings) - authorization = self.resolve_passwords(credentials) + authorization = self.authorization(credentials) - merged = self.devops_factory.merge(input, {}, authorization) + context = self.context() + + merged = self.devops_factory.merge(input, context, authorization) if MixinType.RELEASE in mixin_types: primary_build_file_id = merged.get( @@ -62,7 +68,23 @@ class InitService: return self.devops_factory.build_devops(merged, version=version) - def resolve_passwords(self, credentials: Credentials) -> List[str]: + def context(self) -> dict: + result = {} + + release_type = self.environment_api.get("RELEASE_TYPE") + if not release_type: + latest_commit = self.git_api.get_latest_commit() + if latest_commit in [ReleaseType.MAJOR.name, ReleaseType.MINOR.name, + ReleaseType.PATCH.name, ReleaseType.NONE.name]: + release_type = latest_commit + result["release_type"] = release_type + + result["release_current_branch"] = self.git_api.get_current_branch() + + return result + + + def authorization(self, credentials: Credentials) -> List[str]: result = {} for name in credentials.mappings.keys(): mapping = credentials.mappings[name] diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 5c36fdc..78f69ad 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -11,19 +11,19 @@ from .version import ( ) -class EnvironmentKeys(Enum): - DDADEVOPS_RELEASE_TYPE = 0 - - class Release(Validateable): def __init__(self, input: dict, version: Version): self.release_type = ReleaseType[input.get("release_type", "NONE")] self.release_main_branch = input.get("release_main_branch", "main") self.release_current_branch = input.get("release_current_branch") - self.release_primary_build_file = input.get("release_primary_build_file", "./project.clj") - self.release_secondary_build_files = input.get("release_secondary_build_files", []) + self.release_primary_build_file = input.get( + "release_primary_build_file", "./project.clj" + ) + self.release_secondary_build_files = input.get( + "release_secondary_build_files", [] + ) self.version = version - + def validate(self): result = [] result += self.__validate_is_not_empty__("release_type") @@ -34,12 +34,16 @@ class Release(Validateable): try: Path(self.release_primary_build_file) except Exception as e: - result.append(f"release_primary_build_file must be a valid path but was {e}") + result.append( + f"release_primary_build_file must be a valid path but was {e}" + ) for path in self.release_secondary_build_files: try: Path(path) except Exception as e: - result.append(f"release_secondary_build_file must be contain valid paths but was {e}") + result.append( + f"release_secondary_build_file must be contain valid paths but was {e}" + ) if self.version: result += self.version.validate() if ( diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 9146e3c..54932db 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -3,7 +3,6 @@ from sys import stdout from os import chmod, environ from pkg_resources import resource_string import yaml -import deprecation from subprocess import check_output, Popen, PIPE, run from ..domain import Devops, Image, C4k, Release, BuildFile @@ -135,52 +134,50 @@ class CredentialsApi: return credential -class GitApi(): - +class GitApi: def __init__(self): self.execution_api = ExecutionApi() # pylint: disable=invalid-name def get_latest_n_commits(self, n: int): - return self.execution_api.execute( - f'git log --oneline --format="%s %b" -n {n}') + return self.execution_api.execute(f'git log --oneline --format="%s %b" -n {n}') def get_latest_commit(self): return self.get_latest_n_commits(1) def tag_annotated(self, annotation: str, message: str, count: int): return self.execution_api.execute( - f'git tag -a {annotation} -m {message} HEAD~{count}') + f"git tag -a {annotation} -m {message} HEAD~{count}" + ) - def tag_annotated_second_last(self, annotation: str, message:str): + def tag_annotated_second_last(self, annotation: str, message: str): return self.tag_annotated(annotation, message, 1) def get_latest_tag(self): - return self.execution_api.execute('git describe --tags --abbrev=0') + return self.execution_api.execute("git describe --tags --abbrev=0") def get_current_branch(self): - return ''.join(self.execution_api.execute('git branch --show-current')).rstrip() + return "".join(self.execution_api.execute("git branch --show-current")).rstrip() def init(self, default_branch: str = "main"): - self.execution_api.execute('git init') - self.execution_api.execute(f'git checkout -b {default_branch}') + self.execution_api.execute("git init") + self.execution_api.execute(f"git checkout -b {default_branch}") def set_user_config(self, email: str, name: str): - self.execution_api.execute(f'git config user.email {email}') - self.execution_api.execute(f'git config user.name {name}') + self.execution_api.execute(f"git config user.email {email}") + self.execution_api.execute(f"git config user.name {name}") def add_file(self, file_path: Path): - return self.execution_api.execute(f'git add {file_path}') + return self.execution_api.execute(f"git add {file_path}") def add_remote(self, origin: str, url: str): - return self.execution_api.execute(f'git remote add {origin} {url}') + return self.execution_api.execute(f"git remote add {origin} {url}") def commit(self, commit_message: str): - return self.execution_api.execute( - f'git commit -m "{commit_message}"') + return self.execution_api.execute(f'git commit -m "{commit_message}"') def push(self): - return self.execution_api.execute('git push') + return self.execution_api.execute("git push") def checkout(self, branch: str): - return self.execution_api.execute(f'git checkout {branch}') + return self.execution_api.execute(f"git checkout {branch}") diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 8488f61..15ce281 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -7,7 +7,7 @@ from src.main.python.ddadevops.domain import MixinType class ReleaseMixin(DevopsBuild): def __init__(self, project: Project, input: dict): super().__init__(project, input) - self.release_service = ReleaseService.prod() + self.release_service = ReleaseService.prod(project.basedir) devops = self.devops_repo.get_devops(self.project) if MixinType.RELEASE not in devops.mixins: raise ValueError(f"ReleaseMixin requires MixinType.RELEASE") diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index a810036..3c6fdd0 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -74,3 +74,44 @@ class CredentialsApiMock: def gopass_password_from_path(self, path): return self.mappings.get(path, None) + + +class GitApiMock: + def get_latest_n_commits(self, n: int): + pass + + def get_latest_commit(self): + pass + + def tag_annotated(self, annotation: str, message: str, count: int): + pass + + def tag_annotated_second_last(self, annotation: str, message: str): + pass + + def get_latest_tag(self): + pass + + def get_current_branch(self): + pass + + def init(self, default_branch: str = "main"): + pass + + def set_user_config(self, email: str, name: str): + pass + + def add_file(self, file_path: Path): + pass + + def add_remote(self, origin: str, url: str): + pass + + def commit(self, commit_message: str): + pass + + def push(self): + pass + + def checkout(self, branch: str): + pass diff --git a/src/test/python/domain/test_init_service.py b/src/test/python/domain/test_init_service.py index 475e8fe..686ec98 100644 --- a/src/test/python/domain/test_init_service.py +++ b/src/test/python/domain/test_init_service.py @@ -10,6 +10,7 @@ from .helper import ( BuildFileRepositoryMock, EnvironmentApiMock, CredentialsApiMock, + GitApiMock, devops_config, ) @@ -23,6 +24,7 @@ def test_should_load_build_file(): "server/meissa/grafana-cloud": "gopass-gfc-password", }), EnvironmentApiMock({}), + GitApiMock(), ) assert ( Version.from_str("1.1.5-SNAPSHOT") @@ -41,8 +43,12 @@ def test_should_resolve_passwords(): } ), EnvironmentApiMock({"C4K_GRAFANA_CLOUD_USER": "env-gfc-user"}), + GitApiMock(), ) - devops = sut.initialize(devops_config({})) + config = devops_config({}) + del config["c4k_grafana_cloud_user"] + del config["c4k_grafana_cloud_password"] + devops = sut.initialize(config) c4k = devops.specialized_builds[BuildType.C4K] assert { "mon-auth": { diff --git a/src/test/python/test_c4k_build.py b/src/test/python/test_c4k_build.py index a45e1d2..3db4c35 100644 --- a/src/test/python/test_c4k_build.py +++ b/src/test/python/test_c4k_build.py @@ -5,7 +5,7 @@ from src.main.python.ddadevops.c4k_build import C4kBuild, add_c4k_mixin_config from .domain.helper import devops_config -def test_c4k_mixin(tmp_path): +def test_c4k_build(tmp_path): str_tmp_path = str(tmp_path) project = Project(str_tmp_path, name="name") diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py index 8a579a7..a266630 100644 --- a/src/test/python/test_release_mixin.py +++ b/src/test/python/test_release_mixin.py @@ -1,83 +1,55 @@ import pytest as pt +import os from pathlib import Path from pybuilder.core import Project from src.main.python.ddadevops.release_mixin import ReleaseMixin from src.main.python.ddadevops.domain import Devops, Release - -from .resource_helper import ResourceHelper -from .domain.test_helper import devops_config - -MAIN_BRANCH = "main" -STAGE = "test" -PROJECT_ROOT_PATH = "." -MODULE = "test" -BUILD_DIR_NAME = "build_dir" - - -def change_test_dir(tmp_path: Path, monkeypatch: pt.MonkeyPatch): - monkeypatch.chdir(tmp_path) - - -def initialize_with_object(project, CONFIG_FILE): - project.build_depends_on("ddadevops>=3.1.2") - build = ReleaseMixin( - project, - devops_config( - { - "name": "release_test", - "stage": "test", - "module": MODULE, - "project_root_path": PROJECT_ROOT_PATH, - "build_types": [], - "mixin_types": ["RELEASE"], - "build_dir_name": BUILD_DIR_NAME, - "release_main_branch": MAIN_BRANCH, - "release_config_file": CONFIG_FILE, - } - ), - ) - return build - +from .domain.helper import devops_config +from .resource_helper import copy_resource def test_release_mixin(tmp_path): - tmp_path_str = str(tmp_path) + str_tmp_path = str(tmp_path) + copy_resource(Path('package.json'), tmp_path) + project = Project(str_tmp_path, name="name") - project = Project(tmp_path_str, name="name") sut = ReleaseMixin( project, devops_config( { - "project_root_path": tmp_path_str, - "build_types": [], + "project_root_path": str_tmp_path, "mixin_types": ["RELEASE"], + "build_types": [], + "module": "release-test", } ), ) - assert sut is not None + + sut.initialize_build_dir() + assert sut.build_path() == f"{str_tmp_path}/target/name/release-test" -def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch): - # init - th = ResourceHelper() - th.copy_files(th.TEST_FILE_PATH, tmp_path) - th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME +# def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch): +# # init +# th = ResourceHelper() +# th.copy_files(th.TEST_FILE_PATH, tmp_path) +# th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - change_test_dir(tmp_path, monkeypatch) - project = Project(tmp_path) +# change_test_dir(tmp_path, monkeypatch) +# project = Project(tmp_path) - git_api = GitApi() - git_api.init() - git_api.set_user_config("ex.ample@mail.com", "Ex Ample") - git_api.add_file(th.TEST_FILE_NAME) - git_api.commit("MAJOR release") +# git_api = GitApi() +# git_api.init() +# git_api.set_user_config("ex.ample@mail.com", "Ex Ample") +# git_api.add_file(th.TEST_FILE_NAME) +# git_api.commit("MAJOR release") - build = initialize_with_object(project, th.TEST_FILE_PATH) - build.prepare_release() - release_version = build.release_repo.version_repository.get_version() +# build = initialize_with_object(project, th.TEST_FILE_PATH) +# build.prepare_release() +# release_version = build.release_repo.version_repository.get_version() - # test - assert "124.0.1-SNAPSHOT" in release_version.get_version_string() +# # test +# assert "124.0.1-SNAPSHOT" in release_version.get_version_string() # def test_release_mixin_environment(tmp_path: Path, monkeypatch: pt.MonkeyPatch): From 8aec1be4a50e5c4677912cf8107baeba7a111841 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 18 May 2023 18:24:02 +0200 Subject: [PATCH 091/173] no longer needed --- .../infrastructure/release_mixin/__init__.py | 2 - .../release_mixin/infrastructure_api.py | 184 ------------------ .../infrastructure/release_mixin/repo.py | 125 ------------ src/test/python/release_mixin/__init__.py | 2 - src/test/python/release_mixin/helper.py | 12 -- .../release_mixin/mock_infrastructure.py | 43 ---- .../release_mixin/mock_infrastructure_api.py | 73 ------- .../release_mixin/test_infrastructure.py | 86 -------- .../release_mixin/test_infrastructure_api.py | 103 ---------- .../python/release_mixin/test_services.py | 32 --- 10 files changed, 662 deletions(-) delete mode 100644 src/main/python/ddadevops/infrastructure/release_mixin/__init__.py delete mode 100644 src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py delete mode 100644 src/main/python/ddadevops/infrastructure/release_mixin/repo.py delete mode 100644 src/test/python/release_mixin/__init__.py delete mode 100644 src/test/python/release_mixin/helper.py delete mode 100644 src/test/python/release_mixin/mock_infrastructure.py delete mode 100644 src/test/python/release_mixin/mock_infrastructure_api.py delete mode 100644 src/test/python/release_mixin/test_infrastructure.py delete mode 100644 src/test/python/release_mixin/test_infrastructure_api.py delete mode 100644 src/test/python/release_mixin/test_services.py diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py b/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py deleted file mode 100644 index 9d311aa..0000000 --- a/src/main/python/ddadevops/infrastructure/release_mixin/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .infrastructure_api import FileHandler, JsonFileHandler, GradleFileHandler, PythonFileHandler, ClojureFileHandler -from .repo import VersionRepository, ReleaseContextRepository, ReleaseTypeRepository diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py b/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py deleted file mode 100644 index 0babdc1..0000000 --- a/src/main/python/ddadevops/infrastructure/release_mixin/infrastructure_api.py +++ /dev/null @@ -1,184 +0,0 @@ -import json -import re -from abc import ABC, abstractmethod -from typing import Optional -from pathlib import Path -from os import environ -from ..infrastructure import ExecutionApi - -# TODO: jem, zam - 2023_04_18: Discuss if we can move more functionality to domain? -class FileHandler(ABC): - def __init__(self) -> None: - self.config_file_path: Optional[Path | None] = None - self.config_file_type: Optional[Path | None] = None - - @classmethod - def from_file_path(cls, file_path): - config_file_type = file_path.suffix - match config_file_type: - case '.json': - file_handler = JsonFileHandler() - case '.gradle': - file_handler = GradleFileHandler() - case '.clj': - file_handler = ClojureFileHandler() - case '.py': - file_handler = PythonFileHandler() - case _: - raise RuntimeError( - f'The file type "{config_file_type}" is not implemented') - # TODO: Attribute is only set in classmethod. Should this be initialized outside of this class? - file_handler.config_file_path = file_path - file_handler.config_file_type = config_file_type - return file_handler - - @abstractmethod - def parse(self) -> tuple[list[int], bool]: - pass - - @abstractmethod - def write(self, version_string): - pass - - -class JsonFileHandler(FileHandler): - - def parse(self) -> tuple[list[int], bool]: - if self.config_file_path is None: - raise ValueError("No file name given.") - with open(self.config_file_path, 'r', encoding='utf-8') as json_file: - json_version = json.load(json_file)['version'] - is_snapshot = False - if '-SNAPSHOT' in json_version: - is_snapshot = True - json_version = json_version.replace('-SNAPSHOT', '') - version = [int(x) for x in json_version.split('.')] - return version, is_snapshot - - def write(self, version_string): - with open(self.config_file_path, 'r+', encoding='utf-8') as json_file: - json_data = json.load(json_file) - json_data['version'] = version_string - json_file.seek(0) - json.dump(json_data, json_file, indent=4) - json_file.truncate() - - -class GradleFileHandler(FileHandler): - - def parse(self) -> tuple[list[int], bool]: - if self.config_file_path is None: - raise ValueError("No file name given.") - with open(self.config_file_path, 'r', encoding='utf-8') as gradle_file: - contents = gradle_file.read() - version_line = re.search("\nversion = .*", contents) - exception = Exception("Version not found in gradle file") - if version_line is None: - raise exception - - version_line_group = version_line.group() - version_string = re.search( - '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) - if version_string is None: - raise exception - - version_string_group = version_string.group() - is_snapshot = False - if '-SNAPSHOT' in version_string_group: - is_snapshot = True - version_string_group = version_string_group.replace('-SNAPSHOT', '') - - version = [int(x) for x in version_string_group.split('.')] - - return version, is_snapshot - - def write(self, version_string): - with open(self.config_file_path, 'r+', encoding='utf-8') as gradle_file: - contents = gradle_file.read() - version_substitute = re.sub( - '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents) - gradle_file.seek(0) - gradle_file.write(version_substitute) - gradle_file.truncate() - - -class PythonFileHandler(FileHandler): - - def parse(self) -> tuple[list[int], bool]: - if self.config_file_path is None: - raise ValueError("No file name given.") - with open(self.config_file_path, 'r', encoding='utf-8') as python_file: - contents = python_file.read() - version_line = re.search("\nversion = .*\n", contents) - exception = Exception("Version not found in gradle file") - if version_line is None: - raise exception - - version_line_group = version_line.group() - version_string = re.search( - '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) - if version_string is None: - raise exception - - version_string_group = version_string.group() - is_snapshot = False - if '-SNAPSHOT' in version_string_group: - is_snapshot = True - version_string_group = version_string_group.replace('-SNAPSHOT', '') - - version = [int(x) for x in version_string_group.split('.')] - - return version, is_snapshot - - def write(self, version_string): - with open(self.config_file_path, 'r+', encoding='utf-8') as python_file: - contents = python_file.read() - version_substitute = re.sub( - '\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents) - python_file.seek(0) - python_file.write(version_substitute) - python_file.truncate() - - -class ClojureFileHandler(FileHandler): - - def parse(self) -> tuple[list[int], bool]: - if self.config_file_path is None: - raise ValueError("No file name given.") - with open(self.config_file_path, 'r', encoding='utf-8') as clj_file: - contents = clj_file.read() - version_line = re.search("^\\(defproject .*\n", contents) - exception = Exception("Version not found in clj file") - if version_line is None: - raise exception - - version_line_group = version_line.group() - version_string = re.search( - '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group) - if version_string is None: - raise exception - - version_string_group = version_string.group() - is_snapshot = False - if '-SNAPSHOT' in version_string_group: - is_snapshot = True - version_string_group = version_string_group.replace('-SNAPSHOT', '') - - version = [int(x) for x in version_string_group.split('.')] - - return version, is_snapshot - - def write(self, version_string): - with open(self.config_file_path, 'r+', encoding='utf-8') as clj_file: - clj_first = clj_file.readline() - clj_rest = clj_file.read() - version_substitute = re.sub( - '[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', f'"{version_string}"\n', clj_first) - clj_file.seek(0) - clj_file.write(version_substitute) - clj_file.write(clj_rest) - clj_file.truncate() - - - - diff --git a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py b/src/main/python/ddadevops/infrastructure/release_mixin/repo.py deleted file mode 100644 index 011c58c..0000000 --- a/src/main/python/ddadevops/infrastructure/release_mixin/repo.py +++ /dev/null @@ -1,125 +0,0 @@ - -from src.main.python.ddadevops.domain import ( - Version, - ReleaseType, - EnvironmentKeys, -) -from src.main.python.ddadevops.infrastructure.release_mixin.infrastructure_api import ( - FileHandler, - GitApi, -) -from src.main.python.ddadevops.infrastructure import EnvironmentApi - - -class VersionRepository: - def __init__(self, file): - self.file = file - self.file_handler = None - - def load_file(self): - self.file_handler = FileHandler.from_file_path(self.file) - return self.file_handler - - def write_file(self, version_string): - if self.file_handler is None: - raise RuntimeError("Version was not created by load_file method.") - self.file_handler.write(version_string) - - def parse_file(self): - version_list, is_snapshot = self.file_handler.parse() - return version_list, is_snapshot - - def get_version(self) -> Version: - self.file_handler = self.load_file() - version_list, is_snapshot = self.parse_file() - version = Version(self.file, version_list) - version.is_snapshot = is_snapshot - - return version - - -class ReleaseTypeRepository: - def __init__( - self, - git_api: GitApi = GitApi(), - environment_api: EnvironmentApi = EnvironmentApi(), - ): - self.git_api: GitApi = git_api - self.environment_api: EnvironmentApi = environment_api - self.get_from_git: bool = False - self.get_from_env: bool = False - - @classmethod - def from_git(cls, git_api: GitApi): - release_type_repo = cls(git_api=git_api) - release_type_repo.get_from_git = True - return release_type_repo - - @classmethod - def from_environment(cls, environment_api: EnvironmentApi): - release_type_repo = cls(environment_api=environment_api) - release_type_repo.get_from_env = True - return release_type_repo - - def __get_release_type_git(self) -> ReleaseType | None: - latest_commit = self.git_api.get_latest_commit() - - match latest_commit.upper(): - case ReleaseType.MAJOR.name: - return ReleaseType.MAJOR - case ReleaseType.MINOR.name: - return ReleaseType.MINOR - case ReleaseType.PATCH.name: - return ReleaseType.PATCH - case ReleaseType.SNAPSHOT.name: - return ReleaseType.SNAPSHOT - case _: - return None - - def __get_release_type_environment(self) -> ReleaseType | None: - release_name = self.environment_api.get( - EnvironmentKeys.DDADEVOPS_RELEASE_TYPE.name - ) - - if release_name is None: - raise ValueError( - "Release Name not found. Is the Environment correctly configured?" - ) - - match release_name.upper(): - case ReleaseType.MAJOR.name: - return ReleaseType.MAJOR - case ReleaseType.MINOR.name: - return ReleaseType.MINOR - case ReleaseType.PATCH.name: - return ReleaseType.PATCH - case ReleaseType.SNAPSHOT.name: - return ReleaseType.SNAPSHOT - case _: - return None - - def get_release_type(self) -> ReleaseType | None: - if self.get_from_git: - return self.__get_release_type_git() - if self.get_from_env: - return self.__get_release_type_environment() - raise ValueError("No valid api passed to ReleaseTypeRepository") - - -class ReleaseContextRepository: - def __init__( - self, - version_repository: VersionRepository, - release_type_repository: ReleaseTypeRepository, - ): - self.version_repository = version_repository - self.release_type_repository = release_type_repository - - # def get_release(self, main_branch: str) -> ReleaseContext: - # result = ReleaseContext( - # self.release_type_repository.get_release_type(), - # self.version_repository.get_version(), - # main_branch, - # ) - # result.throw_if_invalid() - # return result diff --git a/src/test/python/release_mixin/__init__.py b/src/test/python/release_mixin/__init__.py deleted file mode 100644 index 1514703..0000000 --- a/src/test/python/release_mixin/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .mock_infrastructure import MockReleaseRepository, MockReleaseTypeRepository, MockVersionRepository -from .mock_infrastructure_api import MockGitApi diff --git a/src/test/python/release_mixin/helper.py b/src/test/python/release_mixin/helper.py deleted file mode 100644 index c9843ca..0000000 --- a/src/test/python/release_mixin/helper.py +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path -from src.main.python.ddadevops.infrastructure import ExecutionApi - -class Helper(): - def __init__(self, file_name = 'config.json'): - self.TEST_FILE_NAME = file_name - self.TEST_FILE_ROOT = Path('src/test/resources/') - self.TEST_FILE_PATH = self.TEST_FILE_ROOT / self.TEST_FILE_NAME - - def copy_files(self, source: Path, target: Path): - api = ExecutionApi() - api.execute(f"cp {source} {target}") diff --git a/src/test/python/release_mixin/mock_infrastructure.py b/src/test/python/release_mixin/mock_infrastructure.py deleted file mode 100644 index 9cf6b06..0000000 --- a/src/test/python/release_mixin/mock_infrastructure.py +++ /dev/null @@ -1,43 +0,0 @@ -from pathlib import Path - -from src.main.python.ddadevops.domain import ReleaseType, Version, Release - -from .mock_infrastructure_api import MockGitApi - -class MockVersionRepository(): - - def __init__(self): - self.file = None - self.file_handler = None - self.write_file_count = 0 - - def load_file(self): - pass - - def write_file(self, version_string): - self.write_file_count += 1 - pass - - def parse_file(self): - pass - - def get_version(self) -> Version: - return Version(Path(), [0,0,0]) - -class MockReleaseTypeRepository(): - def __init__(self, mock_git_api: MockGitApi): - self.git_api = mock_git_api - - def get_release_type(self): - return ReleaseType.MINOR - -class MockReleaseRepository(): - def __init__(self, version_repository: MockVersionRepository, release_type_repository: MockReleaseTypeRepository, main_branch: str): - self.version_repository = version_repository - self.release_type_repository = release_type_repository - self.main_branch = main_branch - self.get_release_count = 0 - - def get_release(self) -> Release: - self.get_release_count += 1 - return Release(self.release_type_repository.get_release_type(), self.version_repository.get_version(), self.main_branch) diff --git a/src/test/python/release_mixin/mock_infrastructure_api.py b/src/test/python/release_mixin/mock_infrastructure_api.py deleted file mode 100644 index 368203b..0000000 --- a/src/test/python/release_mixin/mock_infrastructure_api.py +++ /dev/null @@ -1,73 +0,0 @@ -class MockSystemApi(): - - def __init__(self): - self.stdout = [""] - self.stderr = [""] - - def run(self, args): - pass - - def run_checked(self, *args): - self.run(args) - pass - -class MockGitApi(): - - def __init__(self, commit_string = ""): - self.system_api = MockSystemApi() - self.get_latest_commit_count = 0 - self.commit_string = commit_string - self.tag_annotated_count = 0 - self.add_file_count = 0 - self.commit_count = 0 - self.push_count = 0 - - def get_latest_n_commits(self, n: int): - return " " - - def get_latest_commit(self): - self.get_latest_commit_count += 1 - return self.commit_string - - def tag_annotated(self, annotation: str, message: str, count: int): - self.tag_annotated_count += 1 - return " " - - def tag_annotated_second_last(self, annotation: str, message: str): - self.tag_annotated(annotation, message, 1) - return " " - - def get_latest_tag(self): - return " " - - def get_current_branch(self): - return " " - - def init(self): - pass - - def add_file(self, file_path): - self.add_file_count += 1 - return " " - - def commit(self, commit_message: str): - self.commit_count += 1 - return commit_message - - def push(self): - self.push_count += 1 - return " " - - def checkout(self, branch: str): - return " " - -class MockEnvironmentApi(): - - def __init__(self, environ_map): - self.environ = environ_map - - def get(self, name): - return self.environ.get(name) - - def set(self, name, value): - self.environ[name] = value diff --git a/src/test/python/release_mixin/test_infrastructure.py b/src/test/python/release_mixin/test_infrastructure.py deleted file mode 100644 index 6233c64..0000000 --- a/src/test/python/release_mixin/test_infrastructure.py +++ /dev/null @@ -1,86 +0,0 @@ -from src.main.python.ddadevops.domain import ReleaseType -from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseTypeRepository, VersionRepository, ReleaseContextRepository -from .mock_infrastructure_api import MockGitApi, MockEnvironmentApi -from .helper import Helper - -def test_version_repository(tmp_path): - # init - th = Helper() - th.copy_files(th.TEST_FILE_PATH, tmp_path) - sut = VersionRepository(th.TEST_FILE_PATH) - version = sut.get_version() - - #test - assert version is not None - - -def test_release_repository(tmp_path): - # init - th = Helper() - th.copy_files( th.TEST_FILE_PATH, tmp_path) - version_repo = VersionRepository(th.TEST_FILE_PATH) - release_type_repo = ReleaseTypeRepository.from_git(MockGitApi('MINOR test')) - - # test - sut = ReleaseContextRepository(version_repo, release_type_repo) - - release = sut.get_release('main') - - assert release is not None - - -def test_release_type_repository_git(): - sut = ReleaseTypeRepository.from_git(MockGitApi('MINOR test')) - release_type = sut.get_release_type() - assert release_type is ReleaseType.MINOR - - sut = ReleaseTypeRepository.from_git(MockGitApi('MINOR bla')) - release_type = sut.get_release_type() - assert release_type is ReleaseType.MINOR - - sut = ReleaseTypeRepository.from_git(MockGitApi('Major bla')) - release_type = sut.get_release_type() - assert release_type == ReleaseType.MAJOR - - sut = ReleaseTypeRepository.from_git(MockGitApi('PATCH bla')) - release_type = sut.get_release_type() - assert release_type == ReleaseType.PATCH - - sut = ReleaseTypeRepository.from_git(MockGitApi('SNAPSHOT bla')) - release_type = sut.get_release_type() - assert release_type == ReleaseType.SNAPSHOT - - sut = ReleaseTypeRepository.from_git(MockGitApi('bla')) - release_type = sut.get_release_type() - assert release_type == None - -def test_release_type_repository_env(): - sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'MINOR test'})) - release_type = sut.get_release_type() - assert release_type is ReleaseType.MINOR - - sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'MINOR'})) - release_type = sut.get_release_type() - assert release_type is ReleaseType.MINOR - - sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Major bla'})) - release_type = sut.get_release_type() - assert release_type == ReleaseType.MAJOR - - sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Patch bla'})) - release_type = sut.get_release_type() - assert release_type == ReleaseType.PATCH - - sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Snapshot bla'})) - release_type = sut.get_release_type() - assert release_type == ReleaseType.SNAPSHOT - - sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Random text'})) - release_type = sut.get_release_type() - assert release_type == None - - sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'REL_TYPE': 'Not the right variable'})) - try: - release_type = sut.get_release_type() - except: - assert release_type == None diff --git a/src/test/python/release_mixin/test_infrastructure_api.py b/src/test/python/release_mixin/test_infrastructure_api.py deleted file mode 100644 index f2ba909..0000000 --- a/src/test/python/release_mixin/test_infrastructure_api.py +++ /dev/null @@ -1,103 +0,0 @@ -from pathlib import Path -import pytest as pt - -from src.main.python.ddadevops.infrastructure.release_mixin import GitApi, EnvironmentApi, JsonFileHandler -from src.main.python.ddadevops.infrastructure.release_mixin import VersionRepository -from src.main.python.ddadevops.domain.release import ReleaseType - -from .helper import Helper - -def change_test_dir( tmp_path: Path, monkeypatch: pt.MonkeyPatch): - monkeypatch.chdir(tmp_path) - -def test_environment_api(): - # init - env_api = EnvironmentApi() - key = "TEST_ENV_KEY" - value = "data" - env_api.set(key, value) - - #test - assert env_api.get(key) == value - -def test_git_api(tmp_path: Path, monkeypatch: pt.MonkeyPatch): - # init - th = Helper() - th.copy_files(th.TEST_FILE_PATH, tmp_path) - - # change the context of the script execution to tmp_path - change_test_dir(tmp_path, monkeypatch) - - git_api = GitApi() - git_api.init() - git_api.set_user_config("ex.ample@mail.com", "Ex Ample") - git_api.add_file(th.TEST_FILE_NAME) - git_api.commit("MINOR release") - - # test - latest_commit = git_api.get_latest_commit() - assert "MINOR release" in latest_commit - -# file handler tests -def test_gradle(tmp_path): - # init - th = Helper('config.gradle') - th.copy_files(th.TEST_FILE_PATH, tmp_path) - th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - - # test - repo = VersionRepository(th.TEST_FILE_PATH) - version = repo.get_version() - version = version.create_release_version(ReleaseType.SNAPSHOT) - repo.write_file(version.get_version_string()) - - # check - assert 'version = "12.4.678-SNAPSHOT"' in th.TEST_FILE_PATH.read_text() - - -def test_json(tmp_path): - # init - th = Helper('config.json') - th.copy_files(th.TEST_FILE_PATH, tmp_path) - th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - - # test - repo = VersionRepository(th.TEST_FILE_PATH) - version = repo.get_version() - version = version.create_release_version(ReleaseType.SNAPSHOT) - repo.write_file(version.get_version_string()) - - # check - assert '"version": "123.123.456-SNAPSHOT"' in th.TEST_FILE_PATH.read_text() - - -def test_clojure(tmp_path): - # init - th = Helper('config.clj') - th.copy_files(th.TEST_FILE_PATH, tmp_path) - th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - - # test - repo = VersionRepository(th.TEST_FILE_PATH) - version = repo.get_version() - version = version.create_release_version(ReleaseType.SNAPSHOT) - repo.write_file(version.get_version_string()) - - # check - assert '1.1.3-SNAPSHOT' in th.TEST_FILE_PATH.read_text() - - -def test_python(tmp_path): - # init - th = Helper('config.py') - th.copy_files(th.TEST_FILE_PATH, tmp_path) - th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - - # test - repo = VersionRepository(th.TEST_FILE_PATH) - version = repo.get_version() - version = version.create_release_version(ReleaseType.SNAPSHOT) - repo.write_file(version.get_version_string()) - - # check - assert '3.1.3-SNAPSHOT' in th.TEST_FILE_PATH.read_text() diff --git a/src/test/python/release_mixin/test_services.py b/src/test/python/release_mixin/test_services.py deleted file mode 100644 index 35063d8..0000000 --- a/src/test/python/release_mixin/test_services.py +++ /dev/null @@ -1,32 +0,0 @@ -from src.main.python.ddadevops.application import PrepareReleaseService, TagAndPushReleaseService -from src.test.python.release_mixin import MockReleaseRepository, MockReleaseTypeRepository, MockVersionRepository -from src.test.python.release_mixin import MockGitApi - -def test_prepare_release_service(): - # init - mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi()), 'main') - prepare_release_service = PrepareReleaseService() - prepare_release_service.git_api = MockGitApi() - prepare_release_service.write_and_commit_release(mock_release_repo.get_release(), mock_release_repo.version_repository) - - #test - assert prepare_release_service.git_api.add_file_count == 1 - assert prepare_release_service.git_api.commit_count == 1 - - # init - prepare_release_service.write_and_commit_bump(mock_release_repo.get_release(), mock_release_repo.version_repository) - - # test - assert prepare_release_service.git_api.add_file_count == 2 - assert prepare_release_service.git_api.commit_count == 2 - -def test_tag_and_push_release_service(): - # init - mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi()), 'main') - tag_and_push_release_service = TagAndPushReleaseService(MockGitApi()) - tag_and_push_release_service.tag_release(mock_release_repo) - tag_and_push_release_service.push_release() - - #test - assert tag_and_push_release_service.git_api.tag_annotated_count == 1 - assert tag_and_push_release_service.git_api.push_count == 1 From 1ea4a5aa3f1fe05f0651b79ddba83ca0d0beaa2f Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 12:17:48 +0200 Subject: [PATCH 092/173] first bugfixes --- build.py | 2 +- infrastructure/clojure/build.py | 48 ++++++++++--------- src/main/python/ddadevops/__init__.py | 1 - .../application/image_build_service.py | 4 +- .../application/release_mixin_services.py | 4 +- .../ddadevops/domain/credentials_service.py | 2 +- .../python/ddadevops/domain/init_service.py | 2 +- src/main/python/ddadevops/release_mixin.py | 6 +-- 8 files changed, 36 insertions(+), 33 deletions(-) diff --git a/build.py b/build.py index b767497..2f0038e 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev16" +version = "4.0.0-dev19" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/infrastructure/clojure/build.py b/infrastructure/clojure/build.py index 515334b..2da9dc6 100644 --- a/infrastructure/clojure/build.py +++ b/infrastructure/clojure/build.py @@ -9,32 +9,36 @@ PROJECT_ROOT_PATH = "../.." @init def initialize(project): - project.build_depends_on("ddadevops>=4.0.0-dev") - stage = "notused" - dockerhub_user = environ.get("DOCKERHUB_USER") - if not dockerhub_user: - dockerhub_user = gopass_field_from_path("meissa/web/docker.com", "login") - dockerhub_password = environ.get("DOCKERHUB_PASSWORD") - if not dockerhub_password: - dockerhub_password = gopass_password_from_path("meissa/web/docker.com") tag = environ.get("CI_COMMIT_TAG") if not tag: tag = get_tag_from_latest_commit() + + input = { + "name": name, + "module": MODULE, + "stage": "notused", + "project_root_path": PROJECT_ROOT_PATH, + "build_types": ["IMAGE"], + "mixin_types": [], + "image_dockerhub_user": "dockerhub_user", + "image_dockerhub_password": "dockerhub_password", + "image_tag": tag, + "credentials_mappings": [ + { + "gopass_path": "meissa/web/docker.com", + "gopass_field": "login", + "name": "image_dockerhub_user" + }, + { + "gopass_path": "meissa/web/docker.com", + "name": "image_dockerhub_password" + }, + ], + } - devops = Devops( - stage=stage, - project_root_path=PROJECT_ROOT_PATH, - module=MODULE, - name=name, - ) - image = Image( - dockerhub_user=dockerhub_user, - dockerhub_password=dockerhub_password, - docker_publish_tag=tag, - devops=devops, - ) - - build = DevopsImageBuild(project, image=image) + project.build_depends_on("ddadevops>=4.0.0-dev") + + build = DevopsImageBuild(project, input) build.initialize_build_dir() diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index 201a228..e73b38f 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -19,6 +19,5 @@ from .devops_terraform_build import DevopsTerraformBuild, create_devops_terrafor from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build from .credential import gopass_password_from_path, gopass_field_from_path from .release_mixin import ReleaseMixin -from .domain import Validateable, DnsRecord, Devops, Image, Release, Version __version__ = "${version}" diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index 33a3d2d..474be9a 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -1,5 +1,5 @@ -from src.main.python.ddadevops.domain import Image, Devops, BuildType -from src.main.python.ddadevops.infrastructure import FileApi, ResourceApi, ImageApi +from ..domain import Image, Devops, BuildType +from ..infrastructure import FileApi, ResourceApi, ImageApi class ImageBuildService: diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index c1744d2..b35a6bc 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,6 +1,6 @@ from typing import Optional, List -from src.main.python.ddadevops.infrastructure import GitApi, BuildFileRepository -from src.main.python.ddadevops.domain import Version, Release +from ..infrastructure import GitApi, BuildFileRepository +from ..domain import Version, Release class ReleaseService: diff --git a/src/main/python/ddadevops/domain/credentials_service.py b/src/main/python/ddadevops/domain/credentials_service.py index 67fb633..39947f6 100644 --- a/src/main/python/ddadevops/domain/credentials_service.py +++ b/src/main/python/ddadevops/domain/credentials_service.py @@ -2,7 +2,7 @@ from pathlib import Path from .common import Devops, MixinType from .devops_factory import DevopsFactory from .version import Version -from src.main.python.ddadevops.infrastructure import ( +from .infrastructure import ( BuildFileRepository ) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 488c783..984fe9e 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -5,7 +5,7 @@ from .credentials import Credentials, GopassType from .devops_factory import DevopsFactory from .version import Version from .release import ReleaseType -from src.main.python.ddadevops.infrastructure import ( +from ..infrastructure import ( BuildFileRepository, CredentialsApi, EnvironmentApi, diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 15ce281..acdcb6a 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -1,7 +1,7 @@ from pybuilder.core import Project -from src.main.python.ddadevops.devops_build import DevopsBuild -from src.main.python.ddadevops.application import ReleaseService -from src.main.python.ddadevops.domain import MixinType +from .devops_build import DevopsBuild +from .application import ReleaseService +from .domain import MixinType class ReleaseMixin(DevopsBuild): From 807739ff359993ba4c7d554bbaafb5dcab47df17 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 May 2023 12:24:31 +0200 Subject: [PATCH 093/173] Get image from devops object --- src/main/python/ddadevops/application/image_build_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index 474be9a..5fe398c 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -28,6 +28,7 @@ class ImageBuildService: ) def __copy_build_resources_from_dir__(self, devops: Devops): + image = devops.specialized_builds[BuildType.C4K] self.file_api.cp_force( image.build_commons_path(), devops.build_path() ) From de1bd0570b0d0dd8c76de77c33a7a3bd78fe2451 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 May 2023 12:25:21 +0200 Subject: [PATCH 094/173] Resolve warnings in release_mixin_service --- .../python/ddadevops/application/release_mixin_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index b35a6bc..8b2a015 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,6 +1,6 @@ from typing import Optional, List from ..infrastructure import GitApi, BuildFileRepository -from ..domain import Version, Release +from ..domain import Version, Release, ReleaseType class ReleaseService: @@ -54,7 +54,7 @@ class ReleaseService: ): for id in build_file_ids: build_file = self.build_file_repository.get(id) - build_file.set_version(release_version) + build_file.set_version(version) self.build_file_repository.write(build_file) self.git_api.add_file(build_file.file_path) self.git_api.commit(message) From db0d560e127cb55c553390470a05b0be68ca702c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 13:39:49 +0200 Subject: [PATCH 095/173] gopass in question --- build.py | 2 +- infrastructure/clojure/build.py | 19 +---- .../python/ddadevops/domain/init_service.py | 77 ++++++++++++------- 3 files changed, 53 insertions(+), 45 deletions(-) diff --git a/build.py b/build.py index 2f0038e..5fcf676 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev19" +version = "4.0.0-dev21" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/infrastructure/clojure/build.py b/infrastructure/clojure/build.py index 2da9dc6..846119c 100644 --- a/infrastructure/clojure/build.py +++ b/infrastructure/clojure/build.py @@ -9,10 +9,7 @@ PROJECT_ROOT_PATH = "../.." @init def initialize(project): - tag = environ.get("CI_COMMIT_TAG") - if not tag: - tag = get_tag_from_latest_commit() - + input = { "name": name, "module": MODULE, @@ -20,20 +17,6 @@ def initialize(project): "project_root_path": PROJECT_ROOT_PATH, "build_types": ["IMAGE"], "mixin_types": [], - "image_dockerhub_user": "dockerhub_user", - "image_dockerhub_password": "dockerhub_password", - "image_tag": tag, - "credentials_mappings": [ - { - "gopass_path": "meissa/web/docker.com", - "gopass_field": "login", - "name": "image_dockerhub_user" - }, - { - "gopass_path": "meissa/web/docker.com", - "name": "image_dockerhub_password" - }, - ], } project.build_depends_on("ddadevops>=4.0.0-dev") diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 984fe9e..3d36e36 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -5,16 +5,18 @@ from .credentials import Credentials, GopassType from .devops_factory import DevopsFactory from .version import Version from .release import ReleaseType -from ..infrastructure import ( - BuildFileRepository, - CredentialsApi, - EnvironmentApi, - GitApi -) +from ..infrastructure import BuildFileRepository, CredentialsApi, EnvironmentApi, GitApi class InitService: - def __init__(self, devops_factory, build_file_repository, credentials_api, environment_api, git_api): + def __init__( + self, + devops_factory, + build_file_repository, + credentials_api, + environment_api, + git_api, + ): self.devops_factory = devops_factory self.build_file_repository = build_file_repository self.credentials_api = credentials_api @@ -39,7 +41,7 @@ class InitService: default_mappings = [] if BuildType.C4K in build_types: - default_mappings = [ + default_mappings += [ { "gopass_path": "server/meissa/grafana-cloud", "gopass_field": "grafana-cloud-user", @@ -50,15 +52,21 @@ class InitService: "name": "c4k_grafana_cloud_password", }, ] - credentials = Credentials(input, default_mappings) - authorization = self.authorization(credentials) + if BuildType.IMAGE in build_types: + default_mappings += [ + { + "gopass_path": "meissa/web/docker.com", + "gopass_field": "login", + "name": "image_dockerhub_user", + }, + { + "gopass_path": "meissa/web/docker.com", + "name": "image_dockerhub_password", + }, + ] - context = self.context() - - merged = self.devops_factory.merge(input, context, authorization) - if MixinType.RELEASE in mixin_types: - primary_build_file_id = merged.get( + primary_build_file_id = input.get( "release_primary_build_file", "./project.clj" ) primary_build_file = self.build_file_repository.get( @@ -66,24 +74,41 @@ class InitService: ) version = primary_build_file.get_version() + credentials = Credentials(input, default_mappings) + authorization = self.authorization(credentials) + + context = self.context(mixin_types, version) + + merged = self.devops_factory.merge(input, context, authorization) + return self.devops_factory.build_devops(merged, version=version) - def context(self) -> dict: + def context(self, mixin_types, version) -> dict: result = {} - - release_type = self.environment_api.get("RELEASE_TYPE") - if not release_type: - latest_commit = self.git_api.get_latest_commit() - if latest_commit in [ReleaseType.MAJOR.name, ReleaseType.MINOR.name, - ReleaseType.PATCH.name, ReleaseType.NONE.name]: - release_type = latest_commit - result["release_type"] = release_type - result["release_current_branch"] = self.git_api.get_current_branch() + tag = self.environment_api.get("IMAGE_TAG") + + if MixinType.RELEASE in mixin_types: + release_type = self.environment_api.get("RELEASE_TYPE") + if not release_type: + latest_commit = self.git_api.get_latest_commit() + if latest_commit in [ + ReleaseType.MAJOR.name, + ReleaseType.MINOR.name, + ReleaseType.PATCH.name, + ReleaseType.NONE.name, + ]: + release_type = latest_commit + result["release_type"] = release_type + result["release_current_branch"] = self.git_api.get_current_branch() + + if not tag: + tag = version.to_string() + + result["image_tag"] = tag return result - def authorization(self, credentials: Credentials) -> List[str]: result = {} for name in credentials.mappings.keys(): From 72b5e19ffe941278c6f2d3f174585b813d0f6758 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 May 2023 13:56:43 +0200 Subject: [PATCH 096/173] Include shell parameter in ExecutionApi --- .../python/ddadevops/infrastructure/infrastructure.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 54932db..b41e890 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -90,14 +90,13 @@ class ImageApi: class ExecutionApi: - def execute(self, command: str, dry_run=False): + def execute(self, command: str, dry_run=False, shell=True): output = "" if dry_run: print(command) else: - output = check_output(command, encoding="UTF-8", shell=True) + output = check_output(command, encoding="UTF-8", shell=shell) output = output.rstrip() - print(output) return output def execute_live(command): @@ -121,7 +120,7 @@ class CredentialsApi: credential = None if path and field: print("get field for: " + path + ", " + field) - credential = self.execution_api.execute(["gopass", "show", path, field]) + credential = self.execution_api.execute(["gopass", "show", path, field], shell=False) return credential def gopass_password_from_path(self, path): @@ -129,7 +128,7 @@ class CredentialsApi: if path: print("get password for: " + path) credential = self.execution_api.execute( - ["gopass", "show", "--password", path] + ["gopass", "show", "--password", path], shell=False ) return credential From 651c1669201a80d26b8dab1d6a5c4d9f4bf043cd Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 14:34:39 +0200 Subject: [PATCH 097/173] fix resource loading from pkg --- build.py | 2 +- .../ddadevops/application/image_build_service.py | 16 ++++++++-------- src/main/python/ddadevops/domain/image.py | 3 +++ .../ddadevops/infrastructure/infrastructure.py | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/build.py b/build.py index 5fcf676..9d4025b 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev21" +version = "4.0.0-dev26" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index 5fe398c..d191752 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -17,30 +17,30 @@ class ImageBuildService: ) def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): - data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}") + data = self.resource_api.read_resource(f"main/resources/docker/{resource_name}") self.file_api.write_data_to_file( f"{devops.build_path()}/{resource_name}", data ) def __copy_build_resources_from_package__(self, devops: Devops): self.__copy_build_resource_file_from_package__( - "image/resources/install_functions.sh", devops.specialized_builds[BuildType.C4K] + "image/resources/install_functions.sh", devops.specialized_builds[BuildType.IMAGE] ) def __copy_build_resources_from_dir__(self, devops: Devops): - image = devops.specialized_builds[BuildType.C4K] + image = devops.specialized_builds[BuildType.IMAGE] self.file_api.cp_force( image.build_commons_path(), devops.build_path() ) def initialize_build_dir(self, devops: Devops): - image = devops.specialized_builds[BuildType.C4K] + image = devops.specialized_builds[BuildType.IMAGE] build_path = devops.build_path() self.file_api.clean_dir(f"{build_path}/image/resources") if image.image_use_package_common_files: - self.__copy_build_resources_from_package__(image) + self.__copy_build_resources_from_package__(devops) else: - self.__copy_build_resources_from_dir__(image) + self.__copy_build_resources_from_dir__(devops) self.file_api.cp_recursive("image", build_path) self.file_api.cp_recursive("test", build_path) @@ -51,13 +51,13 @@ class ImageBuildService: self.image_api.drun(devops.name) def dockerhub_login(self, devops: Devops): - image = devops.specialized_builds[BuildType.C4K] + image = devops.specialized_builds[BuildType.IMAGE] self.image_api.dockerhub_login( image.image_dockerhub_user, image.image_dockerhub_password ) def dockerhub_publish(self, devops: Devops): - image = devops.specialized_builds[BuildType.C4K] + image = devops.specialized_builds[BuildType.IMAGE] if image.image_tag is None or image.image_tag == "": raise ValueError(f"image_tag must not be empty.") self.image_api.dockerhub_publish( diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index 165585d..b959c3c 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -25,6 +25,9 @@ class Image(Validateable): result = [] result += self.__validate_is_not_empty__("image_dockerhub_user") result += self.__validate_is_not_empty__("image_dockerhub_password") + if not self.image_use_package_common_files: + result += self.__validate_is_not_empty__("image_build_commons_path") + result += self.__validate_is_not_empty__("image_build_commons_dir_name") return result def build_commons_path(self): diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index b41e890..6333de1 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -9,7 +9,7 @@ from ..domain import Devops, Image, C4k, Release, BuildFile class ResourceApi: def read_resource(self, path: str) -> bytes: - return resource_string(__name__, path) + return resource_string('src', path) class FileApi: From 8732c374a6c9c1676705230f8d4cc5278d0b406c Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 May 2023 15:02:52 +0200 Subject: [PATCH 098/173] Fix resource_path --- src/main/python/ddadevops/application/image_build_service.py | 4 ++-- src/main/python/ddadevops/infrastructure/infrastructure.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index d191752..f3fa3f2 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -17,14 +17,14 @@ class ImageBuildService: ) def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): - data = self.resource_api.read_resource(f"main/resources/docker/{resource_name}") + data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}") self.file_api.write_data_to_file( f"{devops.build_path()}/{resource_name}", data ) def __copy_build_resources_from_package__(self, devops: Devops): self.__copy_build_resource_file_from_package__( - "image/resources/install_functions.sh", devops.specialized_builds[BuildType.IMAGE] + "image/resources/install_functions.sh", devops ) def __copy_build_resources_from_dir__(self, devops: Devops): diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 6333de1..55cc74a 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -9,7 +9,7 @@ from ..domain import Devops, Image, C4k, Release, BuildFile class ResourceApi: def read_resource(self, path: str) -> bytes: - return resource_string('src', path) + return resource_string('ddadevops', path) class FileApi: From 27c0fe62eb8e0cf08a9300835d8574b134bccd71 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 May 2023 15:03:09 +0200 Subject: [PATCH 099/173] Use devops_build in project --- src/main/python/ddadevops/devops_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index 4ebe93b..f2622a9 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -17,7 +17,7 @@ def create_devops_build_config( def get_devops_build(project): - return project.get_property("devops_build") + return project.get_property("build") class DevopsBuild: @@ -28,6 +28,7 @@ class DevopsBuild: self.devops_repo = DevopsRepository() devops = self.init_service.initialize(input) self.devops_repo.set_devops(self.project, devops) + self.project.set_property("build", self) def name(self): devops = self.devops_repo.get_devops(self.project) From 8226d34c495b1968c9da8f9c5f7ebb39af350b44 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 19 May 2023 15:03:15 +0200 Subject: [PATCH 100/173] Bump version --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 9d4025b..415acf6 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev26" +version = "4.0.0-dev33" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] From 31178e940da206061a495916b2b951ea7524aa7e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 15:49:24 +0200 Subject: [PATCH 101/173] fix image build --- src/main/python/ddadevops/application/image_build_service.py | 2 -- src/main/python/ddadevops/domain/init_service.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index f3fa3f2..811131a 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -58,8 +58,6 @@ class ImageBuildService: def dockerhub_publish(self, devops: Devops): image = devops.specialized_builds[BuildType.IMAGE] - if image.image_tag is None or image.image_tag == "": - raise ValueError(f"image_tag must not be empty.") self.image_api.dockerhub_publish( devops.name, image.image_dockerhub_user, image.image_tag ) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 3d36e36..e05a6e3 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -105,7 +105,8 @@ class InitService: if not tag: tag = version.to_string() - result["image_tag"] = tag + if tag: + result["image_tag"] = tag return result From e51d2ec2a42e319f3c89df613876d62ca83c9f69 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 15:50:15 +0200 Subject: [PATCH 102/173] fixes for release-mixin --- .../python/ddadevops/application/release_mixin_services.py | 2 +- src/main/python/ddadevops/release_mixin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 8b2a015..58f1b03 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -53,7 +53,7 @@ class ReleaseService: self, version: Version, build_file_ids: List[str], message: str ): for id in build_file_ids: - build_file = self.build_file_repository.get(id) + build_file = self.build_file_repository.get(Path(id)) build_file.set_version(version) self.build_file_repository.write(build_file) self.git_api.add_file(build_file.file_path) diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index acdcb6a..79e0a25 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -13,11 +13,11 @@ class ReleaseMixin(DevopsBuild): raise ValueError(f"ReleaseMixin requires MixinType.RELEASE") def prepare_release(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) release = devops.mixins[MixinType.RELEASE] self.release_service.prepare_release(release) def tag_bump_and_push_release(self): - devops = self.repo.get_devops(self.project) + devops = self.devops_repo.get_devops(self.project) release = devops.mixins[MixinType.RELEASE] self.release_service.tag_bump_and_push_release(release) From 76fd45e2dd9725661856a6e2e9e69755ab620518 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 15:50:33 +0200 Subject: [PATCH 103/173] fix for credentials-service --- build.py | 2 +- infrastructure/clojure/build.py | 4 ++-- src/main/python/ddadevops/domain/credentials_service.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build.py b/build.py index 415acf6..5be177e 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev33" +version = "4.0.0-dev35" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/infrastructure/clojure/build.py b/infrastructure/clojure/build.py index 846119c..036592c 100644 --- a/infrastructure/clojure/build.py +++ b/infrastructure/clojure/build.py @@ -2,8 +2,8 @@ from os import environ from pybuilder.core import task, init from ddadevops import * -name = "clojure" -MODULE = "docker" +name = "ddabuild" +MODULE = "clojure" PROJECT_ROOT_PATH = "../.." diff --git a/src/main/python/ddadevops/domain/credentials_service.py b/src/main/python/ddadevops/domain/credentials_service.py index 39947f6..178e932 100644 --- a/src/main/python/ddadevops/domain/credentials_service.py +++ b/src/main/python/ddadevops/domain/credentials_service.py @@ -2,8 +2,8 @@ from pathlib import Path from .common import Devops, MixinType from .devops_factory import DevopsFactory from .version import Version -from .infrastructure import ( - BuildFileRepository +from ..infrastructure import ( + GopassApi, EnvironmentApi ) class CredentialsService: @@ -12,8 +12,8 @@ class CredentialsService: @classmethod def prod(cls): return cls( - DevopsFactory(), - BuildFileRepository(base_dir), + GopassApi(), + EnvironmentApi() ) def initialize(self, input: dict) -> Devops: From 2a4b1880af55487e9af15044c8448299e4b7f10c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 15:52:11 +0200 Subject: [PATCH 104/173] remove unused --- .../ddadevops/domain/credentials_service.py | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 src/main/python/ddadevops/domain/credentials_service.py diff --git a/src/main/python/ddadevops/domain/credentials_service.py b/src/main/python/ddadevops/domain/credentials_service.py deleted file mode 100644 index 178e932..0000000 --- a/src/main/python/ddadevops/domain/credentials_service.py +++ /dev/null @@ -1,28 +0,0 @@ -from pathlib import Path -from .common import Devops, MixinType -from .devops_factory import DevopsFactory -from .version import Version -from ..infrastructure import ( - GopassApi, EnvironmentApi -) - -class CredentialsService: - def __init__(self, gopass_api, environment_api): - - @classmethod - def prod(cls): - return cls( - GopassApi(), - EnvironmentApi() - ) - - def initialize(self, input: dict) -> Devops: - mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) - version = None - - if MixinType.RELEASE in mixin_types: - primary_build_file_id = input.get("release_primary_build_file", "./project.clj") - primary_build_file = self.build_file_repository.get(Path(primary_build_file_id)) - version = primary_build_file.get_version() - - return self.devops_factory.build_devops(input, version=version) From c591ab5abeeb4768e57cbc637845c5dd308918b0 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 16:20:42 +0200 Subject: [PATCH 105/173] do not ask for gopass password in test --- src/test/python/test_c4k_build.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/python/test_c4k_build.py b/src/test/python/test_c4k_build.py index 3db4c35..9e9fab5 100644 --- a/src/test/python/test_c4k_build.py +++ b/src/test/python/test_c4k_build.py @@ -2,7 +2,10 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain import DnsRecord from src.main.python.ddadevops.c4k_build import C4kBuild, add_c4k_mixin_config -from .domain.helper import devops_config +from .domain.helper import ( + CredentialsApiMock, + devops_config, +) def test_c4k_build(tmp_path): @@ -24,6 +27,7 @@ def test_c4k_build(tmp_path): } ), ) + sut.init_service.credentials_api = CredentialsApiMock({}) sut.initialize_build_dir() assert sut.build_path() == f"{str_tmp_path}/target/name/c4k-test" From f8b6a73b60195180a4fef8270aebbcedce4f4788 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 16:21:01 +0200 Subject: [PATCH 106/173] add missing import --- src/main/python/ddadevops/application/release_mixin_services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 58f1b03..fb01a8d 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,4 +1,5 @@ from typing import Optional, List +from pathlib import Path from ..infrastructure import GitApi, BuildFileRepository from ..domain import Version, Release, ReleaseType From 816fcfdaa765a3d0b8dba66ac2aaec09e59be23d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 16:21:12 +0200 Subject: [PATCH 107/173] add test --- src/test/python/domain/test_release.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index ec1ed9b..8690fd1 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -46,3 +46,18 @@ def test_sould_validate_release(): Version.from_str("1.3.1-SNAPSHOT"), ) assert not sut.is_valid() + + +def test_sould_validate_release(): + sut = Release( + devops_config( + { + "release_type": "MINOR", + "release_current_branch": "main", + "release_primary_build_file": "project.clj", + "release_secondary_build_files": ["package.json"], + } + ), + Version.from_str("1.3.1-SNAPSHOT"), + ) + assert ["project.clj", "package.json"] == sut.build_files() From 93fef828f624dbc2f6530852424ed300988f72c0 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 16:49:57 +0200 Subject: [PATCH 108/173] omit gopass usage during tests --- src/test/python/test_c4k_build.py | 4 +++- src/test/python/test_devops_build.py | 5 ++++- src/test/python/test_image_build.py | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/test/python/test_c4k_build.py b/src/test/python/test_c4k_build.py index 9e9fab5..f5a3a46 100644 --- a/src/test/python/test_c4k_build.py +++ b/src/test/python/test_c4k_build.py @@ -12,6 +12,9 @@ def test_c4k_build(tmp_path): str_tmp_path = str(tmp_path) project = Project(str_tmp_path, name="name") + os.environ["C4K_GRAFANA_CLOUD_USER"] = "user" + os.environ["C4K_GRAFANA_CLOUD_PASSWORD"] = "password" + sut = C4kBuild( project, devops_config( @@ -27,7 +30,6 @@ def test_c4k_build(tmp_path): } ), ) - sut.init_service.credentials_api = CredentialsApiMock({}) sut.initialize_build_dir() assert sut.build_path() == f"{str_tmp_path}/target/name/c4k-test" diff --git a/src/test/python/test_devops_build.py b/src/test/python/test_devops_build.py index f31f738..54ff6b3 100644 --- a/src/test/python/test_devops_build.py +++ b/src/test/python/test_devops_build.py @@ -5,9 +5,10 @@ from src.main.python.ddadevops import DevopsBuild from .domain.helper import devops_config from .resource_helper import copy_resource + def test_devops_build(tmp_path): str_tmp_path = str(tmp_path) - copy_resource(Path('package.json'), tmp_path) + copy_resource(Path("package.json"), tmp_path) project = Project(str_tmp_path, name="name") devops_build = DevopsBuild( @@ -15,6 +16,8 @@ def test_devops_build(tmp_path): devops_config( { "project_root_path": str_tmp_path, + "build_types": [], + "mixin_types": [], } ), ) diff --git a/src/test/python/test_image_build.py b/src/test/python/test_image_build.py index 71e99ec..cfcb561 100644 --- a/src/test/python/test_image_build.py +++ b/src/test/python/test_image_build.py @@ -7,6 +7,10 @@ from .domain.helper import devops_config def test_devops_docker_build(tmp_path): str_tmp_path = str(tmp_path) project = Project(str_tmp_path, name="name") + + os.environ["IMAGE_DOCKERHUB_USER"] = "user" + os.environ["IMAGE_DOCKERHUB_PASSWORD"] = "password" + image_build = DevopsImageBuild( project, devops_config( From eb6491a1fb2bbc182d65056b2ecb5f6d514d3f38 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 16:50:28 +0200 Subject: [PATCH 109/173] fix buildfile version replacement for clojure --- build.py | 2 +- src/main/python/ddadevops/domain/build_file.py | 1 + src/test/python/domain/test_build_file.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 5be177e..eaca534 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev35" +version = "4.0.0-dev38" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py index 7fc941d..c69a737 100644 --- a/src/main/python/ddadevops/domain/build_file.py +++ b/src/main/python/ddadevops/domain/build_file.py @@ -121,6 +121,7 @@ class BuildFile(Validateable): '"[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'"{new_version.to_string()}"', self.content, + 1 ) self.content = substitute except: diff --git a/src/test/python/domain/test_build_file.py b/src/test/python/domain/test_build_file.py index cfc0c74..061427b 100644 --- a/src/test/python/domain/test_build_file.py +++ b/src/test/python/domain/test_build_file.py @@ -138,3 +138,14 @@ def test_sould_parse_and_set_version_for_clj(): ) sut.set_version(Version.from_str("1.1.5-SNAPSHOT").create_major()) assert '\n(defproject org.domaindrivenarchitecture/c4k-jira "2.0.0"\n :description "jira c4k-installation package"\n)\n' == sut.content + + sut = BuildFile( + Path("./project.clj"), + """ +(defproject org.domaindrivenarchitecture/c4k-jira "1.1.5-SNAPSHOT" +:dependencies [[org.clojure/clojure "1.11.0"]] +) + """, + ) + sut.set_version(Version.from_str("1.1.5-SNAPSHOT").create_major()) + assert '\n(defproject org.domaindrivenarchitecture/c4k-jira "2.0.0"\n:dependencies [[org.clojure/clojure "1.11.0"]]\n)\n ' == sut.content From d60b7b03a1d4c6c4fe8aaf686633591ea84345d5 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 17:04:23 +0200 Subject: [PATCH 110/173] remove remaing chars in case of shrinking file len --- build.py | 2 +- src/main/python/ddadevops/infrastructure/repository.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index eaca534..83ebe31 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev38" +version = "4.0.0-dev39" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/infrastructure/repository.py b/src/main/python/ddadevops/infrastructure/repository.py index b3c3546..057f083 100644 --- a/src/main/python/ddadevops/infrastructure/repository.py +++ b/src/main/python/ddadevops/infrastructure/repository.py @@ -40,3 +40,4 @@ class BuildFileRepository: ) as file: file.seek(0) file.write(build_file.content) + file.truncate() From 0b14b49b8623f56206695228a71ea70b030d6c81 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 17:14:24 +0200 Subject: [PATCH 111/173] fix tagging --- src/main/python/ddadevops/infrastructure/infrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 55cc74a..7a44048 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -146,7 +146,7 @@ class GitApi: def tag_annotated(self, annotation: str, message: str, count: int): return self.execution_api.execute( - f"git tag -a {annotation} -m {message} HEAD~{count}" + f"git tag -a {annotation} -m '{message}' HEAD~{count}" ) def tag_annotated_second_last(self, annotation: str, message: str): From 6dfc6a5a943d31ffbd2acf55c5cf7f97bac35f40 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 19 May 2023 17:16:52 +0200 Subject: [PATCH 112/173] fix version bumping --- build.py | 2 +- src/main/python/ddadevops/application/release_mixin_services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 83ebe31..38c97cf 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev39" +version = "4.0.0-dev41" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index fb01a8d..5a9565a 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -39,7 +39,7 @@ class ReleaseService: release_version = release.version.create_patch() case ReleaseType.NONE: return - bump_version = release_version.create_bump() + bump_version = release_version.create_bump("SNAPSHOT") release_message = f"release: {release_version.to_string()}" bump_message = f"bump version to: {bump_version.to_string()}" self.git_api.tag_annotated(release_version.to_string(), release_message, 0) From 550f2207598b4273f2fb9ca1a6fd5e183b436435 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 20 May 2023 11:48:49 +0200 Subject: [PATCH 113/173] adjust CI to new inputs --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1abb7b6..649b77c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,9 @@ before_script: - python --version - python -m pip install --upgrade pip - pip install -r requirements.txt + - export IMAGE_TAG=$CI_IMAGE_TAG + - export IMAGE_DOCKERHUB_USER=&DOCKERHUB_USER + - export IMAGE_DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD stages: - lint&test From 69ddcbbf7b8c5cf71cc9e1ffa29508dc48430210 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 20 May 2023 11:49:24 +0200 Subject: [PATCH 114/173] adjust image builds to new build-version --- infrastructure/clojure/build.py | 4 +-- infrastructure/devops-build/build.py | 39 ++++++++++------------------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/infrastructure/clojure/build.py b/infrastructure/clojure/build.py index 036592c..c3d89d2 100644 --- a/infrastructure/clojure/build.py +++ b/infrastructure/clojure/build.py @@ -2,8 +2,8 @@ from os import environ from pybuilder.core import task, init from ddadevops import * -name = "ddabuild" -MODULE = "clojure" +name = "clojure" +MODULE = "image" PROJECT_ROOT_PATH = "../.." diff --git a/infrastructure/devops-build/build.py b/infrastructure/devops-build/build.py index 488d160..4dd891e 100644 --- a/infrastructure/devops-build/build.py +++ b/infrastructure/devops-build/build.py @@ -3,38 +3,25 @@ from pybuilder.core import task, init from ddadevops import * name = "devops-build" -MODULE = "docker" +MODULE = "image" PROJECT_ROOT_PATH = "../.." @init def initialize(project): + + input = { + "name": name, + "module": MODULE, + "stage": "notused", + "project_root_path": PROJECT_ROOT_PATH, + "build_types": ["IMAGE"], + "mixin_types": [], + } + project.build_depends_on("ddadevops>=4.0.0-dev") - stage = "notused" - dockerhub_user = environ.get("DOCKERHUB_USER") - if not dockerhub_user: - dockerhub_user = gopass_field_from_path("meissa/web/docker.com", "login") - dockerhub_password = environ.get("DOCKERHUB_PASSWORD") - if not dockerhub_password: - dockerhub_password = gopass_password_from_path("meissa/web/docker.com") - tag = environ.get("CI_COMMIT_TAG") - if not tag: - tag = get_tag_from_latest_commit() - - devops = Devops( - stage=stage, - project_root_path=PROJECT_ROOT_PATH, - module=MODULE, - name=name, - ) - image = Image( - dockerhub_user=dockerhub_user, - dockerhub_password=dockerhub_password, - docker_publish_tag=tag, - devops=devops, - ) - - build = DevopsImageBuild(project, image=image) + + build = DevopsImageBuild(project, input) build.initialize_build_dir() From d5b75254b53502c0dca1d85cd4fc089307c3d518 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 20 May 2023 13:47:33 +0200 Subject: [PATCH 115/173] fix ci --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18694c4..0f020fb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ before_script: - python -m pip install --upgrade pip - pip install -r requirements.txt - export IMAGE_TAG=$CI_IMAGE_TAG - - export IMAGE_DOCKERHUB_USER=&DOCKERHUB_USER + - export IMAGE_DOCKERHUB_USER=$DOCKERHUB_USER - export IMAGE_DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD stages: From ce35bc2d546caa5ab9aa0821f1d0233ef20ae461 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 20 May 2023 15:40:27 +0200 Subject: [PATCH 116/173] make mypy green --- .gitlab-ci.yml | 2 +- .../ddadevops/application/image_build_service.py | 3 ++- src/main/python/ddadevops/domain/devops_factory.py | 10 +++++----- src/main/python/ddadevops/domain/init_service.py | 6 +++--- src/main/python/ddadevops/domain/release.py | 2 +- src/main/python/ddadevops/domain/version.py | 2 +- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f020fb..33dfff0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ mypy: stage: lint&test script: - pip install -r dev_requirements.txt - - python -m mypy src/main/python/ddadevops/ --ignore-missing-imports + - python -m mypy src/main/python/ddadevops/ --ignore-missing-imports --disable-error-code=attr-defined --disable-error-code=union-attr pylint: stage: lint&test diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index 811131a..65d37b4 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -1,3 +1,4 @@ +from pathlib import Path from ..domain import Image, Devops, BuildType from ..infrastructure import FileApi, ResourceApi, ImageApi @@ -19,7 +20,7 @@ class ImageBuildService: def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}") self.file_api.write_data_to_file( - f"{devops.build_path()}/{resource_name}", data + Path(f"{devops.build_path()}/{resource_name}"), data ) def __copy_build_resources_from_package__(self, devops: Devops): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index f631c31..f933266 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -1,7 +1,7 @@ import deprecation from enum import Enum -from typing import List -from .common import Devops, BuildType, MixinType +from typing import List, Optional, Dict +from .common import Validateable, Devops, BuildType, MixinType from .image import Image from .c4k import C4k from .release import Release @@ -12,17 +12,17 @@ class DevopsFactory: def __init__(self): pass - def build_devops(self, input: dict, version: Version = None) -> Devops: + def build_devops(self, input: dict, version: Optional[Version] = None) -> Devops: build_types = self.__parse_build_types__(input["build_types"]) mixin_types = self.__parse_mixin_types__(input["mixin_types"]) - specialized_builds = {} + specialized_builds: Dict[BuildType, Validateable] = {} if BuildType.IMAGE in build_types: specialized_builds[BuildType.IMAGE] = Image(input) if BuildType.C4K in build_types: specialized_builds[BuildType.C4K] = C4k(input) - mixins = {} + mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: mixins[MixinType.RELEASE] = Release(input, version) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index e05a6e3..bb42860 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -1,7 +1,7 @@ from pathlib import Path -from typing import List +from typing import List, Dict from .common import Devops, MixinType, BuildType -from .credentials import Credentials, GopassType +from .credentials import CredentialMapping, Credentials, GopassType from .devops_factory import DevopsFactory from .version import Version from .release import ReleaseType @@ -110,7 +110,7 @@ class InitService: return result - def authorization(self, credentials: Credentials) -> List[str]: + def authorization(self, credentials: Credentials) -> Dict[str, CredentialMapping]: result = {} for name in credentials.mappings.keys(): mapping = credentials.mappings[name] diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 78f69ad..1edfac8 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -12,7 +12,7 @@ from .version import ( class Release(Validateable): - def __init__(self, input: dict, version: Version): + def __init__(self, input: dict, version: Optional[Version]): self.release_type = ReleaseType[input.get("release_type", "NONE")] self.release_main_branch = input.get("release_main_branch", "main") self.release_current_branch = input.get("release_current_branch") diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 230c1ae..f5cee17 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -60,7 +60,7 @@ class Version(Validateable): ] return result - def create_bump(self, snapshot_suffix: str = None): + def create_bump(self, snapshot_suffix: Optional[str] = None): new_version_list = self.version_list.copy() if self.is_snapshot(): return Version( From b44751149a4e4cf677eeade1999e23e96c97180e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 20 May 2023 16:14:35 +0200 Subject: [PATCH 117/173] fix pylint --- .gitlab-ci.yml | 2 +- .../application/image_build_service.py | 2 +- .../application/release_mixin_services.py | 6 ++-- src/main/python/ddadevops/c4k_build.py | 2 +- src/main/python/ddadevops/devops_build.py | 7 ++--- .../python/ddadevops/devops_image_build.py | 7 ++--- src/main/python/ddadevops/domain/__init__.py | 2 +- .../python/ddadevops/domain/build_file.py | 25 ++++------------- src/main/python/ddadevops/domain/c4k.py | 18 ++++++------ src/main/python/ddadevops/domain/common.py | 28 +++++-------------- .../python/ddadevops/domain/credentials.py | 18 ++++++------ .../python/ddadevops/domain/devops_factory.py | 20 ++++++------- src/main/python/ddadevops/domain/image.py | 16 +++++------ .../python/ddadevops/domain/init_service.py | 15 +++++----- src/main/python/ddadevops/domain/release.py | 14 ++++------ src/main/python/ddadevops/domain/version.py | 5 ++-- .../infrastructure/infrastructure.py | 5 ++-- .../ddadevops/infrastructure/repository.py | 10 ++----- src/main/python/ddadevops/release_mixin.py | 6 ++-- 19 files changed, 82 insertions(+), 126 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33dfff0..dff5793 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,7 @@ pylint: stage: lint&test script: - pip install -r dev_requirements.txt - - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/ + - pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0702,W0702,R0913,R0902,R0914,R1732,R1705,W0707,C0123,W0703,C0103 src/main/python/ddadevops/ pytest: stage: lint&test diff --git a/src/main/python/ddadevops/application/image_build_service.py b/src/main/python/ddadevops/application/image_build_service.py index 65d37b4..cc77ca3 100644 --- a/src/main/python/ddadevops/application/image_build_service.py +++ b/src/main/python/ddadevops/application/image_build_service.py @@ -1,5 +1,5 @@ from pathlib import Path -from ..domain import Image, Devops, BuildType +from ..domain import Devops, BuildType from ..infrastructure import FileApi, ResourceApi, ImageApi diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 5a9565a..1557390 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List from pathlib import Path from ..infrastructure import GitApi, BuildFileRepository from ..domain import Version, Release, ReleaseType @@ -53,8 +53,8 @@ class ReleaseService: def __set_version_and_commit__( self, version: Version, build_file_ids: List[str], message: str ): - for id in build_file_ids: - build_file = self.build_file_repository.get(Path(id)) + for build_file_id in build_file_ids: + build_file = self.build_file_repository.get(Path(build_file_id)) build_file.set_version(version) self.build_file_repository.write(build_file) self.git_api.add_file(build_file.file_path) diff --git a/src/main/python/ddadevops/c4k_build.py b/src/main/python/ddadevops/c4k_build.py index 7964917..c2f9685 100644 --- a/src/main/python/ddadevops/c4k_build.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -49,7 +49,7 @@ class C4kBuild(DevopsBuild): self.execution_api = ExecutionApi() devops = self.devops_repo.get_devops(self.project) if BuildType.C4K not in devops.specialized_builds: - raise ValueError(f"C4kBuild requires BuildType.C4K") + raise ValueError("C4kBuild requires BuildType.C4K") def update_runtime_config(self, dns_record: DnsRecord): devops = self.devops_repo.get_devops(self.project) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index f2622a9..40441b1 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,6 +1,5 @@ -from typing import Optional import deprecation -from .domain import Devops, InitService +from .domain import InitService from .infrastructure import DevopsRepository, FileApi @@ -21,12 +20,12 @@ def get_devops_build(project): class DevopsBuild: - def __init__(self, project, input: dict): + def __init__(self, project, inp: dict): self.project = project self.file_api = FileApi() self.init_service = InitService.prod(project.basedir) self.devops_repo = DevopsRepository() - devops = self.init_service.initialize(input) + devops = self.init_service.initialize(inp) self.devops_repo.set_devops(self.project, devops) self.project.set_property("build", self) diff --git a/src/main/python/ddadevops/devops_image_build.py b/src/main/python/ddadevops/devops_image_build.py index c5684bb..4dd3863 100644 --- a/src/main/python/ddadevops/devops_image_build.py +++ b/src/main/python/ddadevops/devops_image_build.py @@ -1,4 +1,3 @@ -from typing import Optional import deprecation from .domain import BuildType from .application import ImageBuildService @@ -33,12 +32,12 @@ def create_devops_docker_build_config( class DevopsImageBuild(DevopsBuild): - def __init__(self, project, input: dict): - super().__init__(project, input) + def __init__(self, project, inp: dict): + super().__init__(project, inp) self.image_build_service = ImageBuildService.prod() devops = self.devops_repo.get_devops(self.project) if BuildType.IMAGE not in devops.specialized_builds: - raise ValueError(f"ImageBuild requires BuildType.IMAGE") + raise ValueError("ImageBuild requires BuildType.IMAGE") def initialize_build_dir(self): super().initialize_build_dir() diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index f55faa3..0378707 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -6,4 +6,4 @@ from .release import Release from .credentials import Credentials, CredentialMapping, GopassType from .version import Version from .build_file import BuildFileType, BuildFile -from .init_service import InitService \ No newline at end of file +from .init_service import InitService diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py index c69a737..c7537a8 100644 --- a/src/main/python/ddadevops/domain/build_file.py +++ b/src/main/python/ddadevops/domain/build_file.py @@ -1,16 +1,9 @@ -from enum import Enum -from typing import Optional -from pathlib import Path -import re import json -from .common import ( - Validateable, - Devops, - ReleaseType, -) -from .version import ( - Version, -) +import re +from enum import Enum +from pathlib import Path +from .common import Validateable +from .version import Version class BuildFileType(Enum): @@ -110,18 +103,12 @@ class BuildFile(Validateable): ) self.content = substitute case BuildFileType.JAVA_CLOJURE: - version_line = re.search("\\(defproject .*\n", self.content) - version_line_group = version_line.group() # TODO: we should stick here on defproject instead of first line! - version_string = re.search( - "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?", version_line_group - ) - version_str = version_string.group() substitute = re.sub( '"[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'"{new_version.to_string()}"', self.content, - 1 + 1, ) self.content = substitute except: diff --git a/src/main/python/ddadevops/domain/c4k.py b/src/main/python/ddadevops/domain/c4k.py index ca086a6..16b8760 100644 --- a/src/main/python/ddadevops/domain/c4k.py +++ b/src/main/python/ddadevops/domain/c4k.py @@ -7,18 +7,18 @@ from .common import ( class C4k(Validateable): - def __init__(self, input: dict): - self.module = input.get("module") - self.stage = input.get("stage") - self.c4k_executable_name = input.get("c4k_executable_name", input.get("module")) - self.c4k_config = input.get("c4k_config", {}) - self.c4k_grafana_cloud_url = input.get( + def __init__(self, inp: dict): + self.module = inp.get("module") + self.stage = inp.get("stage") + self.c4k_executable_name = inp.get("c4k_executable_name", inp.get("module")) + self.c4k_config = inp.get("c4k_config", {}) + self.c4k_grafana_cloud_url = inp.get( "c4k_grafana_cloud_url", "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", ) - self.c4k_auth = input.get("c4k_auth", {}) - self.c4k_grafana_cloud_user = input.get('c4k_grafana_cloud_user') - self.c4k_grafana_cloud_password = input.get('c4k_grafana_cloud_password') + self.c4k_auth = inp.get("c4k_auth", {}) + self.c4k_grafana_cloud_user = inp.get('c4k_grafana_cloud_user') + self.c4k_grafana_cloud_password = inp.get('c4k_grafana_cloud_password') self.dns_record: Optional[DnsRecord] = None # TODO: these functions should be located at TerraformBuild later on. diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 0ed8664..3eb17a2 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,7 +1,5 @@ -import deprecation from enum import Enum -from typing import List, TypedDict -import deprecation +from typing import List def filter_none(list_to_filter): @@ -70,15 +68,15 @@ class DnsRecord(Validateable): class Devops(Validateable): def __init__( self, - input: dict, + inp: dict, specialized_builds: dict[BuildType, Validateable], mixins: dict[MixinType, Validateable], ): - self.stage = input.get("stage") - self.project_root_path = input.get("project_root_path") - self.module = input.get("module") - self.name = input.get("name", self.module) - self.build_dir_name = input.get("build_dir_name", "target") + self.stage = inp.get("stage") + self.project_root_path = inp.get("project_root_path") + self.module = inp.get("module") + self.name = inp.get("name", self.module) + self.build_dir_name = inp.get("build_dir_name", "target") self.specialized_builds = specialized_builds self.mixins = mixins @@ -99,15 +97,3 @@ class Devops(Validateable): for mixin in self.mixins: result += self.mixins[mixin].validate() return result - - def __put__(self, key, value): - self.stack[key] = value - - def __get(self, key): - return self.stack[key] - - def __get_keys__(self, keys): - result = {} - for key in keys: - result[key] = self.__get(key) - return result diff --git a/src/main/python/ddadevops/domain/credentials.py b/src/main/python/ddadevops/domain/credentials.py index f144394..26bba2a 100644 --- a/src/main/python/ddadevops/domain/credentials.py +++ b/src/main/python/ddadevops/domain/credentials.py @@ -1,8 +1,6 @@ -import deprecation from enum import Enum -from typing import List, TypedDict +from typing import List from inflection import underscore -import deprecation from .common import ( Validateable, ) @@ -23,7 +21,7 @@ class CredentialMapping(Validateable): result = [] result += self.__validate_is_not_empty__("gopass_path") if not self.name and not self.gopass_field: - result.append(f"Either name or gopass field has to be defined.") + result.append("Either name or gopass field has to be defined.") return result def gopass_type(self): @@ -46,14 +44,14 @@ class CredentialMapping(Validateable): class Credentials(Validateable): - def __init__(self, input: dict, default_mappings: list = []): - input_mappings = input.get("credentials_mapping", []) + def __init__(self, inp: dict, default_mappings: list = []): + inp_mappings = inp.get("credentials_mapping", []) self.mappings = {} - for input_mapping in default_mappings: - mapping = CredentialMapping(input_mapping) + for inp_mapping in default_mappings: + mapping = CredentialMapping(inp_mapping) self.mappings[mapping.name_for_input()] = mapping - for input_mapping in input_mappings: - mapping = CredentialMapping(input_mapping) + for inp_mapping in inp_mappings: + mapping = CredentialMapping(inp_mapping) self.mappings[mapping.name_for_input()] = mapping def validate(self) -> List[str]: diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index f933266..48d1089 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -1,5 +1,3 @@ -import deprecation -from enum import Enum from typing import List, Optional, Dict from .common import Validateable, Devops, BuildType, MixinType from .image import Image @@ -12,28 +10,28 @@ class DevopsFactory: def __init__(self): pass - def build_devops(self, input: dict, version: Optional[Version] = None) -> Devops: - build_types = self.__parse_build_types__(input["build_types"]) - mixin_types = self.__parse_mixin_types__(input["mixin_types"]) + def build_devops(self, inp: dict, version: Optional[Version] = None) -> Devops: + build_types = self.__parse_build_types__(inp["build_types"]) + mixin_types = self.__parse_mixin_types__(inp["mixin_types"]) specialized_builds: Dict[BuildType, Validateable] = {} if BuildType.IMAGE in build_types: - specialized_builds[BuildType.IMAGE] = Image(input) + specialized_builds[BuildType.IMAGE] = Image(inp) if BuildType.C4K in build_types: - specialized_builds[BuildType.C4K] = C4k(input) + specialized_builds[BuildType.C4K] = C4k(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: - mixins[MixinType.RELEASE] = Release(input, version) + mixins[MixinType.RELEASE] = Release(inp, version) - devops = Devops(input, specialized_builds=specialized_builds, mixins=mixins) + devops = Devops(inp, specialized_builds=specialized_builds, mixins=mixins) devops.throw_if_invalid() return devops - def merge(self, input: dict, context: dict, authorization: dict) -> dict: - return {} | context | authorization | input + def merge(self, inp: dict, context: dict, authorization: dict) -> dict: + return {} | context | authorization | inp def __parse_build_types__(self, build_types: List[str]) -> List[BuildType]: result = [] diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index b959c3c..a3cd888 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List from .common import ( filter_none, Validateable, @@ -8,16 +8,16 @@ from .common import ( class Image(Validateable): def __init__( self, - input: dict, + inp: dict, ): - self.image_dockerhub_user = input.get("image_dockerhub_user") - self.image_dockerhub_password = input.get("image_dockerhub_password") - self.image_tag = input.get("image_tag") - self.image_build_commons_path = input.get("image_build_commons_path") - self.image_use_package_common_files = input.get( + self.image_dockerhub_user = inp.get("image_dockerhub_user") + self.image_dockerhub_password = inp.get("image_dockerhub_password") + self.image_tag = inp.get("image_tag") + self.image_build_commons_path = inp.get("image_build_commons_path") + self.image_use_package_common_files = inp.get( "image_use_package_common_files", True ) - self.image_build_commons_dir_name = input.get( + self.image_build_commons_dir_name = inp.get( "image_build_commons_dir_name", "docker" ) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index bb42860..3d50203 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -1,9 +1,8 @@ from pathlib import Path -from typing import List, Dict +from typing import Dict from .common import Devops, MixinType, BuildType from .credentials import CredentialMapping, Credentials, GopassType from .devops_factory import DevopsFactory -from .version import Version from .release import ReleaseType from ..infrastructure import BuildFileRepository, CredentialsApi, EnvironmentApi, GitApi @@ -33,9 +32,9 @@ class InitService: GitApi(), ) - def initialize(self, input: dict) -> Devops: - build_types = self.devops_factory.__parse_build_types__(input["build_types"]) - mixin_types = self.devops_factory.__parse_mixin_types__(input["mixin_types"]) + def initialize(self, inp: dict) -> Devops: + build_types = self.devops_factory.__parse_build_types__(inp["build_types"]) + mixin_types = self.devops_factory.__parse_mixin_types__(inp["mixin_types"]) version = None default_mappings = [] @@ -66,7 +65,7 @@ class InitService: ] if MixinType.RELEASE in mixin_types: - primary_build_file_id = input.get( + primary_build_file_id = inp.get( "release_primary_build_file", "./project.clj" ) primary_build_file = self.build_file_repository.get( @@ -74,12 +73,12 @@ class InitService: ) version = primary_build_file.get_version() - credentials = Credentials(input, default_mappings) + credentials = Credentials(inp, default_mappings) authorization = self.authorization(credentials) context = self.context(mixin_types, version) - merged = self.devops_factory.merge(input, context, authorization) + merged = self.devops_factory.merge(inp, context, authorization) return self.devops_factory.build_devops(merged, version=version) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 1edfac8..7be61d8 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -1,9 +1,7 @@ -from enum import Enum from typing import Optional, List from pathlib import Path from .common import ( Validateable, - Devops, ReleaseType, ) from .version import ( @@ -12,14 +10,14 @@ from .version import ( class Release(Validateable): - def __init__(self, input: dict, version: Optional[Version]): - self.release_type = ReleaseType[input.get("release_type", "NONE")] - self.release_main_branch = input.get("release_main_branch", "main") - self.release_current_branch = input.get("release_current_branch") - self.release_primary_build_file = input.get( + def __init__(self, inp: dict, version: Optional[Version]): + self.release_type = ReleaseType[inp.get("release_type", "NONE")] + self.release_main_branch = inp.get("release_main_branch", "main") + self.release_current_branch = inp.get("release_current_branch") + self.release_primary_build_file = inp.get( "release_primary_build_file", "./project.clj" ) - self.release_secondary_build_files = input.get( + self.release_secondary_build_files = inp.get( "release_secondary_build_files", [] ) self.version = version diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index f5cee17..22d2566 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -1,4 +1,3 @@ -from enum import Enum from typing import Optional from .common import ( Validateable, @@ -37,7 +36,7 @@ class Version(Validateable): return self.to_string().__hash__() def is_snapshot(self): - return not self.snapshot_suffix == None + return not self.snapshot_suffix is None def to_string(self) -> str: version_no = ".".join([str(x) for x in self.version_list]) @@ -49,7 +48,7 @@ class Version(Validateable): result = [] result += self.__validate_is_not_empty__("version_list") if self.version_list and len(self.version_list) < 3: - result += [f"version_list must have at least 3 levels."] + result += ["version_list must have at least 3 levels."] if ( self.version_list and self.version_string diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 7a44048..7e1df4b 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -1,10 +1,9 @@ +from subprocess import check_output, Popen, PIPE, run from pathlib import Path from sys import stdout from os import chmod, environ from pkg_resources import resource_string import yaml -from subprocess import check_output, Popen, PIPE, run -from ..domain import Devops, Image, C4k, Release, BuildFile class ResourceApi: @@ -99,7 +98,7 @@ class ExecutionApi: output = output.rstrip() return output - def execute_live(command): + def execute_live(self, command): process = Popen(command, stdout=PIPE) for line in iter(process.stdout.readline, b""): print(line.decode("utf-8"), end="") diff --git a/src/main/python/ddadevops/infrastructure/repository.py b/src/main/python/ddadevops/infrastructure/repository.py index 057f083..07d1ccc 100644 --- a/src/main/python/ddadevops/infrastructure/repository.py +++ b/src/main/python/ddadevops/infrastructure/repository.py @@ -1,12 +1,6 @@ from pathlib import Path -from sys import stdout -from os import chmod -from subprocess import run -from pkg_resources import resource_string -import yaml -import deprecation -from ..domain import Devops, Image, C4k, Release, BuildFile -from ..python_util import execute +from ..domain.common import Devops +from ..domain.build_file import BuildFile class DevopsRepository: diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 79e0a25..fd909e8 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -5,12 +5,12 @@ from .domain import MixinType class ReleaseMixin(DevopsBuild): - def __init__(self, project: Project, input: dict): - super().__init__(project, input) + def __init__(self, project: Project, inp: dict): + super().__init__(project, inp) self.release_service = ReleaseService.prod(project.basedir) devops = self.devops_repo.get_devops(self.project) if MixinType.RELEASE not in devops.mixins: - raise ValueError(f"ReleaseMixin requires MixinType.RELEASE") + raise ValueError("ReleaseMixin requires MixinType.RELEASE") def prepare_release(self): devops = self.devops_repo.get_devops(self.project) From ed2f4d5ad02ac2ed85fceb78b0b9b68e3b0ccb7a Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 20 May 2023 16:23:19 +0200 Subject: [PATCH 118/173] fix pylint --- src/main/python/ddadevops/domain/credentials.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/domain/credentials.py b/src/main/python/ddadevops/domain/credentials.py index 26bba2a..7e1a945 100644 --- a/src/main/python/ddadevops/domain/credentials.py +++ b/src/main/python/ddadevops/domain/credentials.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import List +from typing import List, Optional from inflection import underscore from .common import ( Validateable, @@ -44,7 +44,9 @@ class CredentialMapping(Validateable): class Credentials(Validateable): - def __init__(self, inp: dict, default_mappings: list = []): + def __init__(self, inp: dict, default_mappings: Optional[List] = None): + if default_mappings is None: + default_mappings = [] inp_mappings = inp.get("credentials_mapping", []) self.mappings = {} for inp_mapping in default_mappings: From 8b348771e054e613c1e5cb347315c095d66ef85f Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 09:51:29 +0200 Subject: [PATCH 119/173] fix terraform-build --- build.py | 2 +- .../python/ddadevops/devops_terraform_build.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/build.py b/build.py index 38c97cf..0c579c0 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev41" +version = "4.0.0-dev44" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index f2019fb..a039d51 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -44,7 +44,14 @@ def create_devops_terraform_build_config(stage, class DevopsTerraformBuild(DevopsBuild): def __init__(self, project, config): - super().__init__(project, config) + inp = {} + inp["name"]=project.name + inp["module"]=config.get("module") + inp["stage"]=config.get("stage") + inp["project_root_path"]=config.get("project_root_path") + inp["build_types"]=[] + inp["mixin_types"]=[] + super().__init__(project, inp) project.build_depends_on('dda-python-terraform') self.additional_vars = config['additional_vars'] self.output_json_name = config['output_json_name'] @@ -83,13 +90,13 @@ class DevopsTerraformBuild(DevopsBuild): def copy_build_resources_from_dir(self): run('cp -f ' + self.terraform_build_commons_path() + - '* ' + self.build_path(), shell=False, check=False) + '* ' + self.build_path(), shell=True, check=False) def copy_local_state(self): - run('cp terraform.tfstate ' + self.build_path(), shell=False, check=False) + run('cp terraform.tfstate ' + self.build_path(), shell=True, check=False) def rescue_local_state(self): - run('cp ' + self.build_path() + '/terraform.tfstate .', shell=False, check=False) + run('cp ' + self.build_path() + '/terraform.tfstate .', shell=True, check=False) def initialize_build_dir(self): super().initialize_build_dir() From bae844dfdb803b301a250effd4595d25fe14a777 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 09:53:50 +0200 Subject: [PATCH 120/173] Breaking: remove outdated mixins --- src/main/python/ddadevops/aws_rds_pg_mixin.py | 117 ------------------ src/main/python/ddadevops/exoscale_mixin.py | 31 ----- 2 files changed, 148 deletions(-) delete mode 100644 src/main/python/ddadevops/aws_rds_pg_mixin.py delete mode 100644 src/main/python/ddadevops/exoscale_mixin.py diff --git a/src/main/python/ddadevops/aws_rds_pg_mixin.py b/src/main/python/ddadevops/aws_rds_pg_mixin.py deleted file mode 100644 index 07dd3eb..0000000 --- a/src/main/python/ddadevops/aws_rds_pg_mixin.py +++ /dev/null @@ -1,117 +0,0 @@ -from .python_util import execute -from .credential import gopass_password_from_path, gopass_field_from_path -from .devops_build import DevopsBuild - - -def add_aws_rds_pg_mixin_config(config, rds_host_name, db_name, - rds_resolve_dns=False, - db_port='5432'): - config.update({'AwsRdsPgMixin': - {'rds_host_name': rds_host_name, - 'db_name': db_name, - 'rds_resolve_dns': rds_resolve_dns, - 'db_port': db_port, - }}) - return config - - -class AwsRdsPgMixin(DevopsBuild): - - def __init__(self, project, config): - super().__init__(project, config) - aws_rds_pg_mixin_config = config['AwsRdsPgMixin'] - self.rds_host_name = aws_rds_pg_mixin_config['rds_host_name'] - self.rds_resolve_dns = aws_rds_pg_mixin_config['rds_resolve_dns'] - self.db_name = aws_rds_pg_mixin_config['db_name'] - self.db_port = aws_rds_pg_mixin_config['db_port'] - - def execute_pg_rds_sql(self, user, password, sql): - if self.rds_resolve_dns: - host_cmd = "dig " + self.rds_host_name + " +short | head -n1" - host = execute(host_cmd, shell=True) - else: - host = self.rds_host_name - - cmd = "PGUSER=" + user + " PGPASSWORD=" + password + \ - " psql --dbname=" + self.db_name + " --host=" + host + " --port=" + self.db_port + \ - " --set=sslmode=require -Atc \"" + sql + "\"" - result = execute(cmd, shell=True) - print("PSQL: ", host, result.rstrip()) - return result - - def alter_db_user_password(self, gopass_path): - user_name = gopass_field_from_path(gopass_path, 'user') - user_old_password = gopass_field_from_path(gopass_path, 'old-password') - user_new_password = gopass_password_from_path(gopass_path) - - self.execute_pg_rds_sql(user_name, user_old_password, - "ALTER ROLE " + user_name + " WITH PASSWORD '" + user_new_password + "';") - print("changed password:", user_name) - - def add_new_user(self, gopass_path_superuser, gopass_path_new_user, group_role): - superuser_name = gopass_field_from_path(gopass_path_superuser, 'user') - superuser_password = gopass_password_from_path(gopass_path_superuser) - new_user_name = gopass_field_from_path(gopass_path_new_user, 'user') - new_user_password = gopass_password_from_path(gopass_path_new_user) - - self.execute_pg_rds_sql(superuser_name, superuser_password, - "CREATE ROLE " + new_user_name + " WITH LOGIN INHERIT PASSWORD '" + new_user_password + "';" + - "GRANT " + group_role + " TO " + new_user_name + ";") - print("created user:", new_user_name) - - def deactivate_user(self, gopass_path_superuser, to_deactivate_user_name): - superuser_name = gopass_field_from_path(gopass_path_superuser, 'user') - superuser_password = gopass_password_from_path(gopass_path_superuser) - - owned_by_wrong_user = self.execute_pg_rds_sql(superuser_name, superuser_password, - "SELECT count(*) FROM pg_class c, pg_user u WHERE c.relowner = u.usesysid " + - "and u.usename='" + to_deactivate_user_name + "';") - - if int(owned_by_wrong_user) > 0: - owned_objects = self.execute_pg_rds_sql(superuser_name, superuser_password, - "SELECT c.relname FROM pg_class c, pg_user u WHERE c.relowner = u.usesysid " + - "and u.usename='" + to_deactivate_user_name + "';") - raise AssertionError( - "There are still objects owned by the user to be deactivated:", owned_objects,to_deactivate_user_name) - - connections = self.execute_pg_rds_sql(superuser_name, superuser_password, - "SELECT count(*) FROM pg_stat_activity WHERE application_name = " + - "'PostgreSQL JDBC Driver' AND usename = '" + to_deactivate_user_name + "';") - if int(connections) > 0: - raise AssertionError("User is still connected.") - - self.execute_pg_rds_sql(superuser_name, superuser_password, - "ALTER ROLE " + to_deactivate_user_name + " WITH NOLOGIN NOCREATEROLE;") - print('deactivated user:', to_deactivate_user_name) - - def change_owned_objects(self, gopass_path_superuser, to_deactivate_user_name, owner): - superuser_name = gopass_field_from_path(gopass_path_superuser, 'user') - superuser_password = gopass_password_from_path(gopass_path_superuser) - - alter_objects = f"""SELECT 'ALTER TABLE ' || c.relname || ' OWNER TO {owner};' - FROM pg_class c, pg_user u - WHERE c.relowner = u.usesysid - and c.relkind = 'r' - and u.usename='{to_deactivate_user_name}' - UNION - SELECT 'ALTER INDEX ' || c.relname || ' OWNER TO {owner};' - FROM pg_class c, pg_user u - WHERE c.relowner = u.usesysid - and c.relkind = 'i' - and c.relname not like 'pg_toast%' - and u.usename='{to_deactivate_user_name}' - UNION - SELECT 'ALTER SEQUENCE ' || c.relname || ' OWNER TO {owner};' - FROM pg_class c, pg_user u - WHERE c.relowner = u.usesysid - and c.relkind = 'S' - and u.usename='{to_deactivate_user_name}';""" - - alter_stmt = self.execute_pg_rds_sql(superuser_name, superuser_password, alter_objects) - alter_stmt.strip() - - if alter_stmt != '': - print('apply alter statements? \n', alter_stmt) - proceed = input('\n[y/n] \n') - if proceed == 'y': - self.execute_pg_rds_sql(superuser_name, superuser_password, alter_stmt) diff --git a/src/main/python/ddadevops/exoscale_mixin.py b/src/main/python/ddadevops/exoscale_mixin.py deleted file mode 100644 index ff83690..0000000 --- a/src/main/python/ddadevops/exoscale_mixin.py +++ /dev/null @@ -1,31 +0,0 @@ -from .devops_terraform_build import DevopsTerraformBuild - - -def add_exoscale_mixin_config(config, exoscale_api_key, exoscale_secret_key): - config.update({'ExoscaleMixin': - {'exoscale_api_key': exoscale_api_key, - 'exoscale_secret_key': exoscale_secret_key}}) - return config - - -class ExoscaleMixin(DevopsTerraformBuild): - - def __init__(self, project, config): - super().__init__(project, config) - exoscale_mixin_config = config['ExoscaleMixin'] - self.exoscale_api_key = exoscale_mixin_config['exoscale_api_key'] - self.exoscale_secret_key = exoscale_mixin_config['exoscale_secret_key'] - - def project_vars(self): - ret = super().project_vars() - if self.exoscale_api_key: - ret['exoscale_api_key'] = self.exoscale_api_key - if self.exoscale_secret_key: - ret['exoscale_secret_key'] = self.exoscale_secret_key - return ret - - def copy_build_resources_from_package(self): - super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package('provider_registry.tf') - self.copy_build_resource_file_from_package('exoscale_provider.tf') - self.copy_build_resource_file_from_package('exoscale_mixin_vars.tf') From f8c6af8fd19dc99c29cf680721515b658e6bdae3 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 10:08:06 +0200 Subject: [PATCH 121/173] fix init.py --- build.py | 2 +- src/main/python/ddadevops/__init__.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/build.py b/build.py index 0c579c0..572aec0 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev44" +version = "4.0.0-dev45" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index e73b38f..c856a40 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -6,11 +6,9 @@ terraform, dda-pallet, aws & hetzner-cloud. from .python_util import execute from .provs_k3s_mixin import ProvsK3sMixin, add_provs_k3s_mixin_config -from .aws_rds_pg_mixin import AwsRdsPgMixin, add_aws_rds_pg_mixin_config from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config from .c4k_build import C4kBuild, add_c4k_mixin_config -from .exoscale_mixin import ExoscaleMixin, add_exoscale_mixin_config from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config From 61dfa8266132877b076d0395880d9385c6b8895e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 10:46:11 +0200 Subject: [PATCH 122/173] initial try --- build.py | 2 +- src/main/python/ddadevops/devops_terraform_build.py | 6 +++--- src/main/python/ddadevops/provs_k3s_mixin.py | 9 ++++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/build.py b/build.py index 572aec0..5546503 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev45" +version = "4.0.0-dev47" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index a039d51..37cbc0d 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -44,13 +44,13 @@ def create_devops_terraform_build_config(stage, class DevopsTerraformBuild(DevopsBuild): def __init__(self, project, config): - inp = {} + inp = config.copy() inp["name"]=project.name inp["module"]=config.get("module") inp["stage"]=config.get("stage") inp["project_root_path"]=config.get("project_root_path") - inp["build_types"]=[] - inp["mixin_types"]=[] + inp["build_types"]=config.get("build_types", []) + inp["mixin_types"]=config.get("mixin_types", []) super().__init__(project, inp) project.build_depends_on('dda-python-terraform') self.additional_vars = config['additional_vars'] diff --git a/src/main/python/ddadevops/provs_k3s_mixin.py b/src/main/python/ddadevops/provs_k3s_mixin.py index 1a55c02..028687c 100644 --- a/src/main/python/ddadevops/provs_k3s_mixin.py +++ b/src/main/python/ddadevops/provs_k3s_mixin.py @@ -58,7 +58,14 @@ def add_provs_k3s_mixin_config(config, class ProvsK3sMixin(DevopsBuild): def __init__(self, project, config): - super().__init__(project, config) + inp = config.copy() + inp["name"]=project.name + inp["module"]=config.get("module") + inp["stage"]=config.get("stage") + inp["project_root_path"]=config.get("project_root_path") + inp["build_types"]=config.get("build_types", []) + inp["mixin_types"]=config.get("mixin_types", []) + super().__init__(project, inp) provs_k3s_mixin_config = config['ProvsK3sMixin'] self.fqdn = provs_k3s_mixin_config['fqdn'] self.put('fqdn', self.fqdn) From 0f212782d5bee1a063a12ea5e100fbe0f2535d92 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 11:08:34 +0200 Subject: [PATCH 123/173] introduce k3s --- doc/architecture/Domain.md | 14 +++++++++++++- src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 1 + src/main/python/ddadevops/domain/devops_factory.py | 3 +++ src/main/python/ddadevops/domain/provs_k3s.py | 10 ++++++++++ src/test/python/domain/helper.py | 2 +- src/test/python/domain/test_provs_k3s.py | 13 +++++++++++++ 7 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 src/main/python/ddadevops/domain/provs_k3s.py create mode 100644 src/test/python/domain/test_provs_k3s.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 9a59a4c..fa18bc7 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -25,6 +25,16 @@ classDiagram c4k_executabel_name c4k_mixin_config c4k_mixin_auth + c4k_grafana_cloud_user + c4k_grafana_cloud_password + } + + class ProvsK3s { + k3s_provision_user + k3s_letsencrypt_email + k3s_letsencrypt_endpoint + k3s_enable_echo + k3s_app_filename_to_provision } class DnsRecord { @@ -71,11 +81,13 @@ classDiagram Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds + Devops *-- "0..1" ProvsK3s: spcialized_builds Devops *-- "0..1" Release: mixins Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version - C4k *-- DnsRecord + C4k *-- DnsRecord: dns_record + ProvsK3s *-- DnsRecord: provision_dns Credentials *-- "0..n" CredentialMapping: mappings ``` diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 0378707..022e38f 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -2,6 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k +from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType from .version import Version diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 3eb17a2..946c1c3 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -9,6 +9,7 @@ def filter_none(list_to_filter): class BuildType(Enum): IMAGE = 0 C4K = 1 + K3S = 2 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 48d1089..556348c 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -2,6 +2,7 @@ from typing import List, Optional, Dict from .common import Validateable, Devops, BuildType, MixinType from .image import Image from .c4k import C4k +from .provs_k3s import K3s from .release import Release from .version import Version @@ -19,6 +20,8 @@ class DevopsFactory: specialized_builds[BuildType.IMAGE] = Image(inp) if BuildType.C4K in build_types: specialized_builds[BuildType.C4K] = C4k(inp) + if BuildType.K3S in build_types: + specialized_builds[BuildType.K3S] = K3s(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/provs_k3s.py b/src/main/python/ddadevops/domain/provs_k3s.py new file mode 100644 index 0000000..9a746d4 --- /dev/null +++ b/src/main/python/ddadevops/domain/provs_k3s.py @@ -0,0 +1,10 @@ +from typing import List, Optional +from .common import ( + Validateable, + DnsRecord, + Devops, +) + +class K3s(Validateable): + def __init__(self, inp: dict): + pass \ No newline at end of file diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 3c6fdd0..d9cc9cf 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -9,7 +9,7 @@ def devops_config(overrides: dict) -> dict: "stage": "test", "project_root_path": "root_path", "build_dir_name": "target", - "build_types": ["IMAGE", "C4K"], + "build_types": ["IMAGE", "C4K", "K3S"], "mixin_types": ["RELEASE"], "image_dockerhub_user": "dockerhub_user", "image_dockerhub_password": "dockerhub_password", diff --git a/src/test/python/domain/test_provs_k3s.py b/src/test/python/domain/test_provs_k3s.py new file mode 100644 index 0000000..be81e0e --- /dev/null +++ b/src/test/python/domain/test_provs_k3s.py @@ -0,0 +1,13 @@ +import pytest +from pathlib import Path +from src.main.python.ddadevops.domain import ( + DnsRecord, + BuildType, + K3s +) +from .helper import build_devops + + +def test_creation(): + sut = build_devops({}) + assert BuildType.K3S in sut.specialized_builds From 643602e5b733d69039194d159d3292fe8b66eb15 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 11:54:34 +0200 Subject: [PATCH 124/173] init now will work --- src/main/python/ddadevops/domain/provs_k3s.py | 52 ++++++++++++++++++- src/main/python/ddadevops/provs_k3s_mixin.py | 24 ++------- src/test/python/domain/helper.py | 8 +++ src/test/python/domain/test_provs_k3s.py | 5 ++ 4 files changed, 67 insertions(+), 22 deletions(-) diff --git a/src/main/python/ddadevops/domain/provs_k3s.py b/src/main/python/ddadevops/domain/provs_k3s.py index 9a746d4..0cc3395 100644 --- a/src/main/python/ddadevops/domain/provs_k3s.py +++ b/src/main/python/ddadevops/domain/provs_k3s.py @@ -1,10 +1,60 @@ from typing import List, Optional +from string import Template from .common import ( Validateable, DnsRecord, Devops, ) +CONFIG_BASE = """ +fqdn: $fqdn +""" +CONFIG_IPV4 = """node: + ipv4: $ipv4 +""" +CONFIG_IPV6 = """ ipv6: $ipv6 +""" +CONFIG_CERTMANAGER = """certmanager: + email: $letsencrypt_email + letsencryptEndpoint: $letsencrypt_endpoint +""" +CONFIG_ECHO = """echo: $echo +""" + class K3s(Validateable): def __init__(self, inp: dict): - pass \ No newline at end of file + self.k3s_provision_user = inp.get("k3s_provision_user") + self.k3s_letsencrypt_email = inp.get("k3s_letsencrypt_email") + self.k3s_letsencrypt_endpoint = inp.get("k3s_letsencrypt_endpoint") + self.k3s_app_filename_to_provision = inp.get("k3s_app_filename_to_provision", "provs") + fqdn = inp.get("k3_fqdn") + ipv4 = inp.get("k3_ipv4") + ipv6 = inp.get("k3_ipv6") + self.provision_dns = DnsRecord(fqdn, ipv4=ipv4, ipv6=ipv6) + self.k3s_enable_echo = inp.get("k3s_enable_echo", False) + self.k3s_provs_template = inp.get("k3s_provs_template", None) + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("k3s_provision_user") + result += self.__validate_is_not_empty__("k3s_letsencrypt_email") + result += self.__validate_is_not_empty__("k3s_letsencrypt_endpoint") + result += self.__validate_is_not_empty__("k3s_app_filename_to_provision") + if self.provision_dns: + result += self.provision_dns.validate() + return result + + def config_template(self) -> str: + template_text = self.k3s_provs_template + if template_text is None: + template_text = CONFIG_BASE + if self.k3s_letsencrypt_endpoint is not None: + template_text += CONFIG_CERTMANAGER + if self.k3s_enable_echo is not None: + template_text += CONFIG_ECHO + if self.provision_dns.ipv4 is not None: + template_text += CONFIG_IPV4 + if self.provision_dns.ipv6 is not None: + template_text += CONFIG_IPV6 + return template_text + \ No newline at end of file diff --git a/src/main/python/ddadevops/provs_k3s_mixin.py b/src/main/python/ddadevops/provs_k3s_mixin.py index 028687c..65c1e71 100644 --- a/src/main/python/ddadevops/provs_k3s_mixin.py +++ b/src/main/python/ddadevops/provs_k3s_mixin.py @@ -66,27 +66,9 @@ class ProvsK3sMixin(DevopsBuild): inp["build_types"]=config.get("build_types", []) inp["mixin_types"]=config.get("mixin_types", []) super().__init__(project, inp) - provs_k3s_mixin_config = config['ProvsK3sMixin'] - self.fqdn = provs_k3s_mixin_config['fqdn'] - self.put('fqdn', self.fqdn) - self.provision_user = provs_k3s_mixin_config['provision_user'] - self.put('provision_user', self.provision_user) - self.ipv4 = provs_k3s_mixin_config['ipv4'] - self.put('ipv4', self.ipv4) - self.ipv6 = provs_k3s_mixin_config['ipv6'] - self.put('ipv6', self.ipv6) - self.letsencrypt_email = provs_k3s_mixin_config['letsencrypt_email'] - self.put('letsencrypt_email', self.letsencrypt_email) - self.letsencrypt_endpoint = provs_k3s_mixin_config['letsencrypt_endpoint'] - self.put('letsencrypt_endpoint', self.letsencrypt_endpoint) - self.echo = provs_k3s_mixin_config['echo'] - self.put('echo', self.echo) - self.k3s_config_template_text = provs_k3s_mixin_config['k3s_config_template'] - self.k3s_config_template = Template( - provs_k3s_mixin_config['k3s_config_template']) - self.put('k3s_config_template', self.k3s_config_template) - self.app_filename_to_provision = provs_k3s_mixin_config['app_filename_to_provision'] - self.put('app_filename_to_provision', self.app_filename_to_provision) + devops = self.devops_repo.get_devops(self.project) + if BuildType.K3S not in devops.specialized_builds: + raise ValueError("K3SBuild requires BuildType.K3S") def update_runtime_config(self, fqdn, ipv4, ipv6=None): self.fqdn = fqdn diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index d9cc9cf..ba53370 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -19,6 +19,14 @@ def devops_config(overrides: dict) -> dict: "c4k_grafana_cloud_password": "password", "c4k_grafana_cloud_url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", "c4k_auth": {}, + "k3s_provision_user": "k3s_provision_user", + "k3s_letsencrypt_email": "k3s_letsencrypt_email", + "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", + "k3s_enable_echo": False, + "k3s_app_filename_to_provision": "provs", + "k3_fqdn": "example.org", + "k3_ipv4": "1.2.3.4", + "k3_ipv6": "::1", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_provs_k3s.py b/src/test/python/domain/test_provs_k3s.py index be81e0e..e4693b1 100644 --- a/src/test/python/domain/test_provs_k3s.py +++ b/src/test/python/domain/test_provs_k3s.py @@ -11,3 +11,8 @@ from .helper import build_devops def test_creation(): sut = build_devops({}) assert BuildType.K3S in sut.specialized_builds + assert sut.specialized_builds[BuildType.K3S] + +def test_should_calculate_template(): + sut = build_devops({}).specialized_builds[BuildType.K3S] + assert "fqdn:" in sut.config_template() From fc58f2e80788fa60343c6af10f847b5ce1386a6a Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 12:51:35 +0200 Subject: [PATCH 125/173] provs config now is done by domain object --- doc/architecture/BuildAndMixins.md | 6 +-- src/main/python/ddadevops/__init__.py | 2 +- src/main/python/ddadevops/domain/c4k.py | 2 +- src/main/python/ddadevops/domain/provs_k3s.py | 39 ++++++++++++++----- ...{provs_k3s_mixin.py => provs_k3s_build.py} | 37 ++++++------------ src/test/python/domain/helper.py | 3 -- src/test/python/domain/test_c4k.py | 4 +- src/test/python/domain/test_provs_k3s.py | 8 +++- 8 files changed, 53 insertions(+), 48 deletions(-) rename src/main/python/ddadevops/{provs_k3s_mixin.py => provs_k3s_build.py} (72%) diff --git a/doc/architecture/BuildAndMixins.md b/doc/architecture/BuildAndMixins.md index 75090a1..b4280cf 100644 --- a/doc/architecture/BuildAndMixins.md +++ b/doc/architecture/BuildAndMixins.md @@ -72,8 +72,8 @@ classDiagram tag_and_push_release() } - class ProvsK3sMixin { - // ProvsK3sMixin -> ProvsK3sBuild + class ProvsK3sBuild { + // ProvsK3sBuild -> ProvsK3sBuild def update_runtime_config(fqdn, ipv4, ipv6=None) write_provs_config() provs_apply(dry_run=False) @@ -95,7 +95,7 @@ classDiagram DevopsTerraformBuild <|-- DigitaloceanTerraformBuild DevopsTerraformBuild <|--ExoscaleMixin DevopsTerraformBuild <|--HetznerMixin - DevopsBuild <|-- ProvsK3sMixin + DevopsBuild <|-- ProvsK3sBuild DigitaloceanTerraformBuild <|-- DigitaloceanBackendPropertiesMixin AwsBackendPropertiesMixin <|-- AwsMfaMixin diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index c856a40..aab35f6 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -5,7 +5,7 @@ terraform, dda-pallet, aws & hetzner-cloud. """ from .python_util import execute -from .provs_k3s_mixin import ProvsK3sMixin, add_provs_k3s_mixin_config +from .provs_k3s_build import ProvsK3sBuild, add_provs_k3s_mixin_config from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config from .c4k_build import C4kBuild, add_c4k_mixin_config diff --git a/src/main/python/ddadevops/domain/c4k.py b/src/main/python/ddadevops/domain/c4k.py index 16b8760..e5cb060 100644 --- a/src/main/python/ddadevops/domain/c4k.py +++ b/src/main/python/ddadevops/domain/c4k.py @@ -21,9 +21,9 @@ class C4k(Validateable): self.c4k_grafana_cloud_password = inp.get('c4k_grafana_cloud_password') self.dns_record: Optional[DnsRecord] = None - # TODO: these functions should be located at TerraformBuild later on. def update_runtime_config(self, dns_record: DnsRecord): self.dns_record = dns_record + self.throw_if_invalid() def validate(self) -> List[str]: result = [] diff --git a/src/main/python/ddadevops/domain/provs_k3s.py b/src/main/python/ddadevops/domain/provs_k3s.py index 0cc3395..6170bd8 100644 --- a/src/main/python/ddadevops/domain/provs_k3s.py +++ b/src/main/python/ddadevops/domain/provs_k3s.py @@ -21,18 +21,16 @@ CONFIG_CERTMANAGER = """certmanager: CONFIG_ECHO = """echo: $echo """ + class K3s(Validateable): def __init__(self, inp: dict): self.k3s_provision_user = inp.get("k3s_provision_user") self.k3s_letsencrypt_email = inp.get("k3s_letsencrypt_email") self.k3s_letsencrypt_endpoint = inp.get("k3s_letsencrypt_endpoint") - self.k3s_app_filename_to_provision = inp.get("k3s_app_filename_to_provision", "provs") - fqdn = inp.get("k3_fqdn") - ipv4 = inp.get("k3_ipv4") - ipv6 = inp.get("k3_ipv6") - self.provision_dns = DnsRecord(fqdn, ipv4=ipv4, ipv6=ipv6) - self.k3s_enable_echo = inp.get("k3s_enable_echo", False) + self.k3s_app_filename_to_provision = inp.get("k3s_app_filename_to_provision") + self.k3s_enable_echo = inp.get("k3s_enable_echo", "false") self.k3s_provs_template = inp.get("k3s_provs_template", None) + self.provision_dns: Optional[DnsRecord] = None def validate(self) -> List[str]: result = [] @@ -44,7 +42,29 @@ class K3s(Validateable): result += self.provision_dns.validate() return result - def config_template(self) -> str: + def update_runtime_config(self, dns_record: DnsRecord): + self.provision_dns = dns_record + self.throw_if_invalid() + + def provs_config(self) -> str: + if not self.provision_dns: + raise ValueError("provision_dns was not set.") + substitutes = { + "fqdn": self.provision_dns.fqdn, + } + if self.provision_dns.ipv4 is not None: + substitutes["ipv4"] = self.provision_dns.ipv4 + if self.provision_dns.ipv6 is not None: + substitutes["ipv6"] = self.provision_dns.ipv6 + if self.k3s_letsencrypt_email is not None: + substitutes["letsencrypt_email"] = self.k3s_letsencrypt_email + if self.k3s_letsencrypt_endpoint is not None: + substitutes["letsencrypt_endpoint"] = self.k3s_letsencrypt_endpoint + if self.k3s_enable_echo is not None: + substitutes["echo"] = self.k3s_enable_echo + return self.__config_template__().substitute(substitutes) + + def __config_template__(self) -> Template: template_text = self.k3s_provs_template if template_text is None: template_text = CONFIG_BASE @@ -54,7 +74,6 @@ class K3s(Validateable): template_text += CONFIG_ECHO if self.provision_dns.ipv4 is not None: template_text += CONFIG_IPV4 - if self.provision_dns.ipv6 is not None: + if self.provision_dns.ipv6 is not None: template_text += CONFIG_IPV6 - return template_text - \ No newline at end of file + return Template(template_text) diff --git a/src/main/python/ddadevops/provs_k3s_mixin.py b/src/main/python/ddadevops/provs_k3s_build.py similarity index 72% rename from src/main/python/ddadevops/provs_k3s_mixin.py rename to src/main/python/ddadevops/provs_k3s_build.py index 65c1e71..386b3c1 100644 --- a/src/main/python/ddadevops/provs_k3s_mixin.py +++ b/src/main/python/ddadevops/provs_k3s_build.py @@ -1,9 +1,11 @@ from string import Template import deprecation from .python_util import execute_live +from .domain import DnsRecord from .devops_build import DevopsBuild + CONFIG_BASE = """ fqdn: $fqdn """ @@ -42,7 +44,7 @@ def add_provs_k3s_mixin_config(config, if ipv6 is not None: template_text += CONFIG_IPV6 - config.update({'ProvsK3sMixin': + config.update({'ProvsK3sBuild': {'fqdn': fqdn, 'provision_user': provision_user, 'ipv4': ipv4, @@ -55,7 +57,7 @@ def add_provs_k3s_mixin_config(config, return config -class ProvsK3sMixin(DevopsBuild): +class ProvsK3sBuild(DevopsBuild): def __init__(self, project, config): inp = config.copy() @@ -70,33 +72,16 @@ class ProvsK3sMixin(DevopsBuild): if BuildType.K3S not in devops.specialized_builds: raise ValueError("K3SBuild requires BuildType.K3S") - def update_runtime_config(self, fqdn, ipv4, ipv6=None): - self.fqdn = fqdn - self.put('fqdn', fqdn) - self.ipv4 = ipv4 - self.put('ipv4', ipv4) - self.ipv6 = ipv6 - self.put('ipv6', ipv6) - template_text = self.k3s_config_template_text - if ipv4 is not None: - template_text += CONFIG_IPV4 - if ipv6 is not None: - template_text += CONFIG_IPV6 - self.k3s_config_template_text = template_text - self.put('k3s_config_template_text', template_text) - template = Template(template_text) - self.k3s_config_template = template - self.put('k3s_config_template', template) + def update_runtime_config(self, dns_record: DnsRecord): + devops = self.devops_repo.get_devops(self.project) + devops.specialized_builds[BuildType.K3S].update_runtime_config(dns_record) + self.devops_repo.set_devops(self.project, devops) def write_provs_config(self): - substitutes = self.get_keys(['fqdn', 'ipv4', 'ipv6', 'letsencrypt_email', - 'letsencrypt_endpoint', 'echo']) + devops = self.devops_repo.get_devops(self.project) + k3s = devops.specialized_builds[BuildType.K3S] with open(self.build_path() + '/out_k3sServerConfig.yaml', "w", encoding="utf-8") as output_file: - output_file.write(self.k3s_config_template.substitute(substitutes)) - - @deprecation.deprecated(deprecated_in="3.1") - def provs_server(self, dry_run=False): - self.provs_apply(dry_run) + output_file.write(k3s.provs_config()) def provs_apply(self, dry_run=False): cmd = ['provs-server.jar', 'k3s', self.provision_user + '@' + self.fqdn, '-c', diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index ba53370..a966272 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -24,9 +24,6 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": False, "k3s_app_filename_to_provision": "provs", - "k3_fqdn": "example.org", - "k3_ipv4": "1.2.3.4", - "k3_ipv6": "::1", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_c4k.py b/src/test/python/domain/test_c4k.py index bc69f1c..029b829 100644 --- a/src/test/python/domain/test_c4k.py +++ b/src/test/python/domain/test_c4k.py @@ -20,7 +20,7 @@ def test_c4k_should_calculate_config(): sut = build_devops({}) c4k = sut.specialized_builds[BuildType.C4K] - c4k.update_runtime_config(DnsRecord("fqdn")) + c4k.update_runtime_config(DnsRecord("fqdn", ipv6="::1")) assert { "fqdn": "fqdn", "mon-cfg": { @@ -36,7 +36,7 @@ def test_c4k_should_calculate_config(): } ) c4k = sut.specialized_builds[BuildType.C4K] - c4k.update_runtime_config(DnsRecord("fqdn")) + c4k.update_runtime_config(DnsRecord("fqdn", ipv6="::1")) assert { "test": "test", "fqdn": "fqdn", diff --git a/src/test/python/domain/test_provs_k3s.py b/src/test/python/domain/test_provs_k3s.py index e4693b1..bbde9f0 100644 --- a/src/test/python/domain/test_provs_k3s.py +++ b/src/test/python/domain/test_provs_k3s.py @@ -13,6 +13,10 @@ def test_creation(): assert BuildType.K3S in sut.specialized_builds assert sut.specialized_builds[BuildType.K3S] -def test_should_calculate_template(): +def test_should_calculate_provs_config(): sut = build_devops({}).specialized_builds[BuildType.K3S] - assert "fqdn:" in sut.config_template() + sut.update_runtime_config( + DnsRecord("example.org", ipv6="::1") + ) + assert "fqdn:" in sut.provs_config() + assert not "$" in sut.provs_config() From 0e28cbd52d4814a60d143b01bed831f8e670374c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 13:23:10 +0200 Subject: [PATCH 126/173] provs_k3s now migth work --- src/main/python/ddadevops/__init__.py | 6 +- src/main/python/ddadevops/domain/provs_k3s.py | 14 +++ src/main/python/ddadevops/provs_k3s_build.py | 85 +++---------------- src/test/python/domain/helper.py | 2 +- src/test/python/domain/test_provs_k3s.py | 27 ++++-- 5 files changed, 51 insertions(+), 83 deletions(-) diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index aab35f6..7cde593 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -5,14 +5,14 @@ terraform, dda-pallet, aws & hetzner-cloud. """ from .python_util import execute -from .provs_k3s_build import ProvsK3sBuild, add_provs_k3s_mixin_config +from .provs_k3s_build import ProvsK3sBuild from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config -from .c4k_build import C4kBuild, add_c4k_mixin_config +from .c4k_build import C4kBuild from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config -from .devops_image_build import DevopsImageBuild, create_devops_docker_build_config +from .devops_image_build import DevopsImageBuild from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build from .credential import gopass_password_from_path, gopass_field_from_path diff --git a/src/main/python/ddadevops/domain/provs_k3s.py b/src/main/python/ddadevops/domain/provs_k3s.py index 6170bd8..5e809d9 100644 --- a/src/main/python/ddadevops/domain/provs_k3s.py +++ b/src/main/python/ddadevops/domain/provs_k3s.py @@ -64,6 +64,20 @@ class K3s(Validateable): substitutes["echo"] = self.k3s_enable_echo return self.__config_template__().substitute(substitutes) + def command(self, devops: Devops): + if not self.provision_dns: + raise ValueError("provision_dns was not set.") + cmd = [ + "provs-server.jar", + "k3s", + f"{self.k3s_provision_user}@{self.provision_dns.fqdn}", + "-c", + f"{devops.build_path()}/out_k3sServerConfig.yaml", + "-a", + f"{devops.build_path()}/{self.k3s_app_filename_to_provision}", + ] + return " ".join(cmd) + def __config_template__(self) -> Template: template_text = self.k3s_provs_template if template_text is None: diff --git a/src/main/python/ddadevops/provs_k3s_build.py b/src/main/python/ddadevops/provs_k3s_build.py index 386b3c1..190434a 100644 --- a/src/main/python/ddadevops/provs_k3s_build.py +++ b/src/main/python/ddadevops/provs_k3s_build.py @@ -1,73 +1,18 @@ -from string import Template -import deprecation -from .python_util import execute_live from .domain import DnsRecord +from .infrastructure import ExecutionApi from .devops_build import DevopsBuild - - -CONFIG_BASE = """ -fqdn: $fqdn -""" -CONFIG_IPV4 = """node: - ipv4: $ipv4 -""" -CONFIG_IPV6 = """ ipv6: $ipv6 -""" -CONFIG_CERTMANAGER = """certmanager: - email: $letsencrypt_email - letsencryptEndpoint: $letsencrypt_endpoint -""" -CONFIG_ECHO = """echo: $echo -""" - - -def add_provs_k3s_mixin_config(config, - provision_user='root', - echo=None, - k3s_config_template=None, - letsencrypt_email=None, - letsencrypt_endpoint=None, - fqdn=None, - ipv4=None, - ipv6=None, - app_filename_to_provision=None): - template_text = k3s_config_template - if template_text is None: - template_text = CONFIG_BASE - if letsencrypt_endpoint is not None: - template_text += CONFIG_CERTMANAGER - if echo is not None: - template_text += CONFIG_ECHO - if ipv4 is not None: - template_text += CONFIG_IPV4 - if ipv6 is not None: - template_text += CONFIG_IPV6 - - config.update({'ProvsK3sBuild': - {'fqdn': fqdn, - 'provision_user': provision_user, - 'ipv4': ipv4, - 'ipv6': ipv6, - 'letsencrypt_email': letsencrypt_email, - 'letsencrypt_endpoint': letsencrypt_endpoint, - 'echo': echo, - 'k3s_config_template': template_text, - 'app_filename_to_provision': app_filename_to_provision}}) - return config - - class ProvsK3sBuild(DevopsBuild): - def __init__(self, project, config): inp = config.copy() - inp["name"]=project.name - inp["module"]=config.get("module") - inp["stage"]=config.get("stage") - inp["project_root_path"]=config.get("project_root_path") - inp["build_types"]=config.get("build_types", []) - inp["mixin_types"]=config.get("mixin_types", []) + inp["name"] = project.name + inp["module"] = config.get("module") + inp["stage"] = config.get("stage") + inp["project_root_path"] = config.get("project_root_path") + inp["build_types"] = config.get("build_types", []) + inp["mixin_types"] = config.get("mixin_types", []) super().__init__(project, inp) + self.execution_api = ExecutionApi() devops = self.devops_repo.get_devops(self.project) if BuildType.K3S not in devops.specialized_builds: raise ValueError("K3SBuild requires BuildType.K3S") @@ -80,14 +25,12 @@ class ProvsK3sBuild(DevopsBuild): def write_provs_config(self): devops = self.devops_repo.get_devops(self.project) k3s = devops.specialized_builds[BuildType.K3S] - with open(self.build_path() + '/out_k3sServerConfig.yaml', "w", encoding="utf-8") as output_file: + with open( + self.build_path() + "/out_k3sServerConfig.yaml", "w", encoding="utf-8" + ) as output_file: output_file.write(k3s.provs_config()) def provs_apply(self, dry_run=False): - cmd = ['provs-server.jar', 'k3s', self.provision_user + '@' + self.fqdn, '-c', - self.build_path() + '/out_k3sServerConfig.yaml', - '-a', self.build_path() + '/' + self.app_filename_to_provision] - if dry_run: - print(" ".join(cmd)) - else: - execute_live(cmd) + devops = self.devops_repo.get_devops(self.project) + k3s = devops.specialized_builds[BuildType.K3S] + self.execution_api.execute_live(k3s.command(devops), dry_run=dry_run) diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index a966272..dc83665 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -23,7 +23,7 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_email": "k3s_letsencrypt_email", "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": False, - "k3s_app_filename_to_provision": "provs", + "k3s_app_filename_to_provision": "k3s_app.yaml", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_provs_k3s.py b/src/test/python/domain/test_provs_k3s.py index bbde9f0..12f3203 100644 --- a/src/test/python/domain/test_provs_k3s.py +++ b/src/test/python/domain/test_provs_k3s.py @@ -1,10 +1,6 @@ import pytest from pathlib import Path -from src.main.python.ddadevops.domain import ( - DnsRecord, - BuildType, - K3s -) +from src.main.python.ddadevops.domain import DnsRecord, BuildType, K3s from .helper import build_devops @@ -13,10 +9,25 @@ def test_creation(): assert BuildType.K3S in sut.specialized_builds assert sut.specialized_builds[BuildType.K3S] + def test_should_calculate_provs_config(): sut = build_devops({}).specialized_builds[BuildType.K3S] - sut.update_runtime_config( - DnsRecord("example.org", ipv6="::1") - ) + sut.update_runtime_config(DnsRecord("example.org", ipv6="::1")) assert "fqdn:" in sut.provs_config() assert not "$" in sut.provs_config() + + +def test_should_calculate_command(): + devops = build_devops({}) + sut = devops.specialized_builds[BuildType.K3S] + sut.update_runtime_config(DnsRecord("example.org", ipv6="::1")) + assert ( + "provs-server.jar " + + "k3s " + + "k3s_provision_user@example.org " + + "-c " + + "root_path/target/name/module/out_k3sServerConfig.yaml " + + "-a " + + "root_path/target/name/module/k3s_app.yaml" + == sut.command(devops) + ) From c2fee0f52e22d8242ad0efc837768ec246a2c2f6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 13:37:14 +0200 Subject: [PATCH 127/173] fix linting --- .../ddadevops/aws_backend_properties_mixin.py | 35 ++++---- src/main/python/ddadevops/c4k_build.py | 1 + .../digitalocean_backend_properties_mixin.py | 80 ++++++++++--------- .../infrastructure/infrastructure.py | 15 ++-- src/main/python/ddadevops/provs_k3s_build.py | 3 +- 5 files changed, 72 insertions(+), 62 deletions(-) diff --git a/src/main/python/ddadevops/aws_backend_properties_mixin.py b/src/main/python/ddadevops/aws_backend_properties_mixin.py index 841b519..ef6cebe 100644 --- a/src/main/python/ddadevops/aws_backend_properties_mixin.py +++ b/src/main/python/ddadevops/aws_backend_properties_mixin.py @@ -1,33 +1,33 @@ from dda_python_terraform import Terraform from .devops_terraform_build import DevopsTerraformBuild + def add_aws_backend_properties_mixin_config(config, account_name): - config.update({'AwsBackendPropertiesMixin': - {'account_name': account_name}}) + config.update({"AwsBackendPropertiesMixin": {"account_name": account_name}}) return config -class AwsBackendPropertiesMixin(DevopsTerraformBuild): +class AwsBackendPropertiesMixin(DevopsTerraformBuild): def __init__(self, project, config): super().__init__(project, config) - aws_mixin_config = config['AwsBackendPropertiesMixin'] - self.account_name = aws_mixin_config['account_name'] - self.backend_config = "backend." + self.account_name + "." + self.stage + ".properties" + aws_mixin_config = config["AwsBackendPropertiesMixin"] + self.account_name = aws_mixin_config["account_name"] + self.backend_config = ( + "backend." + self.account_name + "." + self.stage + ".properties" + ) self.additional_tfvar_files.append(self.backend_config) def project_vars(self): ret = super().project_vars() - ret.update({'account_name': self.account_name}) + ret.update({"account_name": self.account_name}) return ret def copy_build_resources_from_package(self): super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package('provider_registry.tf') - self.copy_build_resource_file_from_package('aws_provider.tf') - self.copy_build_resource_file_from_package( - 'aws_backend_properties_vars.tf') - self.copy_build_resource_file_from_package( - 'aws_backend_with_properties.tf') + self.copy_build_resource_file_from_package("provider_registry.tf") + self.copy_build_resource_file_from_package("aws_provider.tf") + self.copy_build_resource_file_from_package("aws_backend_properties_vars.tf") + self.copy_build_resource_file_from_package("aws_backend_with_properties.tf") def copy_local_state(self): pass @@ -36,14 +36,17 @@ class AwsBackendPropertiesMixin(DevopsTerraformBuild): pass def init_client(self): - terraform = Terraform(working_dir=self.build_path(), terraform_semantic_version=self.terraform_semantic_version) + terraform = Terraform( + working_dir=self.build_path(), + terraform_semantic_version=self.terraform_semantic_version, + ) terraform.init(backend_config=self.backend_config) self.print_terraform_command(terraform) if self.use_workspace: try: - terraform.workspace('select', self.stage) + terraform.workspace("select", self.stage) self.print_terraform_command(terraform) except: - terraform.workspace('new', self.stage) + terraform.workspace("new", self.stage) self.print_terraform_command(terraform) return terraform diff --git a/src/main/python/ddadevops/c4k_build.py b/src/main/python/ddadevops/c4k_build.py index c2f9685..4b95c63 100644 --- a/src/main/python/ddadevops/c4k_build.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -43,6 +43,7 @@ def add_c4k_mixin_config( ) return config + class C4kBuild(DevopsBuild): def __init__(self, project, config): super().__init__(project, config) diff --git a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py index 47526f3..49ac57b 100644 --- a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py +++ b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py @@ -1,55 +1,55 @@ from dda_python_terraform import Terraform from .digitalocean_terraform_build import DigitaloceanTerraformBuild -def add_digitalocean_backend_properties_mixin_config(config, - account_name, - endpoint, - bucket, - key, - region='eu-central-1'): - config.update({'DigitaloceanBackendPropertiesMixin': - {'account_name': account_name, - 'endpoint': endpoint, - 'bucket': bucket, - 'key': key, - 'region': region}}) + +def add_digitalocean_backend_properties_mixin_config( + config, account_name, endpoint, bucket, key, region="eu-central-1" +): + config.update( + { + "DigitaloceanBackendPropertiesMixin": { + "account_name": account_name, + "endpoint": endpoint, + "bucket": bucket, + "key": key, + "region": region, + } + } + ) return config class DigitaloceanBackendPropertiesMixin(DigitaloceanTerraformBuild): - def __init__(self, project, config): super().__init__(project, config) - do_mixin_config = config['DigitaloceanBackendPropertiesMixin'] - self.account_name = do_mixin_config['account_name'] - self.endpoint = do_mixin_config['endpoint'] - self.bucket = do_mixin_config['bucket'] - self.key = do_mixin_config['account_name'] + \ - '/' + do_mixin_config['key'] - self.region = do_mixin_config['region'] + do_mixin_config = config["DigitaloceanBackendPropertiesMixin"] + self.account_name = do_mixin_config["account_name"] + self.endpoint = do_mixin_config["endpoint"] + self.bucket = do_mixin_config["bucket"] + self.key = do_mixin_config["account_name"] + "/" + do_mixin_config["key"] + self.region = do_mixin_config["region"] self.backend_config = { - 'access_key': self.do_spaces_access_id, - 'secret_key': self.do_spaces_secret_key, - 'endpoint': self.endpoint, - 'bucket': self.bucket, - 'key': self.key, - 'region': self.region} + "access_key": self.do_spaces_access_id, + "secret_key": self.do_spaces_secret_key, + "endpoint": self.endpoint, + "bucket": self.bucket, + "key": self.key, + "region": self.region, + } def project_vars(self): ret = super().project_vars() - ret.update({'account_name': self.account_name}) - ret.update({'endpoint': self.endpoint}) - ret.update({'bucket': self.bucket}) - ret.update({'key': self.key}) - ret.update({'region': self.region}) + ret.update({"account_name": self.account_name}) + ret.update({"endpoint": self.endpoint}) + ret.update({"bucket": self.bucket}) + ret.update({"key": self.key}) + ret.update({"region": self.region}) return ret def copy_build_resources_from_package(self): super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package( - 'do_backend_properties_vars.tf') - self.copy_build_resource_file_from_package( - 'do_backend_with_properties.tf') + self.copy_build_resource_file_from_package("do_backend_properties_vars.tf") + self.copy_build_resource_file_from_package("do_backend_with_properties.tf") def copy_local_state(self): pass @@ -58,15 +58,17 @@ class DigitaloceanBackendPropertiesMixin(DigitaloceanTerraformBuild): pass def init_client(self): - terraform = Terraform(working_dir=self.build_path(), - terraform_semantic_version=self.terraform_semantic_version) + terraform = Terraform( + working_dir=self.build_path(), + terraform_semantic_version=self.terraform_semantic_version, + ) terraform.init(backend_config=self.backend_config) self.print_terraform_command(terraform) if self.use_workspace: try: - terraform.workspace('select', self.stage) + terraform.workspace("select", self.stage) self.print_terraform_command(terraform) except: - terraform.workspace('new', self.stage) + terraform.workspace("new", self.stage) self.print_terraform_command(terraform) return terraform diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 7e1df4b..814660e 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -98,12 +98,15 @@ class ExecutionApi: output = output.rstrip() return output - def execute_live(self, command): - process = Popen(command, stdout=PIPE) - for line in iter(process.stdout.readline, b""): - print(line.decode("utf-8"), end="") - process.stdout.close() - process.wait() + def execute_live(self, command, dry_run=False): + if dry_run: + print(command) + else: + process = Popen(command, stdout=PIPE) + for line in iter(process.stdout.readline, b""): + print(line.decode("utf-8"), end="") + process.stdout.close() + process.wait() class EnvironmentApi: diff --git a/src/main/python/ddadevops/provs_k3s_build.py b/src/main/python/ddadevops/provs_k3s_build.py index 190434a..a6baabb 100644 --- a/src/main/python/ddadevops/provs_k3s_build.py +++ b/src/main/python/ddadevops/provs_k3s_build.py @@ -1,7 +1,8 @@ -from .domain import DnsRecord +from .domain import DnsRecord, BuildType from .infrastructure import ExecutionApi from .devops_build import DevopsBuild + class ProvsK3sBuild(DevopsBuild): def __init__(self, project, config): inp = config.copy() From 483d98a3d51836f21735025b7b38e8753b178d55 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 22 May 2023 13:56:53 +0200 Subject: [PATCH 128/173] k3s now works --- build.py | 2 +- src/main/python/ddadevops/domain/provs_k3s.py | 1 - src/test/python/domain/helper.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 5546503..7a5a489 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev47" +version = "4.0.0-dev48" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/domain/provs_k3s.py b/src/main/python/ddadevops/domain/provs_k3s.py index 5e809d9..20e0a1d 100644 --- a/src/main/python/ddadevops/domain/provs_k3s.py +++ b/src/main/python/ddadevops/domain/provs_k3s.py @@ -34,7 +34,6 @@ class K3s(Validateable): def validate(self) -> List[str]: result = [] - result += self.__validate_is_not_empty__("k3s_provision_user") result += self.__validate_is_not_empty__("k3s_letsencrypt_email") result += self.__validate_is_not_empty__("k3s_letsencrypt_endpoint") result += self.__validate_is_not_empty__("k3s_app_filename_to_provision") diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index dc83665..9386531 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -22,7 +22,7 @@ def devops_config(overrides: dict) -> dict: "k3s_provision_user": "k3s_provision_user", "k3s_letsencrypt_email": "k3s_letsencrypt_email", "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", - "k3s_enable_echo": False, + "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", "release_type": "NONE", "release_main_branch": "main", From 15335ae73fb4add3d5c60043f571cb0bf0d4c646 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 08:34:43 +0200 Subject: [PATCH 129/173] introduce terraform --- doc/architecture/Domain.md | 13 + .../ddadevops/devops_terraform_build.py | 226 ++++++++++-------- src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 1 + .../python/ddadevops/domain/devops_factory.py | 3 + src/main/python/ddadevops/domain/terraform.py | 36 +++ src/test/python/domain/helper.py | 12 +- src/test/python/domain/test_terraform.py | 19 ++ 8 files changed, 211 insertions(+), 100 deletions(-) create mode 100644 src/main/python/ddadevops/domain/terraform.py create mode 100644 src/test/python/domain/test_terraform.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index fa18bc7..8b36d39 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -37,6 +37,18 @@ classDiagram k3s_app_filename_to_provision } + class Terraform { + tf_additional_vars + tf_output_json_name + tf_use_workspace + tf_use_package_common_files + tf_build_commons_path + tf_commons_dir_name + tf_debug_print_terraform_command + tf_additional_tfvar_files + tf_terraform_semantic_version + } + class DnsRecord { fqdn ipv4 @@ -82,6 +94,7 @@ classDiagram Devops *-- "0..1" Image: spcialized_builds Devops *-- "0..1" C4k: spcialized_builds Devops *-- "0..1" ProvsK3s: spcialized_builds + Devops *-- "0..1" Terraform: spcialized_builds Devops *-- "0..1" Release: mixins Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 37cbc0d..a8c024c 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -9,94 +9,87 @@ from .python_util import filter_none from .devops_build import DevopsBuild, create_devops_build_config - - -def create_devops_terraform_build_config(stage, - project_root_path, - module, - additional_vars, - build_dir_name='target', - output_json_name=None, - use_workspace=True, - use_package_common_files=True, - build_commons_path=None, - terraform_build_commons_dir_name='terraform', - debug_print_terraform_command=False, - additional_tfvar_files=None, - terraform_semantic_version="1.0.8"): +def create_devops_terraform_build_config( + stage, + project_root_path, + module, + additional_vars, + build_dir_name="target", + output_json_name=None, + use_workspace=True, + use_package_common_files=True, + build_commons_path=None, + terraform_build_commons_dir_name="terraform", + debug_print_terraform_command=False, + additional_tfvar_files=None, + terraform_semantic_version="1.0.8", +): if not output_json_name: - output_json_name = 'out_' + module + '.json' + output_json_name = "out_" + module + ".json" if not additional_tfvar_files: additional_tfvar_files = [] - ret = create_devops_build_config( - stage, project_root_path, module, build_dir_name) - ret.update({'additional_vars': additional_vars, - 'output_json_name': output_json_name, - 'use_workspace': use_workspace, - 'use_package_common_files': use_package_common_files, - 'build_commons_path': build_commons_path, - 'terraform_build_commons_dir_name': terraform_build_commons_dir_name, - 'debug_print_terraform_command': debug_print_terraform_command, - 'additional_tfvar_files': additional_tfvar_files, - 'terraform_semantic_version': terraform_semantic_version}) + ret = create_devops_build_config(stage, project_root_path, module, build_dir_name) + ret.update( + { + "additional_vars": additional_vars, + "output_json_name": output_json_name, + "use_workspace": use_workspace, + "use_package_common_files": use_package_common_files, + "build_commons_path": build_commons_path, + "terraform_build_commons_dir_name": terraform_build_commons_dir_name, + "debug_print_terraform_command": debug_print_terraform_command, + "additional_tfvar_files": additional_tfvar_files, + "terraform_semantic_version": terraform_semantic_version, + } + ) return ret -class DevopsTerraformBuild(DevopsBuild): +class DevopsTerraformBuild(DevopsBuild): def __init__(self, project, config): inp = config.copy() - inp["name"]=project.name - inp["module"]=config.get("module") - inp["stage"]=config.get("stage") - inp["project_root_path"]=config.get("project_root_path") - inp["build_types"]=config.get("build_types", []) - inp["mixin_types"]=config.get("mixin_types", []) + inp["name"] = project.name + inp["module"] = config.get("module") + inp["stage"] = config.get("stage") + inp["project_root_path"] = config.get("project_root_path") + inp["build_types"] = config.get("build_types", []) + inp["mixin_types"] = config.get("mixin_types", []) super().__init__(project, inp) - project.build_depends_on('dda-python-terraform') - self.additional_vars = config['additional_vars'] - self.output_json_name = config['output_json_name'] - self.use_workspace = config['use_workspace'] - self.use_package_common_files = config['use_package_common_files'] - self.build_commons_path = config['build_commons_path'] - self.terraform_build_commons_dir_name = config['terraform_build_commons_dir_name'] - self.debug_print_terraform_command = config['debug_print_terraform_command'] - self.additional_tfvar_files = config['additional_tfvar_files'] - self.terraform_semantic_version = config['terraform_semantic_version'] - self.stage = config["stage"] - self.module = config["module"] + project.build_depends_on("dda-python-terraform") def terraform_build_commons_path(self): - mylist = [self.build_commons_path, - self.terraform_build_commons_dir_name] - return '/'.join(filter_none(mylist)) + '/' + mylist = [self.build_commons_path, self.terraform_build_commons_dir_name] + return "/".join(filter_none(mylist)) + "/" def project_vars(self): - ret = {'stage': self.stage} + ret = {"stage": self.stage} if self.module: - ret['module'] = self.module + ret["module"] = self.module if self.additional_vars: ret.update(self.additional_vars) return ret def copy_build_resource_file_from_package(self, name): - my_data = resource_string( - __name__, "src/main/resources/terraform/" + name) - with open(self.build_path() + '/' + name, "w", encoding="utf-8") as output_file: + my_data = resource_string(__name__, "src/main/resources/terraform/" + name) + with open(self.build_path() + "/" + name, "w", encoding="utf-8") as output_file: output_file.write(my_data.decode(sys.stdout.encoding)) def copy_build_resources_from_package(self): - self.copy_build_resource_file_from_package('versions.tf') - self.copy_build_resource_file_from_package('terraform_build_vars.tf') + self.copy_build_resource_file_from_package("versions.tf") + self.copy_build_resource_file_from_package("terraform_build_vars.tf") def copy_build_resources_from_dir(self): - run('cp -f ' + self.terraform_build_commons_path() + - '* ' + self.build_path(), shell=True, check=False) + run( + "cp -f " + self.terraform_build_commons_path() + "* " + self.build_path(), + shell=True, + check=False, + ) def copy_local_state(self): - run('cp terraform.tfstate ' + self.build_path(), shell=True, check=False) + run("cp terraform.tfstate " + self.build_path(), shell=True, check=False) def rescue_local_state(self): - run('cp ' + self.build_path() + '/terraform.tfstate .', shell=True, check=False) + run("cp " + self.build_path() + "/terraform.tfstate .", shell=True, check=False) def initialize_build_dir(self): super().initialize_build_dir() @@ -105,43 +98,54 @@ class DevopsTerraformBuild(DevopsBuild): else: self.copy_build_resources_from_dir() self.copy_local_state() - run('cp *.tf ' + self.build_path(), shell=True, check=False) - run('cp *.properties ' + self.build_path(), shell=True, check=False) - run('cp *.tfvars ' + self.build_path(), shell=True, check=False) - run('cp -r scripts ' + self.build_path(), shell=True, check=False) + run("cp *.tf " + self.build_path(), shell=True, check=False) + run("cp *.properties " + self.build_path(), shell=True, check=False) + run("cp *.tfvars " + self.build_path(), shell=True, check=False) + run("cp -r scripts " + self.build_path(), shell=True, check=False) def post_build(self): self.rescue_local_state() def init_client(self): - terraform = Terraform(working_dir=self.build_path(), terraform_semantic_version=self.terraform_semantic_version) + terraform = Terraform( + working_dir=self.build_path(), + terraform_semantic_version=self.terraform_semantic_version, + ) terraform.init() self.print_terraform_command(terraform) if self.use_workspace: try: - terraform.workspace('select', self.stage) + terraform.workspace("select", self.stage) self.print_terraform_command(terraform) except: - terraform.workspace('new', self.stage) + terraform.workspace("new", self.stage) self.print_terraform_command(terraform) return terraform def write_output(self, terraform): result = terraform.output(json=IsFlagged) self.print_terraform_command(terraform) - with open(self.build_path() + self.output_json_name, "w", encoding="utf-8") as output_file: + with open( + self.build_path() + self.output_json_name, "w", encoding="utf-8" + ) as output_file: output_file.write(dumps(result)) chmod(self.build_path() + self.output_json_name, 0o600) def read_output_json(self): - with open(self.build_path() + self.output_json_name, 'r', encoding="utf-8") as file: + with open( + self.build_path() + self.output_json_name, "r", encoding="utf-8" + ) as file: return load(file) def plan(self): terraform = self.init_client() - return_code, _, stderr = terraform.plan(detailed_exitcode=None, capture_output=False, raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.plan( + detailed_exitcode=None, + capture_output=False, + raise_on_error=False, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code > 0: @@ -149,9 +153,13 @@ class DevopsTerraformBuild(DevopsBuild): def plan_fail_on_diff(self): terraform = self.init_client() - return_code, _, stderr = terraform.plan(detailed_exitcode=IsFlagged, capture_output=False, raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.plan( + detailed_exitcode=IsFlagged, + capture_output=False, + raise_on_error=False, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code not in (0, 2): @@ -166,15 +174,21 @@ class DevopsTerraformBuild(DevopsBuild): else: auto_approve_flag = None if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.apply(capture_output=False, raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) else: - return_code, _, stderr = terraform.apply(capture_output=False, raise_on_error=True, - skip_plan=auto_approve, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + skip_plan=auto_approve, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.write_output(terraform) self.post_build() self.print_terraform_command(terraform) @@ -184,8 +198,8 @@ class DevopsTerraformBuild(DevopsBuild): def refresh(self): terraform = self.init_client() return_code, _, stderr = terraform.refresh( - var=self.project_vars(), - var_file=self.additional_tfvar_files) + var=self.project_vars(), var_file=self.additional_tfvar_files + ) self.write_output(terraform) self.post_build() self.print_terraform_command(terraform) @@ -199,26 +213,40 @@ class DevopsTerraformBuild(DevopsBuild): else: auto_approve_flag = None if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.destroy(capture_output=False, raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) else: - return_code, _, stderr = terraform.destroy(capture_output=False, raise_on_error=True, - force=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + force=auto_approve_flag, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code > 0: raise RuntimeError(return_code, "terraform error:", stderr) - def tf_import(self, tf_import_name, tf_import_resource,): + def tf_import( + self, + tf_import_name, + tf_import_resource, + ): terraform = self.init_client() - return_code, _, stderr = terraform.import_cmd(tf_import_name, tf_import_resource, - capture_output=False, raise_on_error=True, - var=self.project_vars(), - var_file=self.additional_tfvar_files) + return_code, _, stderr = terraform.import_cmd( + tf_import_name, + tf_import_resource, + capture_output=False, + raise_on_error=True, + var=self.project_vars(), + var_file=self.additional_tfvar_files, + ) self.post_build() self.print_terraform_command(terraform) if return_code > 0: @@ -226,5 +254,5 @@ class DevopsTerraformBuild(DevopsBuild): def print_terraform_command(self, terraform): if self.debug_print_terraform_command: - output = 'cd ' + self.build_path() + ' && ' + terraform.latest_cmd() + output = "cd " + self.build_path() + " && " + terraform.latest_cmd() print(output) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 022e38f..967c0f2 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -2,6 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k +from .terraform import Terraform from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 946c1c3..c270a49 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -10,6 +10,7 @@ class BuildType(Enum): IMAGE = 0 C4K = 1 K3S = 2 + TERRAFORM = 3 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 556348c..d045b50 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -3,6 +3,7 @@ from .common import Validateable, Devops, BuildType, MixinType from .image import Image from .c4k import C4k from .provs_k3s import K3s +from .terraform import Terraform from .release import Release from .version import Version @@ -22,6 +23,8 @@ class DevopsFactory: specialized_builds[BuildType.C4K] = C4k(inp) if BuildType.K3S in build_types: specialized_builds[BuildType.K3S] = K3s(inp) + if BuildType.K3S in build_types: + specialized_builds[BuildType.TERRAFORM] = Terraform(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py new file mode 100644 index 0000000..28eaaa8 --- /dev/null +++ b/src/main/python/ddadevops/domain/terraform.py @@ -0,0 +1,36 @@ +from typing import List, Optional +from .common import ( + Validateable, + DnsRecord, + Devops, +) + + +class Terraform(Validateable): + def __init__(self, inp: dict): + self.module = inp.get("module") + self.stage = inp.get("stage") + self.tf_additional_vars = inp.get("tf_additional_vars") + self.tf_output_json_name = inp.get("tf_output_json_name") + self.tf_build_commons_path = inp.get("tf_build_commons_path") + self.tf_additional_tfvar_files = inp.get("tf_additional_tfvar_files", []) + self.tf_use_workspace = inp.get("tf_use_workspace", True) + self.tf_debug_print_terraform_command = inp.get( + "tf_debug_print_terraform_command", False + ) + self.tf_commons_dir_name = inp.get("tf_commons_dir_name", "terraform") + self.tf_terraform_semantic_version = inp.get( + "tf_terraform_semantic_version", "1.0.8" + ) + self.tf_use_package_common_files = inp.get("tf_use_package_common_files", True) + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("module") + return result + + def output_json_name(self) -> str: + if self.tf_output_json_name: + return self.tf_output_json_name + else: + return f"out_{self.module}.json" diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 9386531..2b09198 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -9,7 +9,7 @@ def devops_config(overrides: dict) -> dict: "stage": "test", "project_root_path": "root_path", "build_dir_name": "target", - "build_types": ["IMAGE", "C4K", "K3S"], + "build_types": ["IMAGE", "C4K", "K3S", "TERRAFORM"], "mixin_types": ["RELEASE"], "image_dockerhub_user": "dockerhub_user", "image_dockerhub_password": "dockerhub_password", @@ -24,6 +24,16 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", + "tf_additional_vars": None, + "tf_output_json_name": "the_out.json", + "tf_use_workspace": None, + "tf_use_package_common_files": None, + "tf_build_commons_path": None, + "tf_commons_dir_name": None, + "tf_debug_print_terraform_command": None, + "tf_additional_tfvar_files": None, + "tf_terraform_semantic_version": None, + "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py new file mode 100644 index 0000000..013bc94 --- /dev/null +++ b/src/test/python/domain/test_terraform.py @@ -0,0 +1,19 @@ +import pytest +from pathlib import Path +from src.main.python.ddadevops.domain import DnsRecord, BuildType, Terraform +from .helper import build_devops + + +def test_creation(): + sut = build_devops({}) + assert BuildType.TERRAFORM in sut.specialized_builds + assert sut.specialized_builds[BuildType.TERRAFORM] + +def test_should_calculate_output_json_name(): + devops = build_devops({}) + sut = devops.specialized_builds[BuildType.TERRAFORM] + assert 'the_out.json' == sut.output_json_name() + + devops = build_devops({"tf_output_json_name": None,}) + sut = devops.specialized_builds[BuildType.TERRAFORM] + assert 'out_module.json' == sut.output_json_name() From 74a95c8c592ea382ad1f2dd06d0615b69a9dca40 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 11:29:49 +0200 Subject: [PATCH 130/173] add terraform_build_commons_path() --- src/main/python/ddadevops/domain/terraform.py | 11 ++++++- src/test/python/domain/helper.py | 4 +-- src/test/python/domain/test_terraform.py | 30 ++++++++++++++----- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 28eaaa8..95b29bb 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -1,8 +1,10 @@ from typing import List, Optional +from pathlib import Path from .common import ( Validateable, DnsRecord, Devops, + filter_none, ) @@ -18,7 +20,9 @@ class Terraform(Validateable): self.tf_debug_print_terraform_command = inp.get( "tf_debug_print_terraform_command", False ) - self.tf_commons_dir_name = inp.get("tf_commons_dir_name", "terraform") + self.tf_build_commons_dir_name = inp.get( + "tf_build_commons_dir_name", "terraform" + ) self.tf_terraform_semantic_version = inp.get( "tf_terraform_semantic_version", "1.0.8" ) @@ -27,6 +31,7 @@ class Terraform(Validateable): def validate(self) -> List[str]: result = [] result += self.__validate_is_not_empty__("module") + result += self.__validate_is_not_empty__("tf_build_commons_dir_name") return result def output_json_name(self) -> str: @@ -34,3 +39,7 @@ class Terraform(Validateable): return self.tf_output_json_name else: return f"out_{self.module}.json" + + def terraform_build_commons_path(self) -> Path: + mylist = [self.tf_build_commons_path, self.tf_build_commons_dir_name] + return Path("/".join(filter_none(mylist)) + "/") diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 2b09198..358803f 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -28,8 +28,8 @@ def devops_config(overrides: dict) -> dict: "tf_output_json_name": "the_out.json", "tf_use_workspace": None, "tf_use_package_common_files": None, - "tf_build_commons_path": None, - "tf_commons_dir_name": None, + "tf_build_commons_path": "build_commons_path", + "tf_build_commons_dir_name": "terraform", "tf_debug_print_terraform_command": None, "tf_additional_tfvar_files": None, "tf_terraform_semantic_version": None, diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 013bc94..d46457d 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -1,7 +1,7 @@ import pytest from pathlib import Path from src.main.python.ddadevops.domain import DnsRecord, BuildType, Terraform -from .helper import build_devops +from .helper import build_devops, devops_config def test_creation(): @@ -9,11 +9,25 @@ def test_creation(): assert BuildType.TERRAFORM in sut.specialized_builds assert sut.specialized_builds[BuildType.TERRAFORM] -def test_should_calculate_output_json_name(): - devops = build_devops({}) - sut = devops.specialized_builds[BuildType.TERRAFORM] - assert 'the_out.json' == sut.output_json_name() - devops = build_devops({"tf_output_json_name": None,}) - sut = devops.specialized_builds[BuildType.TERRAFORM] - assert 'out_module.json' == sut.output_json_name() +def test_should_calculate_output_json_name(): + config = devops_config({}) + sut = Terraform(config) + assert "the_out.json" == sut.output_json_name() + + config = devops_config({}) + del config["tf_output_json_name"] + sut = Terraform(config) + assert "out_module.json" == sut.output_json_name() + + +def test_should_calculate_terraform_build_commons_path(): + config = devops_config({}) + del config["tf_build_commons_path"] + del config["tf_build_commons_dir_name"] + sut = Terraform(config) + assert Path("terraform") == sut.terraform_build_commons_path() + + config = devops_config({}) + sut = Terraform(config) + assert Path("build_commons_path/terraform") == sut.terraform_build_commons_path() From e4fef8b61c5a2ee59f717ccdbc218843ee3d1dd6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 14:16:27 +0200 Subject: [PATCH 131/173] add a terraform api --- .../python/ddadevops/infrastructure/__init__.py | 1 + .../ddadevops/infrastructure/infrastructure.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index a0e975e..1bb11b5 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -6,5 +6,6 @@ from .infrastructure import ( EnvironmentApi, CredentialsApi, GitApi, + TerraformApi, ) from .repository import DevopsRepository, BuildFileRepository diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 814660e..6cef62f 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -8,7 +8,7 @@ import yaml class ResourceApi: def read_resource(self, path: str) -> bytes: - return resource_string('ddadevops', path) + return resource_string("ddadevops", path) class FileApi: @@ -19,6 +19,9 @@ class FileApi: self.execution_api.execute("rm -rf " + directory) self.execution_api.execute("mkdir -p " + directory) + def cp(self, src: str, target_dir: str): + self.execution_api.execute(f"cp {src} {target_dir}") + def cp_force(self, src: str, target_dir: str): self.execution_api.execute("cp -f " + src + "* " + target_dir) @@ -122,7 +125,9 @@ class CredentialsApi: credential = None if path and field: print("get field for: " + path + ", " + field) - credential = self.execution_api.execute(["gopass", "show", path, field], shell=False) + credential = self.execution_api.execute( + ["gopass", "show", path, field], shell=False + ) return credential def gopass_password_from_path(self, path): @@ -182,3 +187,7 @@ class GitApi: def checkout(self, branch: str): return self.execution_api.execute(f"git checkout {branch}") + + +class TerraformApi: + pass From d909251c08ba9428e5e8fbe43aa11dc116693978 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 14:17:13 +0200 Subject: [PATCH 132/173] add project vars fkt --- src/main/python/ddadevops/domain/terraform.py | 7 +++++++ src/test/python/domain/test_terraform.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 95b29bb..acd1292 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -31,6 +31,7 @@ class Terraform(Validateable): def validate(self) -> List[str]: result = [] result += self.__validate_is_not_empty__("module") + result += self.__validate_is_not_empty__("stage") result += self.__validate_is_not_empty__("tf_build_commons_dir_name") return result @@ -43,3 +44,9 @@ class Terraform(Validateable): def terraform_build_commons_path(self) -> Path: mylist = [self.tf_build_commons_path, self.tf_build_commons_dir_name] return Path("/".join(filter_none(mylist)) + "/") + + def project_vars(self): + ret = {"stage": self.stage, "module": self.module} + if self.tf_additional_vars: + ret.update(self.tf_additional_vars) + return ret diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index d46457d..b042bf0 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -31,3 +31,8 @@ def test_should_calculate_terraform_build_commons_path(): config = devops_config({}) sut = Terraform(config) assert Path("build_commons_path/terraform") == sut.terraform_build_commons_path() + +def test_should_calculate_project_vars(): + config = devops_config({}) + sut = Terraform(config) + assert {'module': 'module', 'stage': 'test'} == sut.project_vars() From 7ede8e345af2a914ed95bea0b25652f7c88cae52 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 14:17:47 +0200 Subject: [PATCH 133/173] migrate fkt to service --- .../application/terraform_service.py | 62 +++++++++++++++++++ .../ddadevops/devops_terraform_build.py | 49 ++------------- 2 files changed, 67 insertions(+), 44 deletions(-) create mode 100644 src/main/python/ddadevops/application/terraform_service.py diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py new file mode 100644 index 0000000..f3ba585 --- /dev/null +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -0,0 +1,62 @@ +from pathlib import Path +from ..domain import Devops, BuildType +from ..infrastructure import FileApi, ResourceApi, ImageApi + + +class TerraformService: + def __init__( + self, file_api: FileApi, resource_api: ResourceApi, image_api: ImageApi + ): + self.file_api = file_api + self.resource_api = resource_api + self.image_api = image_api + + @classmethod + def prod(cls): + return cls( + FileApi(), + ResourceApi(), + ImageApi(), + ) + + def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): + data = self.resource_api.read_resource( + f"src/main/resources/terraform/{resource_name}" + ) + self.file_api.write_data_to_file( + Path(f"{devops.build_path()}/{resource_name}"), data + ) + + def __copy_build_resources_from_package__(self, devops: Devops): + self.__copy_build_resource_file_from_package__("versions.tf", devops) + self.__copy_build_resource_file_from_package__( + "terraform_build_vars.tf", devops + ) + + def __copy_build_resources_from_dir__(self, devops: Devops): + terraform = devops.specialized_builds[BuildType.TERRAFORM] + self.file_api.cp_force( + f"{terraform.build_commons_path()}/*", devops.build_path() + ) + + def copy_local_state(self, devops: Devops): + # TODO: orignal was unchecked ... + self.file_api.cp("terraform.tfstate", devops.build_path()) + + def rescue_local_state(self, devops: Devops): + # TODO: orignal was unchecked ... + self.file_api.cp(f"{devops.build_path()}/terraform.tfstate", ".") + + def initialize_build_dir(self, devops: Devops): + terraform = devops.specialized_builds[BuildType.TERRAFORM] + if terraform.tf_use_package_common_files: + self.__copy_build_resources_from_package__(devops) + else: + self.__copy_build_resources_from_dir__(devops) + # TODO: orignal was unchecked ... + self.copy_local_state(devops) + self.file_api.cp("*.tf", devops.build_path()) + self.file_api.cp("*.properties", devops.build_path()) + self.file_api.cp("*.tfvars", devops.build_path()) + self.file_api.cp_recursive("scripts", devops.build_path()) + diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index a8c024c..769823f 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -56,55 +56,16 @@ class DevopsTerraformBuild(DevopsBuild): inp["mixin_types"] = config.get("mixin_types", []) super().__init__(project, inp) project.build_depends_on("dda-python-terraform") - - def terraform_build_commons_path(self): - mylist = [self.build_commons_path, self.terraform_build_commons_dir_name] - return "/".join(filter_none(mylist)) + "/" - - def project_vars(self): - ret = {"stage": self.stage} - if self.module: - ret["module"] = self.module - if self.additional_vars: - ret.update(self.additional_vars) - return ret - - def copy_build_resource_file_from_package(self, name): - my_data = resource_string(__name__, "src/main/resources/terraform/" + name) - with open(self.build_path() + "/" + name, "w", encoding="utf-8") as output_file: - output_file.write(my_data.decode(sys.stdout.encoding)) - - def copy_build_resources_from_package(self): - self.copy_build_resource_file_from_package("versions.tf") - self.copy_build_resource_file_from_package("terraform_build_vars.tf") - - def copy_build_resources_from_dir(self): - run( - "cp -f " + self.terraform_build_commons_path() + "* " + self.build_path(), - shell=True, - check=False, - ) - - def copy_local_state(self): - run("cp terraform.tfstate " + self.build_path(), shell=True, check=False) - - def rescue_local_state(self): - run("cp " + self.build_path() + "/terraform.tfstate .", shell=True, check=False) + self.teraform_service = Terraform.prod() def initialize_build_dir(self): super().initialize_build_dir() - if self.use_package_common_files: - self.copy_build_resources_from_package() - else: - self.copy_build_resources_from_dir() - self.copy_local_state() - run("cp *.tf " + self.build_path(), shell=True, check=False) - run("cp *.properties " + self.build_path(), shell=True, check=False) - run("cp *.tfvars " + self.build_path(), shell=True, check=False) - run("cp -r scripts " + self.build_path(), shell=True, check=False) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.initialize_build_dir(devops) def post_build(self): - self.rescue_local_state() + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.rescue_local_state(devops) def init_client(self): terraform = Terraform( From 62464afb83ff28b9da1bcf3829940d6e53dd5df8 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 16:04:42 +0200 Subject: [PATCH 134/173] devops_terraform_build now might work --- .../application/terraform_service.py | 160 +++++++++++++++++- .../ddadevops/devops_terraform_build.py | 151 ++--------------- src/main/python/ddadevops/domain/__init__.py | 2 +- .../python/ddadevops/domain/devops_factory.py | 4 +- src/main/python/ddadevops/domain/terraform.py | 2 +- .../infrastructure/infrastructure.py | 9 + src/test/python/domain/test_terraform.py | 12 +- 7 files changed, 189 insertions(+), 151 deletions(-) diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index f3ba585..a0986ab 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -1,22 +1,26 @@ from pathlib import Path -from ..domain import Devops, BuildType -from ..infrastructure import FileApi, ResourceApi, ImageApi +from dda_python_terraform import Terraform, IsFlagged +from packaging import version + +from ..domain import Devops, BuildType, TerraformDomain +from ..infrastructure import FileApi, ResourceApi, TerraformApi +# TODO: mv more fkt to Terraform_api ? class TerraformService: def __init__( - self, file_api: FileApi, resource_api: ResourceApi, image_api: ImageApi + self, file_api: FileApi, resource_api: ResourceApi, terraform_api: TerraformApi ): self.file_api = file_api self.resource_api = resource_api - self.image_api = image_api + self.terraform_api = terraform_api @classmethod def prod(cls): return cls( FileApi(), ResourceApi(), - ImageApi(), + TerraformApi(), ) def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): @@ -39,6 +43,12 @@ class TerraformService: f"{terraform.build_commons_path()}/*", devops.build_path() ) + def __print_terraform_command__(self, terraform: Terraform, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if terraform_domain.tf_debug_print_terraform_command: + output = f"cd {devops.build_path()} && {terraform.latest_cmd()}" + print(output) + def copy_local_state(self, devops: Devops): # TODO: orignal was unchecked ... self.file_api.cp("terraform.tfstate", devops.build_path()) @@ -60,3 +70,143 @@ class TerraformService: self.file_api.cp("*.tfvars", devops.build_path()) self.file_api.cp_recursive("scripts", devops.build_path()) + def init_client(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + terraform = Terraform( + working_dir=devops.build_path(), + terraform_semantic_version=terraform_domain.tf_terraform_semantic_version, + ) + terraform.init() + self.__print_terraform_command__(terraform, devops) + if terraform_domain.tf_use_workspace: + try: + terraform.workspace("select", self.stage) + self.__print_terraform_command__(terraform, devops) + except: + terraform.workspace("new", self.stage) + self.__print_terraform_command__(terraform, devops) + return terraform + + def write_output(self, terraform, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + result = terraform.output(json=IsFlagged) + self.__print_terraform_command__(terraform, devops) + self.file_api.write_json_to_file( + Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}"), result + ) + + def read_output(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + return self.file_api.read_json_fro_file( + Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}") + ) + + def plan(self, devops: Devops, fail_on_diff=False): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if fail_on_diff: + detailed_exitcode = IsFlagged + else: + detailed_exitcode = None + terraform = self.init_client(devops) + return_code, _, stderr = terraform.plan( + detailed_exitcode=detailed_exitcode, + capture_output=False, + raise_on_error=False, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform) + if return_code not in (0, 2): + raise RuntimeError(return_code, "terraform error:", stderr) + if return_code == 2: + raise RuntimeError(return_code, "diff in config found:", stderr) + + def apply(self, devops: Devops, auto_approve=False): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if auto_approve: + auto_approve_flag = IsFlagged + else: + auto_approve_flag = None + terraform = self.init_client(devops) + if version.parse( + terraform_domain.tf_terraform_semantic_version + ) >= version.parse("1.0.0"): + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + else: + return_code, _, stderr = terraform.apply( + capture_output=False, + raise_on_error=True, + skip_plan=auto_approve, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) + self.write_output(terraform, devops) + + def refresh(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + terraform = self.init_client(devops) + return_code, _, stderr = terraform.refresh( + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) + self.write_output(terraform, devops) + + def destroy(self, devops: Devops, auto_approve=False): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if auto_approve: + auto_approve_flag = IsFlagged + else: + auto_approve_flag = None + terraform = self.init_client(devops) + if version.parse( + terraform_domain.tf_terraform_semantic_version + ) >= version.parse("1.0.0"): + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + auto_approve=auto_approve_flag, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + else: + return_code, _, stderr = terraform.destroy( + capture_output=False, + raise_on_error=True, + force=auto_approve_flag, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.__print_terraform_command__(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) + + def tf_import( + self, + devops: Devops, + tf_import_name, + tf_import_resource, + ): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + return_code, _, stderr = terraform.import_cmd( + tf_import_name, + tf_import_resource, + capture_output=False, + raise_on_error=True, + var=terraform_domain.project_vars(), + var_file=terraform_domain.tf_additional_tfvar_files, + ) + self.print_terraform_command(terraform, devops) + if return_code > 0: + raise RuntimeError(return_code, "terraform error:", stderr) diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 769823f..830c12f 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -1,11 +1,3 @@ -import sys -from os import chmod -from json import load, dumps -from subprocess import run -from packaging import version -from pkg_resources import resource_string -from dda_python_terraform import Terraform, IsFlagged -from .python_util import filter_none from .devops_build import DevopsBuild, create_devops_build_config @@ -67,153 +59,40 @@ class DevopsTerraformBuild(DevopsBuild): devops = self.devops_repo.get_devops(self.project) self.teraform_service.rescue_local_state(devops) - def init_client(self): - terraform = Terraform( - working_dir=self.build_path(), - terraform_semantic_version=self.terraform_semantic_version, - ) - terraform.init() - self.print_terraform_command(terraform) - if self.use_workspace: - try: - terraform.workspace("select", self.stage) - self.print_terraform_command(terraform) - except: - terraform.workspace("new", self.stage) - self.print_terraform_command(terraform) - return terraform - - def write_output(self, terraform): - result = terraform.output(json=IsFlagged) - self.print_terraform_command(terraform) - with open( - self.build_path() + self.output_json_name, "w", encoding="utf-8" - ) as output_file: - output_file.write(dumps(result)) - chmod(self.build_path() + self.output_json_name, 0o600) - - def read_output_json(self): - with open( - self.build_path() + self.output_json_name, "r", encoding="utf-8" - ) as file: - return load(file) + def read_output_json(self) -> map: + devops = self.devops_repo.get_devops(self.project) + return self.teraform_service.read_output(devops) def plan(self): - terraform = self.init_client() - return_code, _, stderr = terraform.plan( - detailed_exitcode=None, - capture_output=False, - raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.plan(devops) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def plan_fail_on_diff(self): - terraform = self.init_client() - return_code, _, stderr = terraform.plan( - detailed_exitcode=IsFlagged, - capture_output=False, - raise_on_error=False, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.plan(devops, fail_on_diff=True) self.post_build() - self.print_terraform_command(terraform) - if return_code not in (0, 2): - raise RuntimeError(return_code, "terraform error:", stderr) - if return_code == 2: - raise RuntimeError(return_code, "diff in config found:", stderr) def apply(self, auto_approve=False): - terraform = self.init_client() - if auto_approve: - auto_approve_flag = IsFlagged - else: - auto_approve_flag = None - if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.apply( - capture_output=False, - raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) - else: - return_code, _, stderr = terraform.apply( - capture_output=False, - raise_on_error=True, - skip_plan=auto_approve, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) - self.write_output(terraform) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.apply(devops, auto_approve=auto_approve) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def refresh(self): - terraform = self.init_client() - return_code, _, stderr = terraform.refresh( - var=self.project_vars(), var_file=self.additional_tfvar_files - ) - self.write_output(terraform) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.refresh(devops) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def destroy(self, auto_approve=False): - terraform = self.init_client() - if auto_approve: - auto_approve_flag = IsFlagged - else: - auto_approve_flag = None - if version.parse(self.terraform_semantic_version) >= version.parse("1.0.0"): - return_code, _, stderr = terraform.destroy( - capture_output=False, - raise_on_error=True, - auto_approve=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) - else: - return_code, _, stderr = terraform.destroy( - capture_output=False, - raise_on_error=True, - force=auto_approve_flag, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.refresh(devops) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) def tf_import( self, tf_import_name, tf_import_resource, ): - terraform = self.init_client() - return_code, _, stderr = terraform.import_cmd( - tf_import_name, - tf_import_resource, - capture_output=False, - raise_on_error=True, - var=self.project_vars(), - var_file=self.additional_tfvar_files, - ) + devops = self.devops_repo.get_devops(self.project) + self.teraform_service.tf_import(devops, tf_import_name, tf_import_resource) self.post_build() - self.print_terraform_command(terraform) - if return_code > 0: - raise RuntimeError(return_code, "terraform error:", stderr) - - def print_terraform_command(self, terraform): - if self.debug_print_terraform_command: - output = "cd " + self.build_path() + " && " + terraform.latest_cmd() - print(output) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 967c0f2..c45278d 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -2,7 +2,7 @@ from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, Relea from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k -from .terraform import Terraform +from .terraform import TerraformDomain from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index d045b50..f43e69e 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -3,7 +3,7 @@ from .common import Validateable, Devops, BuildType, MixinType from .image import Image from .c4k import C4k from .provs_k3s import K3s -from .terraform import Terraform +from .terraform import TerraformDomain from .release import Release from .version import Version @@ -24,7 +24,7 @@ class DevopsFactory: if BuildType.K3S in build_types: specialized_builds[BuildType.K3S] = K3s(inp) if BuildType.K3S in build_types: - specialized_builds[BuildType.TERRAFORM] = Terraform(inp) + specialized_builds[BuildType.TERRAFORM] = TerraformDomain(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index acd1292..e0a6e38 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -8,7 +8,7 @@ from .common import ( ) -class Terraform(Validateable): +class TerraformDomain(Validateable): def __init__(self, inp: dict): self.module = inp.get("module") self.stage = inp.get("stage") diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 6cef62f..82ba879 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -3,6 +3,7 @@ from pathlib import Path from sys import stdout from os import chmod, environ from pkg_resources import resource_string +from json import load, dumps import yaml @@ -37,6 +38,14 @@ class FileApi: yaml.dump(data, output_file) chmod(path, 0o600) + def write_json_to_file(self, path: Path, data: map): + with open(path, "w", encoding="utf-8") as output_file: + output_file.write(dumps(data)) + chmod(path, 0o600) + + def read_json_fro_file(self, path: Path) -> map: + with open(path, "r", encoding="utf-8") as input_file: + return load(input_file) class ImageApi: def image(self, name: str, path: Path): diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index b042bf0..720e9ff 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -1,6 +1,6 @@ import pytest from pathlib import Path -from src.main.python.ddadevops.domain import DnsRecord, BuildType, Terraform +from src.main.python.ddadevops.domain import DnsRecord, BuildType, TerraformDomain from .helper import build_devops, devops_config @@ -12,12 +12,12 @@ def test_creation(): def test_should_calculate_output_json_name(): config = devops_config({}) - sut = Terraform(config) + sut = TerraformDomain(config) assert "the_out.json" == sut.output_json_name() config = devops_config({}) del config["tf_output_json_name"] - sut = Terraform(config) + sut = TerraformDomain(config) assert "out_module.json" == sut.output_json_name() @@ -25,14 +25,14 @@ def test_should_calculate_terraform_build_commons_path(): config = devops_config({}) del config["tf_build_commons_path"] del config["tf_build_commons_dir_name"] - sut = Terraform(config) + sut = TerraformDomain(config) assert Path("terraform") == sut.terraform_build_commons_path() config = devops_config({}) - sut = Terraform(config) + sut = TerraformDomain(config) assert Path("build_commons_path/terraform") == sut.terraform_build_commons_path() def test_should_calculate_project_vars(): config = devops_config({}) - sut = Terraform(config) + sut = TerraformDomain(config) assert {'module': 'module', 'stage': 'test'} == sut.project_vars() From de29c1dad844dc64fca1cd6c6fd203b836670c07 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 24 May 2023 17:50:09 +0200 Subject: [PATCH 135/173] fix terraform issues --- build.py | 2 +- .../python/ddadevops/application/__init__.py | 1 + .../application/terraform_service.py | 15 ++++++-------- .../ddadevops/devops_terraform_build.py | 3 ++- .../infrastructure/infrastructure.py | 20 +++++++++++-------- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/build.py b/build.py index 7a5a489..06c0dc9 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev48" +version = "4.0.0-dev52" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/application/__init__.py b/src/main/python/ddadevops/application/__init__.py index 9f46f1a..f087165 100644 --- a/src/main/python/ddadevops/application/__init__.py +++ b/src/main/python/ddadevops/application/__init__.py @@ -1,2 +1,3 @@ from .image_build_service import ImageBuildService from .release_mixin_services import ReleaseService +from .terraform_service import TerraformService diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index a0986ab..b093f2a 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -50,12 +50,10 @@ class TerraformService: print(output) def copy_local_state(self, devops: Devops): - # TODO: orignal was unchecked ... - self.file_api.cp("terraform.tfstate", devops.build_path()) + self.file_api.cp("terraform.tfstate", devops.build_path(), check=False) def rescue_local_state(self, devops: Devops): - # TODO: orignal was unchecked ... - self.file_api.cp(f"{devops.build_path()}/terraform.tfstate", ".") + self.file_api.cp(f"{devops.build_path()}/terraform.tfstate", ".", check=False) def initialize_build_dir(self, devops: Devops): terraform = devops.specialized_builds[BuildType.TERRAFORM] @@ -63,12 +61,11 @@ class TerraformService: self.__copy_build_resources_from_package__(devops) else: self.__copy_build_resources_from_dir__(devops) - # TODO: orignal was unchecked ... self.copy_local_state(devops) - self.file_api.cp("*.tf", devops.build_path()) - self.file_api.cp("*.properties", devops.build_path()) - self.file_api.cp("*.tfvars", devops.build_path()) - self.file_api.cp_recursive("scripts", devops.build_path()) + self.file_api.cp("*.tf", devops.build_path(), check=False) + self.file_api.cp("*.properties", devops.build_path(), check=False) + self.file_api.cp("*.tfvars", devops.build_path(), check=False) + self.file_api.cp_recursive("scripts", devops.build_path(), check=False) def init_client(self, devops: Devops): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 830c12f..6b0f8ce 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -1,4 +1,5 @@ from .devops_build import DevopsBuild, create_devops_build_config +from .application import TerraformService def create_devops_terraform_build_config( @@ -48,7 +49,7 @@ class DevopsTerraformBuild(DevopsBuild): inp["mixin_types"] = config.get("mixin_types", []) super().__init__(project, inp) project.build_depends_on("dda-python-terraform") - self.teraform_service = Terraform.prod() + self.teraform_service = TerraformService.prod() def initialize_build_dir(self): super().initialize_build_dir() diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 82ba879..80224ca 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -20,14 +20,14 @@ class FileApi: self.execution_api.execute("rm -rf " + directory) self.execution_api.execute("mkdir -p " + directory) - def cp(self, src: str, target_dir: str): - self.execution_api.execute(f"cp {src} {target_dir}") + def cp(self, src: str, target_dir: str, check=True): + self.execution_api.execute(f"cp {src} {target_dir}", check=check) - def cp_force(self, src: str, target_dir: str): - self.execution_api.execute("cp -f " + src + "* " + target_dir) + def cp_force(self, src: str, target_dir: str, check=True): + self.execution_api.execute(f"cp -f {src}* {target_dir}", check=check) - def cp_recursive(self, src: str, target_dir: str): - self.execution_api.execute("cp -r " + src + " " + target_dir) + def cp_recursive(self, src: str, target_dir: str, check=True): + self.execution_api.execute(f"cp -r {src} {target_dir}", check=check) def write_data_to_file(self, path: Path, data: bytes): with open(path, "w", encoding="utf-8") as output_file: @@ -47,6 +47,7 @@ class FileApi: with open(path, "r", encoding="utf-8") as input_file: return load(input_file) + class ImageApi: def image(self, name: str, path: Path): run( @@ -101,12 +102,15 @@ class ImageApi: class ExecutionApi: - def execute(self, command: str, dry_run=False, shell=True): + def execute(self, command: str, dry_run=False, shell=True, check=True): output = "" if dry_run: print(command) else: - output = check_output(command, encoding="UTF-8", shell=shell) + # output = check_output(command, encoding="UTF-8", shell=shell) + output = run( + command, encoding="UTF-8", shell=shell, stdout=PIPE, check=check + ).stdout output = output.rstrip() return output From e95ff3ead20929ca69d880cdc6ce95ee60036e15 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 25 May 2023 14:49:14 +0200 Subject: [PATCH 136/173] Update architecture with digitalocean --- doc/architecture/Domain.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 8b36d39..33d8f95 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -37,7 +37,7 @@ classDiagram k3s_app_filename_to_provision } - class Terraform { + class TerraformDomain { tf_additional_vars tf_output_json_name tf_use_workspace @@ -91,10 +91,17 @@ classDiagram create_bump(snapshot_suffix) } - Devops *-- "0..1" Image: spcialized_builds - Devops *-- "0..1" C4k: spcialized_builds - Devops *-- "0..1" ProvsK3s: spcialized_builds - Devops *-- "0..1" Terraform: spcialized_builds + class DigitaloceanTerraform { + do_api_key + do_spaces_access_key + do_spaces_secret_key + } + + Devops *-- "0..1" Image: specialized_builds + Devops *-- "0..1" C4k: specialized_builds + Devops *-- "0..1" ProvsK3s: specialized_builds + Devops *-- "0..1" TerraformDomain: specialized_builds + Devops *-- "0..1" DigitaloceanTerraform: specialized_builds Devops *-- "0..1" Release: mixins Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files From 664f3cfd799538d1c0c7b4ec4fef5155aa2dd74f Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 25 May 2023 14:49:34 +0200 Subject: [PATCH 137/173] Implement DigitaloceanTerraform build type --- src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 1 + .../python/ddadevops/domain/devops_factory.py | 5 ++++- .../domain/digitalocean_terraform.py | 22 +++++++++++++++++++ src/test/python/domain/helper.py | 3 +++ src/test/python/domain/test_devops_factory.py | 22 ++++++++++++++++++- .../domain/test_digitalocean_terraform.py | 15 +++++++++++++ 7 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 src/main/python/ddadevops/domain/digitalocean_terraform.py create mode 100644 src/test/python/domain/test_digitalocean_terraform.py diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index c45278d..e8f0454 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -3,6 +3,7 @@ from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k from .terraform import TerraformDomain +from .digitalocean_terraform import DigitaloceanTerraform from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index c270a49..af62ed0 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -11,6 +11,7 @@ class BuildType(Enum): C4K = 1 K3S = 2 TERRAFORM = 3 + DIGITALOCEAN_TERRAFORM = 4 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index f43e69e..26eb72f 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -4,6 +4,7 @@ from .image import Image from .c4k import C4k from .provs_k3s import K3s from .terraform import TerraformDomain +from .digitalocean_terraform import DigitaloceanTerraform from .release import Release from .version import Version @@ -23,8 +24,10 @@ class DevopsFactory: specialized_builds[BuildType.C4K] = C4k(inp) if BuildType.K3S in build_types: specialized_builds[BuildType.K3S] = K3s(inp) - if BuildType.K3S in build_types: + if BuildType.TERRAFORM in build_types: specialized_builds[BuildType.TERRAFORM] = TerraformDomain(inp) + if BuildType.DIGITALOCEAN_TERRAFORM in build_types: + specialized_builds[BuildType.DIGITALOCEAN_TERRAFORM] = DigitaloceanTerraform(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/digitalocean_terraform.py b/src/main/python/ddadevops/domain/digitalocean_terraform.py new file mode 100644 index 0000000..98f541c --- /dev/null +++ b/src/main/python/ddadevops/domain/digitalocean_terraform.py @@ -0,0 +1,22 @@ +from typing import List +from .common import ( + Validateable, +) + + +class DigitaloceanTerraform(Validateable): + def __init__( + self, + inp: dict, + ): + self.do_api_key = inp.get("do_api_key") + self.do_spaces_access_id = inp.get("do_spaces_access_id") + self.do_spaces_secret_key = inp.get("do_spaces_secret_key") + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("do_api_key") + result += self.__validate_is_not_empty__("do_spaces_access_id") + result += self.__validate_is_not_empty__("do_spaces_secret_key") + return result + diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 358803f..af6a9df 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -33,6 +33,9 @@ def devops_config(overrides: dict) -> dict: "tf_debug_print_terraform_command": None, "tf_additional_tfvar_files": None, "tf_terraform_semantic_version": None, + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", "release_type": "NONE", "release_main_branch": "main", diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index 7287869..4d470cb 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -1,6 +1,6 @@ import pytest from src.main.python.ddadevops.domain import ( - DevopsFactory, Version + DevopsFactory, Version, BuildType, MixinType ) @@ -29,6 +29,7 @@ def test_devops_factory(): } ) assert sut is not None + assert sut.specialized_builds[BuildType.IMAGE] is not None sut = DevopsFactory().build_devops( { @@ -44,6 +45,24 @@ def test_devops_factory(): Version.from_str("1.0.0") ) assert sut is not None + assert sut.specialized_builds[BuildType.C4K] is not None + + sut = DevopsFactory().build_devops( + { + "stage": "test", + "name": "mybuild", + "module": "test_image", + "project_root_path": "../../..", + "build_types": ["DIGITALOCEAN_TERRAFORM"], + "mixin_types": [], + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + }, + Version.from_str("1.0.0") + ) + assert sut is not None + assert sut.specialized_builds[BuildType.DIGITALOCEAN_TERRAFORM] is not None sut = DevopsFactory().build_devops( { @@ -61,3 +80,4 @@ def test_devops_factory(): Version.from_str("1.0.0") ) assert sut is not None + assert sut.mixins[MixinType.RELEASE] is not None diff --git a/src/test/python/domain/test_digitalocean_terraform.py b/src/test/python/domain/test_digitalocean_terraform.py new file mode 100644 index 0000000..2e3bb32 --- /dev/null +++ b/src/test/python/domain/test_digitalocean_terraform.py @@ -0,0 +1,15 @@ +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain import ( + BuildType, + DigitaloceanTerraform, +) +from .helper import devops_config + + +def test_digitalocean_terraform(): + sut = DigitaloceanTerraform(devops_config({'do_api_key': 'api_key', + 'do_spaces_access_id': 'spaces_id', + 'do_spaces_secret_key': 'spaces_secret'})) + assert sut is not None + assert sut.is_valid() From 20edebf61cdd374ff792eb1dee6cd36e6dceb35c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 25 May 2023 14:50:12 +0200 Subject: [PATCH 138/173] added some todos --- src/main/python/ddadevops/application/terraform_service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index b093f2a..a2bf4ed 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -49,9 +49,11 @@ class TerraformService: output = f"cd {devops.build_path()} && {terraform.latest_cmd()}" print(output) + # TODO: internal? def copy_local_state(self, devops: Devops): self.file_api.cp("terraform.tfstate", devops.build_path(), check=False) + # TODO: internal? def rescue_local_state(self, devops: Devops): self.file_api.cp(f"{devops.build_path()}/terraform.tfstate", ".", check=False) @@ -67,6 +69,7 @@ class TerraformService: self.file_api.cp("*.tfvars", devops.build_path(), check=False) self.file_api.cp_recursive("scripts", devops.build_path(), check=False) + # TODO: internal? def init_client(self, devops: Devops): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] terraform = Terraform( @@ -84,6 +87,7 @@ class TerraformService: self.__print_terraform_command__(terraform, devops) return terraform + # TODO: internal? def write_output(self, terraform, devops: Devops): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] result = terraform.output(json=IsFlagged) @@ -92,7 +96,7 @@ class TerraformService: Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}"), result ) - def read_output(self, devops: Devops): + def read_output(self, devops: Devops) -> map: terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] return self.file_api.read_json_fro_file( Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}") From b24288976ef3de4681f947e18d2fbd4e959c4d82 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 25 May 2023 14:51:49 +0200 Subject: [PATCH 139/173] format --- src/main/python/ddadevops/domain/devops_factory.py | 4 +++- .../ddadevops/domain/digitalocean_terraform.py | 1 - src/test/python/domain/helper.py | 1 - src/test/python/domain/test_devops_factory.py | 11 +++++++---- .../python/domain/test_digitalocean_terraform.py | 14 ++++++++++---- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 26eb72f..07ce39a 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -27,7 +27,9 @@ class DevopsFactory: if BuildType.TERRAFORM in build_types: specialized_builds[BuildType.TERRAFORM] = TerraformDomain(inp) if BuildType.DIGITALOCEAN_TERRAFORM in build_types: - specialized_builds[BuildType.DIGITALOCEAN_TERRAFORM] = DigitaloceanTerraform(inp) + specialized_builds[ + BuildType.DIGITALOCEAN_TERRAFORM + ] = DigitaloceanTerraform(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/digitalocean_terraform.py b/src/main/python/ddadevops/domain/digitalocean_terraform.py index 98f541c..10585ce 100644 --- a/src/main/python/ddadevops/domain/digitalocean_terraform.py +++ b/src/main/python/ddadevops/domain/digitalocean_terraform.py @@ -19,4 +19,3 @@ class DigitaloceanTerraform(Validateable): result += self.__validate_is_not_empty__("do_spaces_access_id") result += self.__validate_is_not_empty__("do_spaces_secret_key") return result - diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index af6a9df..8dde2e8 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -36,7 +36,6 @@ def devops_config(overrides: dict) -> dict: "do_api_key": "api_key", "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", - "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index 4d470cb..2ea8d1b 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -1,6 +1,9 @@ import pytest from src.main.python.ddadevops.domain import ( - DevopsFactory, Version, BuildType, MixinType + DevopsFactory, + Version, + BuildType, + MixinType, ) @@ -42,7 +45,7 @@ def test_devops_factory(): "c4k_grafana_cloud_user": "user", "c4k_grafana_cloud_password": "password", }, - Version.from_str("1.0.0") + Version.from_str("1.0.0"), ) assert sut is not None assert sut.specialized_builds[BuildType.C4K] is not None @@ -59,7 +62,7 @@ def test_devops_factory(): "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", }, - Version.from_str("1.0.0") + Version.from_str("1.0.0"), ) assert sut is not None assert sut.specialized_builds[BuildType.DIGITALOCEAN_TERRAFORM] is not None @@ -77,7 +80,7 @@ def test_devops_factory(): "release_current_branch": "my_feature", "release_config_file": "project.clj", }, - Version.from_str("1.0.0") + Version.from_str("1.0.0"), ) assert sut is not None assert sut.mixins[MixinType.RELEASE] is not None diff --git a/src/test/python/domain/test_digitalocean_terraform.py b/src/test/python/domain/test_digitalocean_terraform.py index 2e3bb32..7e6e6d5 100644 --- a/src/test/python/domain/test_digitalocean_terraform.py +++ b/src/test/python/domain/test_digitalocean_terraform.py @@ -1,15 +1,21 @@ from pybuilder.core import Project from pathlib import Path from src.main.python.ddadevops.domain import ( - BuildType, + BuildType, DigitaloceanTerraform, ) from .helper import devops_config def test_digitalocean_terraform(): - sut = DigitaloceanTerraform(devops_config({'do_api_key': 'api_key', - 'do_spaces_access_id': 'spaces_id', - 'do_spaces_secret_key': 'spaces_secret'})) + sut = DigitaloceanTerraform( + devops_config( + { + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + } + ) + ) assert sut is not None assert sut.is_valid() From 4cf3ec84600cb3a158bb416819b17427729ec11d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 25 May 2023 15:21:13 +0200 Subject: [PATCH 140/173] enhance terraform class --- doc/architecture/Domain.md | 1 + src/main/python/ddadevops/domain/terraform.py | 10 ++++++++++ src/test/python/domain/test_terraform.py | 9 +++++++++ 3 files changed, 20 insertions(+) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 33d8f95..e0971e2 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -39,6 +39,7 @@ classDiagram class TerraformDomain { tf_additional_vars + tf_additional_resources_from_package tf_output_json_name tf_use_workspace tf_use_package_common_files diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index e0a6e38..f5cecf4 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -15,6 +15,9 @@ class TerraformDomain(Validateable): self.tf_additional_vars = inp.get("tf_additional_vars") self.tf_output_json_name = inp.get("tf_output_json_name") self.tf_build_commons_path = inp.get("tf_build_commons_path") + self.tf_additional_resources_from_package = inp.get( + "tf_additional_resources_from_package", [] + ) self.tf_additional_tfvar_files = inp.get("tf_additional_tfvar_files", []) self.tf_use_workspace = inp.get("tf_use_workspace", True) self.tf_debug_print_terraform_command = inp.get( @@ -33,6 +36,8 @@ class TerraformDomain(Validateable): result += self.__validate_is_not_empty__("module") result += self.__validate_is_not_empty__("stage") result += self.__validate_is_not_empty__("tf_build_commons_dir_name") + result += self.__validate_is_not_none__("tf_additional_resources_from_package") + result += self.__validate_is_not_none__("tf_additional_tfvar_files") return result def output_json_name(self) -> str: @@ -50,3 +55,8 @@ class TerraformDomain(Validateable): if self.tf_additional_vars: ret.update(self.tf_additional_vars) return ret + + def resources_from_package(self) -> List[str]: + result = ["version.tf", "terraform_build_vars.tf"] + result += self.tf_additional_resources_from_package + return result diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 720e9ff..86d8907 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -36,3 +36,12 @@ def test_should_calculate_project_vars(): config = devops_config({}) sut = TerraformDomain(config) assert {'module': 'module', 'stage': 'test'} == sut.project_vars() + +def test_should_calculate_resources_from_package(): + config = devops_config({}) + sut = TerraformDomain(config) + assert ["version.tf", "terraform_build_vars.tf"] == sut.resources_from_package() + + config = devops_config({"tf_additional_resources_from_package": ["my.file"]}) + sut = TerraformDomain(config) + assert ["version.tf", "terraform_build_vars.tf", "my.file"] == sut.resources_from_package() \ No newline at end of file From 85aa41b2a87a52fdf365d48aaa54a8cc765f33b3 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 25 May 2023 16:48:05 +0200 Subject: [PATCH 141/173] use enhanced terraform structure --- .../application/terraform_service.py | 7 +- .../ddadevops/digitalocean_terraform_build.py | 113 +++++++++++------- .../python/ddadevops/domain/init_service.py | 13 ++ src/test/python/domain/helper.py | 4 +- 4 files changed, 85 insertions(+), 52 deletions(-) diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index a2bf4ed..de7d68d 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -32,10 +32,9 @@ class TerraformService: ) def __copy_build_resources_from_package__(self, devops: Devops): - self.__copy_build_resource_file_from_package__("versions.tf", devops) - self.__copy_build_resource_file_from_package__( - "terraform_build_vars.tf", devops - ) + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + for resource in terraform_domain.resources_from_package(): + self.__copy_build_resource_file_from_package__(resource, devops) def __copy_build_resources_from_dir__(self, devops: Devops): terraform = devops.specialized_builds[BuildType.TERRAFORM] diff --git a/src/main/python/ddadevops/digitalocean_terraform_build.py b/src/main/python/ddadevops/digitalocean_terraform_build.py index 3c2f286..7d8ae0a 100644 --- a/src/main/python/ddadevops/digitalocean_terraform_build.py +++ b/src/main/python/ddadevops/digitalocean_terraform_build.py @@ -1,63 +1,84 @@ -from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config +from .devops_terraform_build import ( + DevopsTerraformBuild, + create_devops_terraform_build_config, +) -def create_digitalocean_terraform_build_config(stage, - project_root_path, - module, - additional_vars, - do_api_key, - do_spaces_access_id, - do_spaces_secret_key, - build_dir_name='target', - output_json_name=None, - use_workspace=True, - use_package_common_files=True, - build_commons_path=None, - terraform_build_commons_dir_name='terraform', - debug_print_terraform_command=False, - additional_tfvar_files=None, - terraform_semantic_version="1.0.8", - ): +def create_digitalocean_terraform_build_config( + stage, + project_root_path, + module, + additional_vars, + do_api_key, + do_spaces_access_id, + do_spaces_secret_key, + build_dir_name="target", + output_json_name=None, + use_workspace=True, + use_package_common_files=True, + build_commons_path=None, + terraform_build_commons_dir_name="terraform", + debug_print_terraform_command=False, + additional_tfvar_files=None, + terraform_semantic_version="1.0.8", +): if not additional_tfvar_files: additional_tfvar_files = [] - config = create_devops_terraform_build_config(stage, - project_root_path, - module, - additional_vars, - build_dir_name, - output_json_name, - use_workspace, - use_package_common_files, - build_commons_path, - terraform_build_commons_dir_name, - debug_print_terraform_command, - additional_tfvar_files, - terraform_semantic_version) - config.update({'DigitaloceanTerraformBuild': - {'do_api_key': do_api_key, - 'do_spaces_access_id': do_spaces_access_id, - 'do_spaces_secret_key': do_spaces_secret_key}}) + config = create_devops_terraform_build_config( + stage, + project_root_path, + module, + additional_vars, + build_dir_name, + output_json_name, + use_workspace, + use_package_common_files, + build_commons_path, + terraform_build_commons_dir_name, + debug_print_terraform_command, + additional_tfvar_files, + terraform_semantic_version, + ) + config.update( + { + "DigitaloceanTerraformBuild": { + "do_api_key": do_api_key, + "do_spaces_access_id": do_spaces_access_id, + "do_spaces_secret_key": do_spaces_secret_key, + } + } + ) return config class DigitaloceanTerraformBuild(DevopsTerraformBuild): - def __init__(self, project, config): + additional_resources = config.get("tf_additional_resources_from_package", []) + additional_resources += [ + "provider_registry.tf", + "do_provider.tf", + "do_mixin_vars.tf", + ] + config["tf_additional_resources_from_package"] = additional_resources + + additional_vars = config.get("tf_additional_vars", {}) + additional_vars.update( + { + "do_api_key": config.get("do_api_key"), + "do_spaces_access_id": config.get("do_spaces_access_id"), + "do_spaces_secret_key": config.get("do_spaces_secret_key"), + } + ) super().__init__(project, config) - do_mixin_config = config['DigitaloceanTerraformBuild'] - self.do_api_key = do_mixin_config['do_api_key'] - self.do_spaces_access_id = do_mixin_config['do_spaces_access_id'] - self.do_spaces_secret_key = do_mixin_config['do_spaces_secret_key'] + + self.terraform_service = TerraformService.prod() def project_vars(self): ret = super().project_vars() - ret['do_api_key'] = self.do_api_key - ret['do_spaces_access_id'] = self.do_spaces_access_id - ret['do_spaces_secret_key'] = self.do_spaces_secret_key + ret["do_api_key"] = self.do_api_key + ret["do_spaces_access_id"] = self.do_spaces_access_id + ret["do_spaces_secret_key"] = self.do_spaces_secret_key return ret def copy_build_resources_from_package(self): super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package('provider_registry.tf') - self.copy_build_resource_file_from_package('do_provider.tf') - self.copy_build_resource_file_from_package('do_mixin_vars.tf') diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 3d50203..cf8f37c 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -63,6 +63,19 @@ class InitService: "name": "image_dockerhub_password", }, ] + if BuildType.DIGITALOCEAN_TERRAFORM in build_types: + default_mappings += [ + { + "gopass_path": "server/devops/digitalocean/s3", + "gopass_field": "id", + "name": "do_spaces_access_id", + }, + { + "gopass_path": "server/devops/digitalocean/s3", + "gopass_field": "secret", + "name": "do_spaces_secret_key", + }, + ] if MixinType.RELEASE in mixin_types: primary_build_file_id = inp.get( diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 8dde2e8..c0012fa 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -24,14 +24,14 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", - "tf_additional_vars": None, + "tf_additional_vars": [], "tf_output_json_name": "the_out.json", "tf_use_workspace": None, "tf_use_package_common_files": None, "tf_build_commons_path": "build_commons_path", "tf_build_commons_dir_name": "terraform", "tf_debug_print_terraform_command": None, - "tf_additional_tfvar_files": None, + "tf_additional_tfvar_files": [], "tf_terraform_semantic_version": None, "do_api_key": "api_key", "do_spaces_access_id": "spaces_id", From cb17b39433d2dd8509936ab853911094a967c71d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 25 May 2023 17:57:57 +0200 Subject: [PATCH 142/173] refactor terraform_devops_build to provider domain object --- doc/architecture/Domain.md | 13 ++-- src/main/python/ddadevops/__init__.py | 1 - .../ddadevops/digitalocean_terraform_build.py | 78 +------------------ src/main/python/ddadevops/domain/__init__.py | 4 +- src/main/python/ddadevops/domain/common.py | 5 +- .../python/ddadevops/domain/devops_factory.py | 5 -- .../python/ddadevops/domain/init_service.py | 2 +- ..._terraform.py => provider_digitalocean.py} | 10 ++- src/main/python/ddadevops/domain/terraform.py | 29 ++++++- src/test/python/domain/helper.py | 1 + src/test/python/domain/test_devops_factory.py | 17 ---- ...aform.py => test_provider_digitalocean.py} | 6 +- src/test/python/domain/test_terraform.py | 72 +++++++++++++++-- 13 files changed, 121 insertions(+), 122 deletions(-) rename src/main/python/ddadevops/domain/{digitalocean_terraform.py => provider_digitalocean.py} (61%) rename src/test/python/domain/{test_digitalocean_terraform.py => test_provider_digitalocean.py} (81%) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index e0971e2..c6d6da6 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -50,6 +50,12 @@ classDiagram tf_terraform_semantic_version } + class Digitalocean { + do_api_key + do_spaces_access_key + do_spaces_secret_key + } + class DnsRecord { fqdn ipv4 @@ -92,18 +98,13 @@ classDiagram create_bump(snapshot_suffix) } - class DigitaloceanTerraform { - do_api_key - do_spaces_access_key - do_spaces_secret_key - } Devops *-- "0..1" Image: specialized_builds Devops *-- "0..1" C4k: specialized_builds Devops *-- "0..1" ProvsK3s: specialized_builds Devops *-- "0..1" TerraformDomain: specialized_builds - Devops *-- "0..1" DigitaloceanTerraform: specialized_builds Devops *-- "0..1" Release: mixins + TerraformDomain *-- "0..1" Digitalocean: provider Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index 7cde593..f66936f 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -10,7 +10,6 @@ from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config from .c4k_build import C4kBuild from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config -from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config from .devops_image_build import DevopsImageBuild from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config diff --git a/src/main/python/ddadevops/digitalocean_terraform_build.py b/src/main/python/ddadevops/digitalocean_terraform_build.py index 7d8ae0a..685de23 100644 --- a/src/main/python/ddadevops/digitalocean_terraform_build.py +++ b/src/main/python/ddadevops/digitalocean_terraform_build.py @@ -4,81 +4,9 @@ from .devops_terraform_build import ( ) -def create_digitalocean_terraform_build_config( - stage, - project_root_path, - module, - additional_vars, - do_api_key, - do_spaces_access_id, - do_spaces_secret_key, - build_dir_name="target", - output_json_name=None, - use_workspace=True, - use_package_common_files=True, - build_commons_path=None, - terraform_build_commons_dir_name="terraform", - debug_print_terraform_command=False, - additional_tfvar_files=None, - terraform_semantic_version="1.0.8", -): - if not additional_tfvar_files: - additional_tfvar_files = [] - config = create_devops_terraform_build_config( - stage, - project_root_path, - module, - additional_vars, - build_dir_name, - output_json_name, - use_workspace, - use_package_common_files, - build_commons_path, - terraform_build_commons_dir_name, - debug_print_terraform_command, - additional_tfvar_files, - terraform_semantic_version, - ) - config.update( - { - "DigitaloceanTerraformBuild": { - "do_api_key": do_api_key, - "do_spaces_access_id": do_spaces_access_id, - "do_spaces_secret_key": do_spaces_secret_key, - } - } - ) - return config +def create_digitalocean_terraform_build_config(): + pass class DigitaloceanTerraformBuild(DevopsTerraformBuild): - def __init__(self, project, config): - additional_resources = config.get("tf_additional_resources_from_package", []) - additional_resources += [ - "provider_registry.tf", - "do_provider.tf", - "do_mixin_vars.tf", - ] - config["tf_additional_resources_from_package"] = additional_resources - - additional_vars = config.get("tf_additional_vars", {}) - additional_vars.update( - { - "do_api_key": config.get("do_api_key"), - "do_spaces_access_id": config.get("do_spaces_access_id"), - "do_spaces_secret_key": config.get("do_spaces_secret_key"), - } - ) - super().__init__(project, config) - - self.terraform_service = TerraformService.prod() - - def project_vars(self): - ret = super().project_vars() - ret["do_api_key"] = self.do_api_key - ret["do_spaces_access_id"] = self.do_spaces_access_id - ret["do_spaces_secret_key"] = self.do_spaces_secret_key - return ret - - def copy_build_resources_from_package(self): - super().copy_build_resources_from_package() + pass diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index e8f0454..d6258b4 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,9 +1,9 @@ -from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, ReleaseType +from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, ReleaseType, ProviderType from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k from .terraform import TerraformDomain -from .digitalocean_terraform import DigitaloceanTerraform +from .provider_digitalocean import Digitalocean from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index af62ed0..3495b2b 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -11,7 +11,10 @@ class BuildType(Enum): C4K = 1 K3S = 2 TERRAFORM = 3 - DIGITALOCEAN_TERRAFORM = 4 + + +class ProviderType(Enum): + DIGITALOCEAN = 0 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/devops_factory.py b/src/main/python/ddadevops/domain/devops_factory.py index 07ce39a..915bc4c 100644 --- a/src/main/python/ddadevops/domain/devops_factory.py +++ b/src/main/python/ddadevops/domain/devops_factory.py @@ -4,7 +4,6 @@ from .image import Image from .c4k import C4k from .provs_k3s import K3s from .terraform import TerraformDomain -from .digitalocean_terraform import DigitaloceanTerraform from .release import Release from .version import Version @@ -26,10 +25,6 @@ class DevopsFactory: specialized_builds[BuildType.K3S] = K3s(inp) if BuildType.TERRAFORM in build_types: specialized_builds[BuildType.TERRAFORM] = TerraformDomain(inp) - if BuildType.DIGITALOCEAN_TERRAFORM in build_types: - specialized_builds[ - BuildType.DIGITALOCEAN_TERRAFORM - ] = DigitaloceanTerraform(inp) mixins: Dict[MixinType, Validateable] = {} if MixinType.RELEASE in mixin_types: diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index cf8f37c..ca0b9ec 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -63,7 +63,7 @@ class InitService: "name": "image_dockerhub_password", }, ] - if BuildType.DIGITALOCEAN_TERRAFORM in build_types: + if False: default_mappings += [ { "gopass_path": "server/devops/digitalocean/s3", diff --git a/src/main/python/ddadevops/domain/digitalocean_terraform.py b/src/main/python/ddadevops/domain/provider_digitalocean.py similarity index 61% rename from src/main/python/ddadevops/domain/digitalocean_terraform.py rename to src/main/python/ddadevops/domain/provider_digitalocean.py index 10585ce..a27057d 100644 --- a/src/main/python/ddadevops/domain/digitalocean_terraform.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -4,7 +4,7 @@ from .common import ( ) -class DigitaloceanTerraform(Validateable): +class Digitalocean(Validateable): def __init__( self, inp: dict, @@ -19,3 +19,11 @@ class DigitaloceanTerraform(Validateable): result += self.__validate_is_not_empty__("do_spaces_access_id") result += self.__validate_is_not_empty__("do_spaces_secret_key") return result + + def resources_from_package(self) -> List[str]: + return ["provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"] + + def project_vars(self): + return {"do_api_key": self.do_api_key, + "do_spaces_access_id": self.do_spaces_access_id, + "do_spaces_secret_key": self.do_spaces_secret_key,} diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index f5cecf4..5a46073 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -2,10 +2,12 @@ from typing import List, Optional from pathlib import Path from .common import ( Validateable, + ProviderType, DnsRecord, Devops, filter_none, ) +from .provider_digitalocean import Digitalocean class TerraformDomain(Validateable): @@ -15,6 +17,7 @@ class TerraformDomain(Validateable): self.tf_additional_vars = inp.get("tf_additional_vars") self.tf_output_json_name = inp.get("tf_output_json_name") self.tf_build_commons_path = inp.get("tf_build_commons_path") + self.tf_provider_types = inp.get("tf_provider_types", []) self.tf_additional_resources_from_package = inp.get( "tf_additional_resources_from_package", [] ) @@ -31,6 +34,11 @@ class TerraformDomain(Validateable): ) self.tf_use_package_common_files = inp.get("tf_use_package_common_files", True) + provider_types = self.__parse_provider_types__(self.tf_provider_types) + self.providers = {} + if ProviderType.DIGITALOCEAN in provider_types: + self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp) + def validate(self) -> List[str]: result = [] result += self.__validate_is_not_empty__("module") @@ -38,6 +46,9 @@ class TerraformDomain(Validateable): result += self.__validate_is_not_empty__("tf_build_commons_dir_name") result += self.__validate_is_not_none__("tf_additional_resources_from_package") result += self.__validate_is_not_none__("tf_additional_tfvar_files") + result += self.__validate_is_not_none__("tf_provider_types") + for provider in self.providers.values(): + result += provider.validate() return result def output_json_name(self) -> str: @@ -51,12 +62,24 @@ class TerraformDomain(Validateable): return Path("/".join(filter_none(mylist)) + "/") def project_vars(self): - ret = {"stage": self.stage, "module": self.module} + result = {"stage": self.stage, "module": self.module} + for provider in self.providers.values(): + result.update(provider.project_vars()) if self.tf_additional_vars: - ret.update(self.tf_additional_vars) - return ret + result.update(self.tf_additional_vars) + return result def resources_from_package(self) -> List[str]: result = ["version.tf", "terraform_build_vars.tf"] + for provider in self.providers.values(): + result += provider.resources_from_package() result += self.tf_additional_resources_from_package return result + + def __parse_provider_types__( + self, tf_provider_types: List[str] + ) -> List[ProviderType]: + result = [] + for provider_type in tf_provider_types: + result.append(ProviderType[provider_type]) + return result diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index c0012fa..d18f276 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -24,6 +24,7 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", + "tf_provider_types": ["DIGITALOCEAN"], "tf_additional_vars": [], "tf_output_json_name": "the_out.json", "tf_use_workspace": None, diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index 2ea8d1b..60bab56 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -50,23 +50,6 @@ def test_devops_factory(): assert sut is not None assert sut.specialized_builds[BuildType.C4K] is not None - sut = DevopsFactory().build_devops( - { - "stage": "test", - "name": "mybuild", - "module": "test_image", - "project_root_path": "../../..", - "build_types": ["DIGITALOCEAN_TERRAFORM"], - "mixin_types": [], - "do_api_key": "api_key", - "do_spaces_access_id": "spaces_id", - "do_spaces_secret_key": "spaces_secret", - }, - Version.from_str("1.0.0"), - ) - assert sut is not None - assert sut.specialized_builds[BuildType.DIGITALOCEAN_TERRAFORM] is not None - sut = DevopsFactory().build_devops( { "stage": "test", diff --git a/src/test/python/domain/test_digitalocean_terraform.py b/src/test/python/domain/test_provider_digitalocean.py similarity index 81% rename from src/test/python/domain/test_digitalocean_terraform.py rename to src/test/python/domain/test_provider_digitalocean.py index 7e6e6d5..d24f45f 100644 --- a/src/test/python/domain/test_digitalocean_terraform.py +++ b/src/test/python/domain/test_provider_digitalocean.py @@ -2,13 +2,13 @@ from pybuilder.core import Project from pathlib import Path from src.main.python.ddadevops.domain import ( BuildType, - DigitaloceanTerraform, + Digitalocean, ) from .helper import devops_config -def test_digitalocean_terraform(): - sut = DigitaloceanTerraform( +def test_digitalocean_creation(): + sut = Digitalocean( devops_config( { "do_api_key": "api_key", diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 86d8907..22285af 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -1,13 +1,20 @@ import pytest from pathlib import Path -from src.main.python.ddadevops.domain import DnsRecord, BuildType, TerraformDomain +from src.main.python.ddadevops.domain import ( + DnsRecord, + BuildType, + ProviderType, + TerraformDomain, +) from .helper import build_devops, devops_config def test_creation(): - sut = build_devops({}) - assert BuildType.TERRAFORM in sut.specialized_builds - assert sut.specialized_builds[BuildType.TERRAFORM] + devops = build_devops({}) + sut = devops.specialized_builds[BuildType.TERRAFORM] + assert BuildType.TERRAFORM in devops.specialized_builds + assert sut + assert sut.providers[ProviderType.DIGITALOCEAN] def test_should_calculate_output_json_name(): @@ -21,6 +28,20 @@ def test_should_calculate_output_json_name(): assert "out_module.json" == sut.output_json_name() +def test_should_validate(): + config = devops_config({}) + sut = TerraformDomain(config) + assert sut.is_valid() + + config = devops_config( + { + "do_api_key": "", + } + ) + sut = TerraformDomain(config) + assert not sut.is_valid() + + def test_should_calculate_terraform_build_commons_path(): config = devops_config({}) del config["tf_build_commons_path"] @@ -32,16 +53,53 @@ def test_should_calculate_terraform_build_commons_path(): sut = TerraformDomain(config) assert Path("build_commons_path/terraform") == sut.terraform_build_commons_path() + def test_should_calculate_project_vars(): + config = devops_config( + { + "tf_provider_types": [], + } + ) + sut = TerraformDomain(config) + assert {"module": "module", "stage": "test"} == sut.project_vars() + config = devops_config({}) sut = TerraformDomain(config) - assert {'module': 'module', 'stage': 'test'} == sut.project_vars() + assert { + "module": "module", + "stage": "test", + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + } == sut.project_vars() + def test_should_calculate_resources_from_package(): - config = devops_config({}) + config = devops_config( + { + "tf_provider_types": [], + } + ) sut = TerraformDomain(config) assert ["version.tf", "terraform_build_vars.tf"] == sut.resources_from_package() + config = devops_config({}) + sut = TerraformDomain(config) + assert [ + "version.tf", + "terraform_build_vars.tf", + "provider_registry.tf", + "do_provider.tf", + "do_mixin_vars.tf", + ] == sut.resources_from_package() + config = devops_config({"tf_additional_resources_from_package": ["my.file"]}) sut = TerraformDomain(config) - assert ["version.tf", "terraform_build_vars.tf", "my.file"] == sut.resources_from_package() \ No newline at end of file + assert [ + "version.tf", + "terraform_build_vars.tf", + "provider_registry.tf", + "do_provider.tf", + "do_mixin_vars.tf", + "my.file", + ] == sut.resources_from_package() From 70a671a06d5dc32fc39c710615d1beeca5e15deb Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 25 May 2023 18:28:45 +0200 Subject: [PATCH 143/173] bring credentials-mapping-defaults closer to the domain --- src/main/python/ddadevops/domain/__init__.py | 2 +- src/main/python/ddadevops/domain/c4k.py | 21 ++++++- src/main/python/ddadevops/domain/common.py | 5 ++ src/main/python/ddadevops/domain/image.py | 14 +++++ .../python/ddadevops/domain/init_service.py | 49 +++++----------- .../ddadevops/domain/provider_digitalocean.py | 29 +++++++--- src/main/python/ddadevops/domain/terraform.py | 7 ++- src/test/python/test_release_mixin.py | 56 +------------------ 8 files changed, 79 insertions(+), 104 deletions(-) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index d6258b4..3972995 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,4 +1,4 @@ -from .common import Validateable, DnsRecord, Devops, BuildType, MixinType, ReleaseType, ProviderType +from .common import Validateable, CredentialMappingDefault, DnsRecord, Devops, BuildType, MixinType, ReleaseType, ProviderType from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k diff --git a/src/main/python/ddadevops/domain/c4k.py b/src/main/python/ddadevops/domain/c4k.py index e5cb060..635d770 100644 --- a/src/main/python/ddadevops/domain/c4k.py +++ b/src/main/python/ddadevops/domain/c4k.py @@ -1,12 +1,13 @@ from typing import List, Optional from .common import ( Validateable, + CredentialMappingDefault, DnsRecord, Devops, ) -class C4k(Validateable): +class C4k(Validateable, CredentialMappingDefault): def __init__(self, inp: dict): self.module = inp.get("module") self.stage = inp.get("stage") @@ -17,8 +18,8 @@ class C4k(Validateable): "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", ) self.c4k_auth = inp.get("c4k_auth", {}) - self.c4k_grafana_cloud_user = inp.get('c4k_grafana_cloud_user') - self.c4k_grafana_cloud_password = inp.get('c4k_grafana_cloud_password') + self.c4k_grafana_cloud_user = inp.get("c4k_grafana_cloud_user") + self.c4k_grafana_cloud_password = inp.get("c4k_grafana_cloud_password") self.dns_record: Optional[DnsRecord] = None def update_runtime_config(self, dns_record: DnsRecord): @@ -63,3 +64,17 @@ class C4k(Validateable): auth_path = f"{build_path}/out_c4k_auth.yaml" output_path = f"{build_path}/out_{module}.yaml" return f"c4k-{self.c4k_executable_name}-standalone.jar {config_path} {auth_path} > {output_path}" + + @classmethod + def get_mapping_default(cls) -> List[map]: + return [ + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + "name": "c4k_grafana_cloud_user", + }, + { + "gopass_path": "server/meissa/grafana-cloud", + "name": "c4k_grafana_cloud_password", + }, + ] diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 3495b2b..6eba93f 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -57,6 +57,11 @@ class Validateable: raise ValueError(f"Invalid Validateable: {issues}") +class CredentialMappingDefault: + @classmethod + def get_mapping_default(cls) -> List[map]: + return [] + class DnsRecord(Validateable): def __init__(self, fqdn, ipv4=None, ipv6=None): self.fqdn = fqdn diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index a3cd888..5a1a5f6 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -36,3 +36,17 @@ class Image(Validateable): self.image_build_commons_dir_name, ] return "/".join(filter_none(commons_path)) + "/" + + @classmethod + def get_mapping_default(cls) -> List[map]: + return [ + { + "gopass_path": "meissa/web/docker.com", + "gopass_field": "login", + "name": "image_dockerhub_user", + }, + { + "gopass_path": "meissa/web/docker.com", + "name": "image_dockerhub_password", + }, + ] diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index ca0b9ec..d44046b 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -1,8 +1,12 @@ from pathlib import Path from typing import Dict -from .common import Devops, MixinType, BuildType +from .common import Devops, MixinType, BuildType, ProviderType from .credentials import CredentialMapping, Credentials, GopassType from .devops_factory import DevopsFactory +from .terraform import TerraformDomain +from .provider_digitalocean import Digitalocean +from .c4k import C4k +from .image import Image from .release import ReleaseType from ..infrastructure import BuildFileRepository, CredentialsApi, EnvironmentApi, GitApi @@ -35,47 +39,20 @@ class InitService: def initialize(self, inp: dict) -> Devops: build_types = self.devops_factory.__parse_build_types__(inp["build_types"]) mixin_types = self.devops_factory.__parse_mixin_types__(inp["mixin_types"]) + provider_types = TerraformDomain.parse_provider_types(inp["tf_provider_types"]) version = None default_mappings = [] if BuildType.C4K in build_types: - default_mappings += [ - { - "gopass_path": "server/meissa/grafana-cloud", - "gopass_field": "grafana-cloud-user", - "name": "c4k_grafana_cloud_user", - }, - { - "gopass_path": "server/meissa/grafana-cloud", - "name": "c4k_grafana_cloud_password", - }, - ] + default_mappings += C4k.get_mapping_default() if BuildType.IMAGE in build_types: - default_mappings += [ - { - "gopass_path": "meissa/web/docker.com", - "gopass_field": "login", - "name": "image_dockerhub_user", - }, - { - "gopass_path": "meissa/web/docker.com", - "name": "image_dockerhub_password", - }, - ] - if False: - default_mappings += [ - { - "gopass_path": "server/devops/digitalocean/s3", - "gopass_field": "id", - "name": "do_spaces_access_id", - }, - { - "gopass_path": "server/devops/digitalocean/s3", - "gopass_field": "secret", - "name": "do_spaces_secret_key", - }, - ] + default_mappings += Image.get_mapping_default() + if ( + BuildType.TERRAFORM in build_types + and ProviderType.DIGITALOCEAN in provider_types + ): + default_mappings += Digitalocean.get_mapping_default() if MixinType.RELEASE in mixin_types: primary_build_file_id = inp.get( diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index a27057d..3325567 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -1,10 +1,8 @@ from typing import List -from .common import ( - Validateable, -) +from .common import Validateable, CredentialMappingDefault -class Digitalocean(Validateable): +class Digitalocean(Validateable, CredentialMappingDefault): def __init__( self, inp: dict, @@ -24,6 +22,23 @@ class Digitalocean(Validateable): return ["provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"] def project_vars(self): - return {"do_api_key": self.do_api_key, - "do_spaces_access_id": self.do_spaces_access_id, - "do_spaces_secret_key": self.do_spaces_secret_key,} + return { + "do_api_key": self.do_api_key, + "do_spaces_access_id": self.do_spaces_access_id, + "do_spaces_secret_key": self.do_spaces_secret_key, + } + + @classmethod + def get_mapping_default(cls) -> List[map]: + return [ + { + "gopass_path": "server/devops/digitalocean/s3", + "gopass_field": "id", + "name": "do_spaces_access_id", + }, + { + "gopass_path": "server/devops/digitalocean/s3", + "gopass_field": "secret", + "name": "do_spaces_secret_key", + }, + ] diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 5a46073..4f5ba89 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -34,7 +34,7 @@ class TerraformDomain(Validateable): ) self.tf_use_package_common_files = inp.get("tf_use_package_common_files", True) - provider_types = self.__parse_provider_types__(self.tf_provider_types) + provider_types = TerraformDomain.parse_provider_types(self.tf_provider_types) self.providers = {} if ProviderType.DIGITALOCEAN in provider_types: self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp) @@ -76,8 +76,9 @@ class TerraformDomain(Validateable): result += self.tf_additional_resources_from_package return result - def __parse_provider_types__( - self, tf_provider_types: List[str] + @classmethod + def parse_provider_types( + cls, tf_provider_types: List[str] ) -> List[ProviderType]: result = [] for provider_type in tf_provider_types: diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py index a266630..291eb73 100644 --- a/src/test/python/test_release_mixin.py +++ b/src/test/python/test_release_mixin.py @@ -8,9 +8,10 @@ from src.main.python.ddadevops.domain import Devops, Release from .domain.helper import devops_config from .resource_helper import copy_resource + def test_release_mixin(tmp_path): str_tmp_path = str(tmp_path) - copy_resource(Path('package.json'), tmp_path) + copy_resource(Path("package.json"), tmp_path) project = Project(str_tmp_path, name="name") sut = ReleaseMixin( @@ -27,56 +28,3 @@ def test_release_mixin(tmp_path): sut.initialize_build_dir() assert sut.build_path() == f"{str_tmp_path}/target/name/release-test" - - -# def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch): -# # init -# th = ResourceHelper() -# th.copy_files(th.TEST_FILE_PATH, tmp_path) -# th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - -# change_test_dir(tmp_path, monkeypatch) -# project = Project(tmp_path) - -# git_api = GitApi() -# git_api.init() -# git_api.set_user_config("ex.ample@mail.com", "Ex Ample") -# git_api.add_file(th.TEST_FILE_NAME) -# git_api.commit("MAJOR release") - -# build = initialize_with_object(project, th.TEST_FILE_PATH) -# build.prepare_release() -# release_version = build.release_repo.version_repository.get_version() - -# # test -# assert "124.0.1-SNAPSHOT" in release_version.get_version_string() - - -# def test_release_mixin_environment(tmp_path: Path, monkeypatch: pt.MonkeyPatch): - -# # init -# th = Helper() -# th.copy_files(th.TEST_FILE_PATH, tmp_path) -# th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME - -# change_test_dir(tmp_path, monkeypatch) -# project = Project(tmp_path) - -# git_api = GitApi() -# git_api.init() -# git_api.set_user_config("ex.ample@mail.com", "Ex Ample") -# git_api.add_file(th.TEST_FILE_NAME) -# git_api.commit("Commit Message") - -# environment_api = EnvironmentApi() -# environment_api.set("DDADEVOPS_RELEASE_TYPE", "MAJOR") - -# build = initialize_with_object(project, th.TEST_FILE_PATH) -# build.prepare_release() -# release_version = build.release_repo.version_repository.get_version() - -# # test -# assert "124.0.1-SNAPSHOT" in release_version.get_version_string() - -# # tear down -# environment_api.set("DDADEVOPS_RELEASE_TYPE", "") From dc9236d7661cbf23b8530c5a303acecc655c64a5 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 25 May 2023 19:19:34 +0200 Subject: [PATCH 144/173] minor fixes --- build.py | 2 +- .../python/ddadevops/application/terraform_service.py | 11 ++++++----- src/main/python/ddadevops/devops_terraform_build.py | 2 +- .../python/ddadevops/digitalocean_terraform_build.py | 1 - src/main/python/ddadevops/domain/c4k.py | 4 ++-- src/main/python/ddadevops/domain/common.py | 5 +++-- src/main/python/ddadevops/domain/image.py | 4 ++-- src/main/python/ddadevops/domain/init_service.py | 6 +++--- .../python/ddadevops/domain/provider_digitalocean.py | 4 ++-- src/main/python/ddadevops/domain/terraform.py | 6 ++---- src/main/python/ddadevops/domain/version.py | 2 +- .../python/ddadevops/infrastructure/infrastructure.py | 4 ++-- 12 files changed, 25 insertions(+), 26 deletions(-) diff --git a/build.py b/build.py index 06c0dc9..8b23190 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev52" +version = "4.0.0-dev57" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index de7d68d..9dfc483 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -2,7 +2,7 @@ from pathlib import Path from dda_python_terraform import Terraform, IsFlagged from packaging import version -from ..domain import Devops, BuildType, TerraformDomain +from ..domain import Devops, BuildType from ..infrastructure import FileApi, ResourceApi, TerraformApi @@ -79,10 +79,10 @@ class TerraformService: self.__print_terraform_command__(terraform, devops) if terraform_domain.tf_use_workspace: try: - terraform.workspace("select", self.stage) + terraform.workspace("select", devops.stage) self.__print_terraform_command__(terraform, devops) except: - terraform.workspace("new", self.stage) + terraform.workspace("new", devops.stage) self.__print_terraform_command__(terraform, devops) return terraform @@ -115,7 +115,7 @@ class TerraformService: var=terraform_domain.project_vars(), var_file=terraform_domain.tf_additional_tfvar_files, ) - self.__print_terraform_command__(terraform) + self.__print_terraform_command__(terraform, devops) if return_code not in (0, 2): raise RuntimeError(return_code, "terraform error:", stderr) if return_code == 2: @@ -199,6 +199,7 @@ class TerraformService: tf_import_resource, ): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + terraform = self.init_client(devops) return_code, _, stderr = terraform.import_cmd( tf_import_name, tf_import_resource, @@ -207,6 +208,6 @@ class TerraformService: var=terraform_domain.project_vars(), var_file=terraform_domain.tf_additional_tfvar_files, ) - self.print_terraform_command(terraform, devops) + self.__print_terraform_command__(terraform, devops) if return_code > 0: raise RuntimeError(return_code, "terraform error:", stderr) diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 6b0f8ce..b475358 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -86,7 +86,7 @@ class DevopsTerraformBuild(DevopsBuild): def destroy(self, auto_approve=False): devops = self.devops_repo.get_devops(self.project) - self.teraform_service.refresh(devops) + self.teraform_service.destroy(devops, auto_approve=auto_approve) self.post_build() def tf_import( diff --git a/src/main/python/ddadevops/digitalocean_terraform_build.py b/src/main/python/ddadevops/digitalocean_terraform_build.py index 685de23..8a285d3 100644 --- a/src/main/python/ddadevops/digitalocean_terraform_build.py +++ b/src/main/python/ddadevops/digitalocean_terraform_build.py @@ -1,6 +1,5 @@ from .devops_terraform_build import ( DevopsTerraformBuild, - create_devops_terraform_build_config, ) diff --git a/src/main/python/ddadevops/domain/c4k.py b/src/main/python/ddadevops/domain/c4k.py index 635d770..ffca857 100644 --- a/src/main/python/ddadevops/domain/c4k.py +++ b/src/main/python/ddadevops/domain/c4k.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List, Dict, Optional from .common import ( Validateable, CredentialMappingDefault, @@ -66,7 +66,7 @@ class C4k(Validateable, CredentialMappingDefault): return f"c4k-{self.c4k_executable_name}-standalone.jar {config_path} {auth_path} > {output_path}" @classmethod - def get_mapping_default(cls) -> List[map]: + def get_mapping_default(cls) -> List[Dict[str, str]]: return [ { "gopass_path": "server/meissa/grafana-cloud", diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 6eba93f..3533b04 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import List +from typing import List, Dict def filter_none(list_to_filter): @@ -59,9 +59,10 @@ class Validateable: class CredentialMappingDefault: @classmethod - def get_mapping_default(cls) -> List[map]: + def get_mapping_default(cls) -> List[Dict[str, str]]: return [] + class DnsRecord(Validateable): def __init__(self, fqdn, ipv4=None, ipv6=None): self.fqdn = fqdn diff --git a/src/main/python/ddadevops/domain/image.py b/src/main/python/ddadevops/domain/image.py index 5a1a5f6..d24b59b 100644 --- a/src/main/python/ddadevops/domain/image.py +++ b/src/main/python/ddadevops/domain/image.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Dict from .common import ( filter_none, Validateable, @@ -38,7 +38,7 @@ class Image(Validateable): return "/".join(filter_none(commons_path)) + "/" @classmethod - def get_mapping_default(cls) -> List[map]: + def get_mapping_default(cls) -> List[Dict[str, str]]: return [ { "gopass_path": "meissa/web/docker.com", diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index d44046b..0adbaa1 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -37,9 +37,9 @@ class InitService: ) def initialize(self, inp: dict) -> Devops: - build_types = self.devops_factory.__parse_build_types__(inp["build_types"]) - mixin_types = self.devops_factory.__parse_mixin_types__(inp["mixin_types"]) - provider_types = TerraformDomain.parse_provider_types(inp["tf_provider_types"]) + build_types = self.devops_factory.__parse_build_types__(inp.get("build_types", [])) + mixin_types = self.devops_factory.__parse_mixin_types__(inp.get("mixin_types", [])) + provider_types = TerraformDomain.parse_provider_types(inp.get("tf_provider_types", [])) version = None default_mappings = [] diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index 3325567..aeded28 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Dict from .common import Validateable, CredentialMappingDefault @@ -29,7 +29,7 @@ class Digitalocean(Validateable, CredentialMappingDefault): } @classmethod - def get_mapping_default(cls) -> List[map]: + def get_mapping_default(cls) -> List[Dict[str, str]]: return [ { "gopass_path": "server/devops/digitalocean/s3", diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 4f5ba89..ddce425 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -1,10 +1,8 @@ -from typing import List, Optional +from typing import List from pathlib import Path from .common import ( Validateable, ProviderType, - DnsRecord, - Devops, filter_none, ) from .provider_digitalocean import Digitalocean @@ -70,7 +68,7 @@ class TerraformDomain(Validateable): return result def resources_from_package(self) -> List[str]: - result = ["version.tf", "terraform_build_vars.tf"] + result = ["versions.tf", "terraform_build_vars.tf"] for provider in self.providers.values(): result += provider.resources_from_package() result += self.tf_additional_resources_from_package diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 22d2566..7fd8b73 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -36,7 +36,7 @@ class Version(Validateable): return self.to_string().__hash__() def is_snapshot(self): - return not self.snapshot_suffix is None + return self.snapshot_suffix is not None def to_string(self) -> str: version_no = ".".join([str(x) for x in self.version_list]) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 80224ca..18edca1 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -1,10 +1,10 @@ -from subprocess import check_output, Popen, PIPE, run +from subprocess import Popen, PIPE, run from pathlib import Path from sys import stdout from os import chmod, environ -from pkg_resources import resource_string from json import load, dumps import yaml +from pkg_resources import resource_string class ResourceApi: From 3d5506ac93c1e810b44aa6c81e04c017d34c7c31 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 26 May 2023 08:19:22 +0200 Subject: [PATCH 145/173] Fix typo --- src/test/python/domain/test_terraform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 22285af..da61342 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -81,12 +81,12 @@ def test_should_calculate_resources_from_package(): } ) sut = TerraformDomain(config) - assert ["version.tf", "terraform_build_vars.tf"] == sut.resources_from_package() + assert ["versions.tf", "terraform_build_vars.tf"] == sut.resources_from_package() config = devops_config({}) sut = TerraformDomain(config) assert [ - "version.tf", + "versions.tf", "terraform_build_vars.tf", "provider_registry.tf", "do_provider.tf", @@ -96,7 +96,7 @@ def test_should_calculate_resources_from_package(): config = devops_config({"tf_additional_resources_from_package": ["my.file"]}) sut = TerraformDomain(config) assert [ - "version.tf", + "versions.tf", "terraform_build_vars.tf", "provider_registry.tf", "do_provider.tf", From b26cf9ff0a0c8cff5808eeca887cfdf2c2b5bc22 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 26 May 2023 08:29:32 +0200 Subject: [PATCH 146/173] Implement Hetzner provider --- src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 1 + .../python/ddadevops/domain/init_service.py | 6 +++++ .../ddadevops/domain/provider_hetzner.py | 26 +++++++++++++++++++ src/main/python/ddadevops/domain/terraform.py | 3 +++ src/test/python/domain/helper.py | 3 ++- .../python/domain/test_provider_hetzner.py | 19 ++++++++++++++ src/test/python/domain/test_terraform.py | 25 +++++++++++++++++- 8 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 src/main/python/ddadevops/domain/provider_hetzner.py create mode 100644 src/test/python/domain/test_provider_hetzner.py diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 3972995..b281a61 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -4,6 +4,7 @@ from .image import Image from .c4k import C4k from .terraform import TerraformDomain from .provider_digitalocean import Digitalocean +from .provider_hetzner import Hetzner from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 3533b04..01dc29d 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -15,6 +15,7 @@ class BuildType(Enum): class ProviderType(Enum): DIGITALOCEAN = 0 + HETZNER = 1 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 0adbaa1..76a23db 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -5,6 +5,7 @@ from .credentials import CredentialMapping, Credentials, GopassType from .devops_factory import DevopsFactory from .terraform import TerraformDomain from .provider_digitalocean import Digitalocean +from .provider_hetzner import Hetzner from .c4k import C4k from .image import Image from .release import ReleaseType @@ -53,6 +54,11 @@ class InitService: and ProviderType.DIGITALOCEAN in provider_types ): default_mappings += Digitalocean.get_mapping_default() + if ( + BuildType.TERRAFORM in build_types + and ProviderType.HETZNER in provider_types + ): + default_mappings += Hetzner.get_mapping_default() if MixinType.RELEASE in mixin_types: primary_build_file_id = inp.get( diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py new file mode 100644 index 0000000..636b007 --- /dev/null +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -0,0 +1,26 @@ +from typing import List, Dict +from .common import Validateable, CredentialMappingDefault + +class Hetzner(Validateable, CredentialMappingDefault): + def __init__( + self, + inp: dict, + ): + self.hetzner_api_key = inp.get("hetzner_api_key") + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("hetzner_api_key") + return result + + def resources_from_package(self) -> List[str]: + return ["provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"] + + def project_vars(self): + return { + "hetzner_api_key": self.hetzner_api_key + } + + @classmethod + def get_mapping_default(cls) -> List[Dict[str, str]]: + return [] \ No newline at end of file diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index ddce425..42304af 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -6,6 +6,7 @@ from .common import ( filter_none, ) from .provider_digitalocean import Digitalocean +from .provider_hetzner import Hetzner class TerraformDomain(Validateable): @@ -36,6 +37,8 @@ class TerraformDomain(Validateable): self.providers = {} if ProviderType.DIGITALOCEAN in provider_types: self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp) + if ProviderType.HETZNER in provider_types: + self.providers[ProviderType.HETZNER] = Hetzner(inp) def validate(self) -> List[str]: result = [] diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index d18f276..a72a0ff 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -24,7 +24,7 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", - "tf_provider_types": ["DIGITALOCEAN"], + "tf_provider_types": ["DIGITALOCEAN", "HETZNER"], "tf_additional_vars": [], "tf_output_json_name": "the_out.json", "tf_use_workspace": None, @@ -37,6 +37,7 @@ def devops_config(overrides: dict) -> dict: "do_api_key": "api_key", "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", + "hetzner_api_key": "hetzner_api_key", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_provider_hetzner.py b/src/test/python/domain/test_provider_hetzner.py new file mode 100644 index 0000000..adc0479 --- /dev/null +++ b/src/test/python/domain/test_provider_hetzner.py @@ -0,0 +1,19 @@ +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain import ( + BuildType, + Hetzner, +) +from .helper import devops_config + + +def test_hetzner_creation(): + sut = Hetzner( + devops_config( + { + "hetzner_api_key": "api_key", + } + ) + ) + assert sut is not None + assert sut.is_valid() diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index da61342..ac6a377 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -15,6 +15,7 @@ def test_creation(): assert BuildType.TERRAFORM in devops.specialized_builds assert sut assert sut.providers[ProviderType.DIGITALOCEAN] + assert sut.providers[ProviderType.HETZNER] def test_should_calculate_output_json_name(): @@ -71,6 +72,7 @@ def test_should_calculate_project_vars(): "do_api_key": "api_key", "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", + "hetzner_api_key": "hetzner_api_key" } == sut.project_vars() @@ -83,7 +85,11 @@ def test_should_calculate_resources_from_package(): sut = TerraformDomain(config) assert ["versions.tf", "terraform_build_vars.tf"] == sut.resources_from_package() - config = devops_config({}) + config = devops_config( + { + "tf_provider_types": ["DIGITALOCEAN"] + } + ) sut = TerraformDomain(config) assert [ "versions.tf", @@ -93,6 +99,20 @@ def test_should_calculate_resources_from_package(): "do_mixin_vars.tf", ] == sut.resources_from_package() + config = devops_config( + { + "tf_provider_types": ["HETZNER"] + } + ) + sut = TerraformDomain(config) + assert [ + "versions.tf", + "terraform_build_vars.tf", + "provider_registry.tf", + "hetzner_provider.tf", + "hetzner_mixin_vars.tf", + ] == sut.resources_from_package() + config = devops_config({"tf_additional_resources_from_package": ["my.file"]}) sut = TerraformDomain(config) assert [ @@ -101,5 +121,8 @@ def test_should_calculate_resources_from_package(): "provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf", + "provider_registry.tf", + "hetzner_provider.tf", + "hetzner_mixin_vars.tf", "my.file", ] == sut.resources_from_package() From 7ad5468a1175641b0c220f7cf57f0e6896f49bd4 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 26 May 2023 08:40:41 +0200 Subject: [PATCH 147/173] Implement additional resources as sets Avoids duplicates when using multiple providers --- .../ddadevops/domain/provider_digitalocean.py | 6 +++--- .../python/ddadevops/domain/provider_hetzner.py | 6 +++--- src/main/python/ddadevops/domain/terraform.py | 12 ++++++------ src/test/python/domain/test_terraform.py | 16 ++++++++-------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index aeded28..af1b4e0 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -1,4 +1,4 @@ -from typing import List, Dict +from typing import List, Dict, Set from .common import Validateable, CredentialMappingDefault @@ -18,8 +18,8 @@ class Digitalocean(Validateable, CredentialMappingDefault): result += self.__validate_is_not_empty__("do_spaces_secret_key") return result - def resources_from_package(self) -> List[str]: - return ["provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"] + def resources_from_package(self) -> Set[str]: + return {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} def project_vars(self): return { diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py index 636b007..f3f691a 100644 --- a/src/main/python/ddadevops/domain/provider_hetzner.py +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -1,4 +1,4 @@ -from typing import List, Dict +from typing import List, Dict, Set from .common import Validateable, CredentialMappingDefault class Hetzner(Validateable, CredentialMappingDefault): @@ -13,8 +13,8 @@ class Hetzner(Validateable, CredentialMappingDefault): result += self.__validate_is_not_empty__("hetzner_api_key") return result - def resources_from_package(self) -> List[str]: - return ["provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"] + def resources_from_package(self) -> Set[str]: + return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"} def project_vars(self): return { diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 42304af..a865f3b 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Set from pathlib import Path from .common import ( Validateable, @@ -18,7 +18,7 @@ class TerraformDomain(Validateable): self.tf_build_commons_path = inp.get("tf_build_commons_path") self.tf_provider_types = inp.get("tf_provider_types", []) self.tf_additional_resources_from_package = inp.get( - "tf_additional_resources_from_package", [] + "tf_additional_resources_from_package", set() ) self.tf_additional_tfvar_files = inp.get("tf_additional_tfvar_files", []) self.tf_use_workspace = inp.get("tf_use_workspace", True) @@ -70,11 +70,11 @@ class TerraformDomain(Validateable): result.update(self.tf_additional_vars) return result - def resources_from_package(self) -> List[str]: - result = ["versions.tf", "terraform_build_vars.tf"] + def resources_from_package(self) -> Set[str]: + result = {"versions.tf", "terraform_build_vars.tf"} for provider in self.providers.values(): - result += provider.resources_from_package() - result += self.tf_additional_resources_from_package + result = result.union(provider.resources_from_package()) + result = result.union(self.tf_additional_resources_from_package) return result @classmethod diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index ac6a377..23af2c7 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -83,7 +83,7 @@ def test_should_calculate_resources_from_package(): } ) sut = TerraformDomain(config) - assert ["versions.tf", "terraform_build_vars.tf"] == sut.resources_from_package() + assert {"versions.tf", "terraform_build_vars.tf"} == sut.resources_from_package() config = devops_config( { @@ -91,13 +91,13 @@ def test_should_calculate_resources_from_package(): } ) sut = TerraformDomain(config) - assert [ + assert { "versions.tf", "terraform_build_vars.tf", "provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf", - ] == sut.resources_from_package() + } == sut.resources_from_package() config = devops_config( { @@ -105,17 +105,17 @@ def test_should_calculate_resources_from_package(): } ) sut = TerraformDomain(config) - assert [ + assert { "versions.tf", "terraform_build_vars.tf", "provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf", - ] == sut.resources_from_package() + } == sut.resources_from_package() - config = devops_config({"tf_additional_resources_from_package": ["my.file"]}) + config = devops_config({"tf_additional_resources_from_package": {"my.file"}}) sut = TerraformDomain(config) - assert [ + assert { "versions.tf", "terraform_build_vars.tf", "provider_registry.tf", @@ -125,4 +125,4 @@ def test_should_calculate_resources_from_package(): "hetzner_provider.tf", "hetzner_mixin_vars.tf", "my.file", - ] == sut.resources_from_package() + } == sut.resources_from_package() From 3b14ce09c9a571e3a323c26ab534cff0313c4c17 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 26 May 2023 08:42:59 +0200 Subject: [PATCH 148/173] Update domain architecture --- doc/architecture/Domain.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index c6d6da6..7e2787e 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -55,6 +55,10 @@ classDiagram do_spaces_access_key do_spaces_secret_key } + + class Hetzner { + hetzner_api_key + } class DnsRecord { fqdn @@ -104,7 +108,8 @@ classDiagram Devops *-- "0..1" ProvsK3s: specialized_builds Devops *-- "0..1" TerraformDomain: specialized_builds Devops *-- "0..1" Release: mixins - TerraformDomain *-- "0..1" Digitalocean: provider + TerraformDomain *-- "0..1" Digitalocean: providers + TerraformDomain *-- "0..1" Hetzner: providers Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version From e3703042b0d5b52a265971492f524388a9d164ef Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 26 May 2023 08:52:57 +0200 Subject: [PATCH 149/173] Resolve linting errors --- src/main/python/ddadevops/domain/provider_hetzner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py index f3f691a..26018a9 100644 --- a/src/main/python/ddadevops/domain/provider_hetzner.py +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -12,15 +12,15 @@ class Hetzner(Validateable, CredentialMappingDefault): result = [] result += self.__validate_is_not_empty__("hetzner_api_key") return result - + def resources_from_package(self) -> Set[str]: return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"} - + def project_vars(self): return { "hetzner_api_key": self.hetzner_api_key } - + @classmethod def get_mapping_default(cls) -> List[Dict[str, str]]: - return [] \ No newline at end of file + return [] From 92e3181323ee8caf2777bb809e635dd118f8249e Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 26 May 2023 08:53:32 +0200 Subject: [PATCH 150/173] Resolve mypy error --- src/main/python/ddadevops/domain/terraform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index a865f3b..03fd5c9 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -1,4 +1,4 @@ -from typing import List, Set +from typing import List, Set, Dict, Any from pathlib import Path from .common import ( Validateable, @@ -34,7 +34,7 @@ class TerraformDomain(Validateable): self.tf_use_package_common_files = inp.get("tf_use_package_common_files", True) provider_types = TerraformDomain.parse_provider_types(self.tf_provider_types) - self.providers = {} + self.providers: Dict[ProviderType, Any] = {} if ProviderType.DIGITALOCEAN in provider_types: self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp) if ProviderType.HETZNER in provider_types: From d8e10b2fdada8a4f03c22e5ff66476041f488d58 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 13:06:04 +0200 Subject: [PATCH 151/173] improve readability --- src/main/python/ddadevops/domain/init_service.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 76a23db..702dd76 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -49,16 +49,11 @@ class InitService: default_mappings += C4k.get_mapping_default() if BuildType.IMAGE in build_types: default_mappings += Image.get_mapping_default() - if ( - BuildType.TERRAFORM in build_types - and ProviderType.DIGITALOCEAN in provider_types - ): - default_mappings += Digitalocean.get_mapping_default() - if ( - BuildType.TERRAFORM in build_types - and ProviderType.HETZNER in provider_types - ): - default_mappings += Hetzner.get_mapping_default() + if BuildType.TERRAFORM in build_types: + if ProviderType.DIGITALOCEAN in provider_types: + default_mappings += Digitalocean.get_mapping_default() + if ProviderType.HETZNER in provider_types: + default_mappings += Hetzner.get_mapping_default() if MixinType.RELEASE in mixin_types: primary_build_file_id = inp.get( From 7476c9a2d37a2b87a7715ac865a9685fdfaefd43 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 13:06:14 +0200 Subject: [PATCH 152/173] no longer used --- src/main/resources/terraform/exoscale_mixin_vars.tf | 5 ----- src/main/resources/terraform/exoscale_provider.tf | 4 ---- 2 files changed, 9 deletions(-) delete mode 100644 src/main/resources/terraform/exoscale_mixin_vars.tf delete mode 100644 src/main/resources/terraform/exoscale_provider.tf diff --git a/src/main/resources/terraform/exoscale_mixin_vars.tf b/src/main/resources/terraform/exoscale_mixin_vars.tf deleted file mode 100644 index 013f2b9..0000000 --- a/src/main/resources/terraform/exoscale_mixin_vars.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "exoscale_api_key" { -} - -variable "exoscale_secret_key" { -} \ No newline at end of file diff --git a/src/main/resources/terraform/exoscale_provider.tf b/src/main/resources/terraform/exoscale_provider.tf deleted file mode 100644 index 2740719..0000000 --- a/src/main/resources/terraform/exoscale_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "exoscale" { - key = "${var.exoscale_api_key}" - secret = "${var.exoscale_secret_key}" -} \ No newline at end of file From 8f61e286ae754ec89602851c7d11015f11ece77e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 14:39:13 +0200 Subject: [PATCH 153/173] add resource handling for do_backend --- doc/architecture/Domain.md | 6 +++ .../digitalocean_backend_properties_mixin.py | 37 ++------------ .../ddadevops/domain/provider_digitalocean.py | 38 +++++++++++++- src/test/python/domain/helper.py | 7 +++ .../domain/test_provider_digitalocean.py | 44 +++++++++++++--- src/test/python/domain/test_terraform.py | 50 +++++++++++++++---- 6 files changed, 129 insertions(+), 53 deletions(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 7e2787e..d557f6c 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -54,6 +54,12 @@ classDiagram do_api_key do_spaces_access_key do_spaces_secret_key + do_as_backend + do_account_name + do_endpoint + do_bucket + do_bucket_key + do_region } class Hetzner { diff --git a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py index 49ac57b..2cd33b9 100644 --- a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py +++ b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py @@ -5,37 +5,11 @@ from .digitalocean_terraform_build import DigitaloceanTerraformBuild def add_digitalocean_backend_properties_mixin_config( config, account_name, endpoint, bucket, key, region="eu-central-1" ): - config.update( - { - "DigitaloceanBackendPropertiesMixin": { - "account_name": account_name, - "endpoint": endpoint, - "bucket": bucket, - "key": key, - "region": region, - } - } - ) - return config + pass -class DigitaloceanBackendPropertiesMixin(DigitaloceanTerraformBuild): - def __init__(self, project, config): - super().__init__(project, config) - do_mixin_config = config["DigitaloceanBackendPropertiesMixin"] - self.account_name = do_mixin_config["account_name"] - self.endpoint = do_mixin_config["endpoint"] - self.bucket = do_mixin_config["bucket"] - self.key = do_mixin_config["account_name"] + "/" + do_mixin_config["key"] - self.region = do_mixin_config["region"] - self.backend_config = { - "access_key": self.do_spaces_access_id, - "secret_key": self.do_spaces_secret_key, - "endpoint": self.endpoint, - "bucket": self.bucket, - "key": self.key, - "region": self.region, - } +class DigitaloceanBackendPropertiesMixin(): + pass def project_vars(self): ret = super().project_vars() @@ -46,11 +20,6 @@ class DigitaloceanBackendPropertiesMixin(DigitaloceanTerraformBuild): ret.update({"region": self.region}) return ret - def copy_build_resources_from_package(self): - super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package("do_backend_properties_vars.tf") - self.copy_build_resource_file_from_package("do_backend_with_properties.tf") - def copy_local_state(self): pass diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index af1b4e0..e71d9af 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -7,19 +7,38 @@ class Digitalocean(Validateable, CredentialMappingDefault): self, inp: dict, ): + self.stage = inp.get("stage") + self.module = inp.get("module") self.do_api_key = inp.get("do_api_key") self.do_spaces_access_id = inp.get("do_spaces_access_id") self.do_spaces_secret_key = inp.get("do_spaces_secret_key") + self.do_as_backend = inp.get("do_as_backend", False) + self.do_account_name = inp.get("do_account_name") + self.do_endpoint = inp.get("do_endpoint") + self.do_bucket = inp.get("do_bucket") + self.do_bucket_key = inp.get("do_bucket_key") + self.do_region = inp.get("do_region") def validate(self) -> List[str]: result = [] + result += self.__validate_is_not_empty__("stage") + result += self.__validate_is_not_empty__("module") result += self.__validate_is_not_empty__("do_api_key") result += self.__validate_is_not_empty__("do_spaces_access_id") result += self.__validate_is_not_empty__("do_spaces_secret_key") + result += self.__validate_is_not_empty__("do_spaces_secret_key") + result += self.__validate_is_not_none__("do_as_backend") + if self.do_as_backend: + result += self.__validate_is_not_empty__("do_endpoint") + result += self.__validate_is_not_empty__("do_bucket") + result += self.__validate_is_not_empty__("do_region") return result def resources_from_package(self) -> Set[str]: - return {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} + result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} + if self.do_as_backend: + result.update({"do_backend_properties_vars.tf", "do_backend_with_properties.tf"}) + return result def project_vars(self): return { @@ -28,6 +47,23 @@ class Digitalocean(Validateable, CredentialMappingDefault): "do_spaces_secret_key": self.do_spaces_secret_key, } + def backend_config(self) -> map: + result = {} + if self.do_as_backend: + if self.do_account_name and self.do_bucket_key: + bucket_key = f"{self.do_account_name}/{self.do_bucket_key}" + else: + bucket_key = f"{self.stage}/{self.module}" + result = { + "access_key": self.do_spaces_access_id, + "secret_key": self.do_spaces_secret_key, + "endpoint": self.do_endpoint, + "bucket": self.do_bucket, + "key": bucket_key, + "region": self.do_region, + } + return result + @classmethod def get_mapping_default(cls) -> List[Dict[str, str]]: return [ diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index a72a0ff..df1a4a1 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -37,6 +37,13 @@ def devops_config(overrides: dict) -> dict: "do_api_key": "api_key", "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + "do_as_backend": True, + "do_endpoint": "endpoint", + "do_bucket": "bucket", + "do_region": "region", "hetzner_api_key": "hetzner_api_key", "release_type": "NONE", "release_main_branch": "main", diff --git a/src/test/python/domain/test_provider_digitalocean.py b/src/test/python/domain/test_provider_digitalocean.py index d24f45f..94fe869 100644 --- a/src/test/python/domain/test_provider_digitalocean.py +++ b/src/test/python/domain/test_provider_digitalocean.py @@ -9,13 +9,43 @@ from .helper import devops_config def test_digitalocean_creation(): sut = Digitalocean( - devops_config( - { - "do_api_key": "api_key", - "do_spaces_access_id": "spaces_id", - "do_spaces_secret_key": "spaces_secret", - } - ) + { + "module": "module", + "stage": "test", + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + } ) assert sut is not None assert sut.is_valid() + + sut = Digitalocean( + { + "module": "module", + "stage": "test", + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + "do_as_backend": True, + "do_account_name": "account_name", + "do_endpoint": "endpoint", + "do_bucket": "bucket", + "do_bucket_key": "bucket_key", + "do_region": "region", + } + ) + assert sut is not None + assert sut.is_valid() + + +def test_should_calculate_backend_config(): + sut = Digitalocean(devops_config({})) + assert { + "access_key": "spaces_id", + "secret_key": "spaces_secret", + "endpoint": "endpoint", + "bucket": "bucket", + "key": "test/module", + "region": "region", + } == sut.backend_config() diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 23af2c7..b7dd38e 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -72,7 +72,7 @@ def test_should_calculate_project_vars(): "do_api_key": "api_key", "do_spaces_access_id": "spaces_id", "do_spaces_secret_key": "spaces_secret", - "hetzner_api_key": "hetzner_api_key" + "hetzner_api_key": "hetzner_api_key", } == sut.project_vars() @@ -87,7 +87,8 @@ def test_should_calculate_resources_from_package(): config = devops_config( { - "tf_provider_types": ["DIGITALOCEAN"] + "tf_provider_types": ["DIGITALOCEAN"], + "do_as_backend": False, } ) sut = TerraformDomain(config) @@ -97,13 +98,23 @@ def test_should_calculate_resources_from_package(): "provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf", - } == sut.resources_from_package() + } == sut.resources_from_package() - config = devops_config( - { - "tf_provider_types": ["HETZNER"] - } - ) + sut = TerraformDomain(devops_config({ + "tf_provider_types": ["DIGITALOCEAN"], + "do_as_backend": True, + })) + assert { + "versions.tf", + "terraform_build_vars.tf", + "provider_registry.tf", + "do_provider.tf", + "do_mixin_vars.tf", + "do_backend_properties_vars.tf", + "do_backend_with_properties.tf", + } == sut.resources_from_package() + + config = devops_config({"tf_provider_types": ["HETZNER"]}) sut = TerraformDomain(config) assert { "versions.tf", @@ -111,9 +122,14 @@ def test_should_calculate_resources_from_package(): "provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf", - } == sut.resources_from_package() + } == sut.resources_from_package() - config = devops_config({"tf_additional_resources_from_package": {"my.file"}}) + config = devops_config( + { + "tf_additional_resources_from_package": {"my.file"}, + "do_as_backend": False, + } + ) sut = TerraformDomain(config) assert { "versions.tf", @@ -125,4 +141,16 @@ def test_should_calculate_resources_from_package(): "hetzner_provider.tf", "hetzner_mixin_vars.tf", "my.file", - } == sut.resources_from_package() + } == sut.resources_from_package() + +def test_should_calculate_local_state_handling(): + sut = TerraformDomain(devops_config({ + "tf_provider_types": [], + })) + assert sut.is_local_state() + + sut = TerraformDomain(devops_config({ + "tf_provider_types": ["DIGITALOCEAN"], + "do_as_backend": True, + })) + assert sut.is_local_state() From f399d1e63729c22ab580fa7d78a4ba74d0fa14a8 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 14:47:22 +0200 Subject: [PATCH 154/173] handle the case of non local state --- .../python/ddadevops/application/terraform_service.py | 8 ++++++-- .../ddadevops/digitalocean_backend_properties_mixin.py | 6 ------ src/main/python/ddadevops/domain/provider_digitalocean.py | 3 +++ src/main/python/ddadevops/domain/provider_hetzner.py | 3 +++ src/main/python/ddadevops/domain/terraform.py | 6 ++++++ src/test/python/domain/test_terraform.py | 2 +- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index 9dfc483..e500735 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -50,11 +50,15 @@ class TerraformService: # TODO: internal? def copy_local_state(self, devops: Devops): - self.file_api.cp("terraform.tfstate", devops.build_path(), check=False) + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if terraform_domain.is_local_state(): + self.file_api.cp("terraform.tfstate", devops.build_path(), check=False) # TODO: internal? def rescue_local_state(self, devops: Devops): - self.file_api.cp(f"{devops.build_path()}/terraform.tfstate", ".", check=False) + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if terraform_domain.is_local_state(): + self.file_api.cp(f"{devops.build_path()}/terraform.tfstate", ".", check=False) def initialize_build_dir(self, devops: Devops): terraform = devops.specialized_builds[BuildType.TERRAFORM] diff --git a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py index 2cd33b9..de8128f 100644 --- a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py +++ b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py @@ -20,12 +20,6 @@ class DigitaloceanBackendPropertiesMixin(): ret.update({"region": self.region}) return ret - def copy_local_state(self): - pass - - def rescue_local_state(self): - pass - def init_client(self): terraform = Terraform( working_dir=self.build_path(), diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index e71d9af..6f3f633 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -63,6 +63,9 @@ class Digitalocean(Validateable, CredentialMappingDefault): "region": self.do_region, } return result + + def is_local_state(self): + return not self.do_as_backend @classmethod def get_mapping_default(cls) -> List[Dict[str, str]]: diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py index 26018a9..078117e 100644 --- a/src/main/python/ddadevops/domain/provider_hetzner.py +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -20,6 +20,9 @@ class Hetzner(Validateable, CredentialMappingDefault): return { "hetzner_api_key": self.hetzner_api_key } + + def is_local_state(self): + return True @classmethod def get_mapping_default(cls) -> List[Dict[str, str]]: diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 03fd5c9..f373a9e 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -77,6 +77,12 @@ class TerraformDomain(Validateable): result = result.union(self.tf_additional_resources_from_package) return result + def is_local_state(self): + result = True + for provider in self.providers.values(): + result = result and provider.is_local_state() + return result + @classmethod def parse_provider_types( cls, tf_provider_types: List[str] diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index b7dd38e..54b4655 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -153,4 +153,4 @@ def test_should_calculate_local_state_handling(): "tf_provider_types": ["DIGITALOCEAN"], "do_as_backend": True, })) - assert sut.is_local_state() + assert not sut.is_local_state() From 4a2fa628f1d0d877c13b74d8f5610ead00b30c0b Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 14:53:59 +0200 Subject: [PATCH 155/173] handle backends in init_client --- .../application/terraform_service.py | 15 +++++--- .../digitalocean_backend_properties_mixin.py | 16 --------- .../ddadevops/domain/provider_digitalocean.py | 6 ++-- src/main/python/ddadevops/domain/terraform.py | 10 ++++-- src/test/python/domain/test_terraform.py | 35 +++++++++++++------ 5 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index e500735..5dc3c74 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -2,7 +2,7 @@ from pathlib import Path from dda_python_terraform import Terraform, IsFlagged from packaging import version -from ..domain import Devops, BuildType +from ..domain import Devops, TerraformDomain, BuildType from ..infrastructure import FileApi, ResourceApi, TerraformApi @@ -58,7 +58,9 @@ class TerraformService: def rescue_local_state(self, devops: Devops): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] if terraform_domain.is_local_state(): - self.file_api.cp(f"{devops.build_path()}/terraform.tfstate", ".", check=False) + self.file_api.cp( + f"{devops.build_path()}/terraform.tfstate", ".", check=False + ) def initialize_build_dir(self, devops: Devops): terraform = devops.specialized_builds[BuildType.TERRAFORM] @@ -74,12 +76,17 @@ class TerraformService: # TODO: internal? def init_client(self, devops: Devops): - terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + terraform_domain: TerraformDomain = devops.specialized_builds[ + BuildType.TERRAFORM + ] terraform = Terraform( working_dir=devops.build_path(), terraform_semantic_version=terraform_domain.tf_terraform_semantic_version, ) - terraform.init() + if terraform_domain.is_local_state(): + terraform.init() + else: + terraform.init(backend_config=terraform_domain.backend_config()) self.__print_terraform_command__(terraform, devops) if terraform_domain.tf_use_workspace: try: diff --git a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py index de8128f..61a99a9 100644 --- a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py +++ b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py @@ -19,19 +19,3 @@ class DigitaloceanBackendPropertiesMixin(): ret.update({"key": self.key}) ret.update({"region": self.region}) return ret - - def init_client(self): - terraform = Terraform( - working_dir=self.build_path(), - terraform_semantic_version=self.terraform_semantic_version, - ) - terraform.init(backend_config=self.backend_config) - self.print_terraform_command(terraform) - if self.use_workspace: - try: - terraform.workspace("select", self.stage) - self.print_terraform_command(terraform) - except: - terraform.workspace("new", self.stage) - self.print_terraform_command(terraform) - return terraform diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index 6f3f633..e8cf494 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -37,7 +37,9 @@ class Digitalocean(Validateable, CredentialMappingDefault): def resources_from_package(self) -> Set[str]: result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} if self.do_as_backend: - result.update({"do_backend_properties_vars.tf", "do_backend_with_properties.tf"}) + result.update( + {"do_backend_properties_vars.tf", "do_backend_with_properties.tf"} + ) return result def project_vars(self): @@ -63,7 +65,7 @@ class Digitalocean(Validateable, CredentialMappingDefault): "region": self.do_region, } return result - + def is_local_state(self): return not self.do_as_backend diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index f373a9e..6c4b8ea 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -83,10 +83,14 @@ class TerraformDomain(Validateable): result = result and provider.is_local_state() return result + def backend_config(self) -> map: + result = {} + for provider in self.providers.values(): + result.update(provider.backend_config()) + return result + @classmethod - def parse_provider_types( - cls, tf_provider_types: List[str] - ) -> List[ProviderType]: + def parse_provider_types(cls, tf_provider_types: List[str]) -> List[ProviderType]: result = [] for provider_type in tf_provider_types: result.append(ProviderType[provider_type]) diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 54b4655..49fd176 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -100,10 +100,14 @@ def test_should_calculate_resources_from_package(): "do_mixin_vars.tf", } == sut.resources_from_package() - sut = TerraformDomain(devops_config({ - "tf_provider_types": ["DIGITALOCEAN"], - "do_as_backend": True, - })) + sut = TerraformDomain( + devops_config( + { + "tf_provider_types": ["DIGITALOCEAN"], + "do_as_backend": True, + } + ) + ) assert { "versions.tf", "terraform_build_vars.tf", @@ -143,14 +147,23 @@ def test_should_calculate_resources_from_package(): "my.file", } == sut.resources_from_package() + def test_should_calculate_local_state_handling(): - sut = TerraformDomain(devops_config({ - "tf_provider_types": [], - })) + sut = TerraformDomain( + devops_config( + { + "tf_provider_types": [], + } + ) + ) assert sut.is_local_state() - sut = TerraformDomain(devops_config({ - "tf_provider_types": ["DIGITALOCEAN"], - "do_as_backend": True, - })) + sut = TerraformDomain( + devops_config( + { + "tf_provider_types": ["DIGITALOCEAN"], + "do_as_backend": True, + } + ) + ) assert not sut.is_local_state() From 39d732a04e71e23aadab4db7a62334b15f18101d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 15:14:12 +0200 Subject: [PATCH 156/173] handle backends in project_vars --- .../digitalocean_backend_properties_mixin.py | 9 ----- .../ddadevops/domain/provider_digitalocean.py | 31 +++++++++++++---- .../domain/test_provider_digitalocean.py | 33 +++++++++++++++++++ src/test/python/domain/test_terraform.py | 2 +- 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py index 61a99a9..62e55ff 100644 --- a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py +++ b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py @@ -10,12 +10,3 @@ def add_digitalocean_backend_properties_mixin_config( class DigitaloceanBackendPropertiesMixin(): pass - - def project_vars(self): - ret = super().project_vars() - ret.update({"account_name": self.account_name}) - ret.update({"endpoint": self.endpoint}) - ret.update({"bucket": self.bucket}) - ret.update({"key": self.key}) - ret.update({"region": self.region}) - return ret diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index e8cf494..074e625 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -13,7 +13,7 @@ class Digitalocean(Validateable, CredentialMappingDefault): self.do_spaces_access_id = inp.get("do_spaces_access_id") self.do_spaces_secret_key = inp.get("do_spaces_secret_key") self.do_as_backend = inp.get("do_as_backend", False) - self.do_account_name = inp.get("do_account_name") + self.do_account_name = inp.get("do_account_name", self.stage) self.do_endpoint = inp.get("do_endpoint") self.do_bucket = inp.get("do_bucket") self.do_bucket_key = inp.get("do_bucket_key") @@ -29,6 +29,7 @@ class Digitalocean(Validateable, CredentialMappingDefault): result += self.__validate_is_not_empty__("do_spaces_secret_key") result += self.__validate_is_not_none__("do_as_backend") if self.do_as_backend: + result += self.__validate_is_not_empty__("do_account_name") result += self.__validate_is_not_empty__("do_endpoint") result += self.__validate_is_not_empty__("do_bucket") result += self.__validate_is_not_empty__("do_region") @@ -43,25 +44,32 @@ class Digitalocean(Validateable, CredentialMappingDefault): return result def project_vars(self): - return { + result = { "do_api_key": self.do_api_key, "do_spaces_access_id": self.do_spaces_access_id, "do_spaces_secret_key": self.do_spaces_secret_key, } + if self.do_as_backend: + result.update( + { + "account_name": self.do_account_name, + "endpoint": self.do_endpoint, + "bucket": self.do_bucket, + "key": self.__bucket_key__(), + "region": self.do_region, + } + ) + return result def backend_config(self) -> map: result = {} if self.do_as_backend: - if self.do_account_name and self.do_bucket_key: - bucket_key = f"{self.do_account_name}/{self.do_bucket_key}" - else: - bucket_key = f"{self.stage}/{self.module}" result = { "access_key": self.do_spaces_access_id, "secret_key": self.do_spaces_secret_key, "endpoint": self.do_endpoint, "bucket": self.do_bucket, - "key": bucket_key, + "key": self.__bucket_key__(), "region": self.do_region, } return result @@ -69,6 +77,15 @@ class Digitalocean(Validateable, CredentialMappingDefault): def is_local_state(self): return not self.do_as_backend + def __bucket_key__(self): + result = "" + if self.do_as_backend: + if self.do_account_name and self.do_bucket_key: + result = f"{self.do_account_name}/{self.do_bucket_key}" + else: + result = f"{self.stage}/{self.module}" + return result + @classmethod def get_mapping_default(cls) -> List[Dict[str, str]]: return [ diff --git a/src/test/python/domain/test_provider_digitalocean.py b/src/test/python/domain/test_provider_digitalocean.py index 94fe869..c4d8782 100644 --- a/src/test/python/domain/test_provider_digitalocean.py +++ b/src/test/python/domain/test_provider_digitalocean.py @@ -49,3 +49,36 @@ def test_should_calculate_backend_config(): "key": "test/module", "region": "region", } == sut.backend_config() + + +def test_should_calculate_project_vars(): + sut = Digitalocean( + devops_config( + { + "do_as_backend": False, + } + ) + ) + assert { + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + } == sut.project_vars() + + sut = Digitalocean( + devops_config( + { + "do_as_backend": True, + } + ) + ) + assert { + "do_api_key": "api_key", + "do_spaces_access_id": "spaces_id", + "do_spaces_secret_key": "spaces_secret", + "account_name": "test", + "endpoint": "endpoint", + "bucket": "bucket", + "key": "test/module", + "region": "region", + } == sut.project_vars() diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 49fd176..5633cd1 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -64,7 +64,7 @@ def test_should_calculate_project_vars(): sut = TerraformDomain(config) assert {"module": "module", "stage": "test"} == sut.project_vars() - config = devops_config({}) + config = devops_config({"do_as_backend": False,}) sut = TerraformDomain(config) assert { "module": "module", From fa4b58b88c6a90b2df9d35df36042206bc58efb3 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 15:15:51 +0200 Subject: [PATCH 157/173] remove unused --- src/main/python/ddadevops/__init__.py | 1 - .../digitalocean_backend_properties_mixin.py | 12 ------------ .../python/ddadevops/digitalocean_terraform_build.py | 11 ----------- 3 files changed, 24 deletions(-) delete mode 100644 src/main/python/ddadevops/digitalocean_backend_properties_mixin.py delete mode 100644 src/main/python/ddadevops/digitalocean_terraform_build.py diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index f66936f..f348694 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -9,7 +9,6 @@ from .provs_k3s_build import ProvsK3sBuild from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config from .c4k_build import C4kBuild -from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config from .devops_image_build import DevopsImageBuild from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config diff --git a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py b/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py deleted file mode 100644 index 62e55ff..0000000 --- a/src/main/python/ddadevops/digitalocean_backend_properties_mixin.py +++ /dev/null @@ -1,12 +0,0 @@ -from dda_python_terraform import Terraform -from .digitalocean_terraform_build import DigitaloceanTerraformBuild - - -def add_digitalocean_backend_properties_mixin_config( - config, account_name, endpoint, bucket, key, region="eu-central-1" -): - pass - - -class DigitaloceanBackendPropertiesMixin(): - pass diff --git a/src/main/python/ddadevops/digitalocean_terraform_build.py b/src/main/python/ddadevops/digitalocean_terraform_build.py deleted file mode 100644 index 8a285d3..0000000 --- a/src/main/python/ddadevops/digitalocean_terraform_build.py +++ /dev/null @@ -1,11 +0,0 @@ -from .devops_terraform_build import ( - DevopsTerraformBuild, -) - - -def create_digitalocean_terraform_build_config(): - pass - - -class DigitaloceanTerraformBuild(DevopsTerraformBuild): - pass From 406a5e6333d853c05dc5f53900c8d65a2c56f9ef Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 15:30:47 +0200 Subject: [PATCH 158/173] minor fixes --- build.py | 2 +- src/main/python/ddadevops/domain/provider_digitalocean.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 8b23190..ebeaf85 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev57" +version = "4.0.0-dev59" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index 074e625..bbfefa4 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -14,10 +14,10 @@ class Digitalocean(Validateable, CredentialMappingDefault): self.do_spaces_secret_key = inp.get("do_spaces_secret_key") self.do_as_backend = inp.get("do_as_backend", False) self.do_account_name = inp.get("do_account_name", self.stage) - self.do_endpoint = inp.get("do_endpoint") self.do_bucket = inp.get("do_bucket") self.do_bucket_key = inp.get("do_bucket_key") - self.do_region = inp.get("do_region") + self.do_endpoint = inp.get("do_endpoint", "fra1.digitaloceanspaces.com") + self.do_region = inp.get("do_region", "eu-central-1") def validate(self) -> List[str]: result = [] From da872af76e28e1b5832835f3be7f97483ae549fb Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 15:38:12 +0200 Subject: [PATCH 159/173] fix linting --- .../ddadevops/application/terraform_service.py | 4 ++-- src/main/python/ddadevops/domain/init_service.py | 14 ++++++++++---- .../ddadevops/domain/provider_digitalocean.py | 4 ++-- .../python/ddadevops/domain/provider_hetzner.py | 7 +++---- src/main/python/ddadevops/domain/terraform.py | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index 5dc3c74..815a62f 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -2,7 +2,7 @@ from pathlib import Path from dda_python_terraform import Terraform, IsFlagged from packaging import version -from ..domain import Devops, TerraformDomain, BuildType +from ..domain import Devops, BuildType from ..infrastructure import FileApi, ResourceApi, TerraformApi @@ -76,7 +76,7 @@ class TerraformService: # TODO: internal? def init_client(self, devops: Devops): - terraform_domain: TerraformDomain = devops.specialized_builds[ + terraform_domain = devops.specialized_builds[ BuildType.TERRAFORM ] terraform = Terraform( diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 702dd76..1261b8e 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -38,9 +38,15 @@ class InitService: ) def initialize(self, inp: dict) -> Devops: - build_types = self.devops_factory.__parse_build_types__(inp.get("build_types", [])) - mixin_types = self.devops_factory.__parse_mixin_types__(inp.get("mixin_types", [])) - provider_types = TerraformDomain.parse_provider_types(inp.get("tf_provider_types", [])) + build_types = self.devops_factory.__parse_build_types__( + inp.get("build_types", []) + ) + mixin_types = self.devops_factory.__parse_mixin_types__( + inp.get("mixin_types", []) + ) + provider_types = TerraformDomain.parse_provider_types( + inp.get("tf_provider_types", []) + ) version = None default_mappings = [] @@ -50,7 +56,7 @@ class InitService: if BuildType.IMAGE in build_types: default_mappings += Image.get_mapping_default() if BuildType.TERRAFORM in build_types: - if ProviderType.DIGITALOCEAN in provider_types: + if ProviderType.DIGITALOCEAN in provider_types: default_mappings += Digitalocean.get_mapping_default() if ProviderType.HETZNER in provider_types: default_mappings += Hetzner.get_mapping_default() diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index bbfefa4..491d93b 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Set +from typing import List, Dict, Set, Any from .common import Validateable, CredentialMappingDefault @@ -61,7 +61,7 @@ class Digitalocean(Validateable, CredentialMappingDefault): ) return result - def backend_config(self) -> map: + def backend_config(self) -> Dict[str, Any]: result = {} if self.do_as_backend: result = { diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py index 078117e..bd532ab 100644 --- a/src/main/python/ddadevops/domain/provider_hetzner.py +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -1,6 +1,7 @@ from typing import List, Dict, Set from .common import Validateable, CredentialMappingDefault + class Hetzner(Validateable, CredentialMappingDefault): def __init__( self, @@ -17,10 +18,8 @@ class Hetzner(Validateable, CredentialMappingDefault): return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"} def project_vars(self): - return { - "hetzner_api_key": self.hetzner_api_key - } - + return {"hetzner_api_key": self.hetzner_api_key} + def is_local_state(self): return True diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 6c4b8ea..8df506f 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -83,7 +83,7 @@ class TerraformDomain(Validateable): result = result and provider.is_local_state() return result - def backend_config(self) -> map: + def backend_config(self) -> Dict[str, Any]: result = {} for provider in self.providers.values(): result.update(provider.backend_config()) From 0b577597e8e47f92b6a3a9050c5d69cbd13b08c7 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 16:56:02 +0200 Subject: [PATCH 160/173] introduce aws_provider --- build.py | 2 +- doc/architecture/Domain.md | 5 ++ src/main/python/ddadevops/__init__.py | 4 +- .../ddadevops/aws_backend_properties_mixin.py | 52 ------------ src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/common.py | 1 + .../python/ddadevops/domain/provider_aws.py | 80 +++++++++++++++++++ .../ddadevops/domain/provider_digitalocean.py | 26 +++--- src/main/python/ddadevops/domain/terraform.py | 3 + .../terraform/aws_backend_properties_vars.tf | 4 +- .../terraform/do_backend_properties_vars.tf | 4 +- src/test/python/domain/helper.py | 6 +- src/test/python/domain/test_provider_aws.py | 77 ++++++++++++++++++ src/test/python/domain/test_terraform.py | 19 ++++- 14 files changed, 209 insertions(+), 75 deletions(-) delete mode 100644 src/main/python/ddadevops/aws_backend_properties_mixin.py create mode 100644 src/main/python/ddadevops/domain/provider_aws.py create mode 100644 src/test/python/domain/test_provider_aws.py diff --git a/build.py b/build.py index ebeaf85..3de1f42 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev59" +version = "4.0.0-dev61" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index d557f6c..778f1dd 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -65,6 +65,10 @@ classDiagram class Hetzner { hetzner_api_key } + + class Aws { + aws_account_name + } class DnsRecord { fqdn @@ -116,6 +120,7 @@ classDiagram Devops *-- "0..1" Release: mixins TerraformDomain *-- "0..1" Digitalocean: providers TerraformDomain *-- "0..1" Hetzner: providers + TerraformDomain *-- "0..1" Aws: providers Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files BuildFile *-- "1" Version diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index f348694..c58913a 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -4,10 +4,8 @@ terraform, dda-pallet, aws & hetzner-cloud. """ -from .python_util import execute from .provs_k3s_build import ProvsK3sBuild -from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config -from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config +#from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .c4k_build import C4kBuild from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config from .devops_image_build import DevopsImageBuild diff --git a/src/main/python/ddadevops/aws_backend_properties_mixin.py b/src/main/python/ddadevops/aws_backend_properties_mixin.py deleted file mode 100644 index ef6cebe..0000000 --- a/src/main/python/ddadevops/aws_backend_properties_mixin.py +++ /dev/null @@ -1,52 +0,0 @@ -from dda_python_terraform import Terraform -from .devops_terraform_build import DevopsTerraformBuild - - -def add_aws_backend_properties_mixin_config(config, account_name): - config.update({"AwsBackendPropertiesMixin": {"account_name": account_name}}) - return config - - -class AwsBackendPropertiesMixin(DevopsTerraformBuild): - def __init__(self, project, config): - super().__init__(project, config) - aws_mixin_config = config["AwsBackendPropertiesMixin"] - self.account_name = aws_mixin_config["account_name"] - self.backend_config = ( - "backend." + self.account_name + "." + self.stage + ".properties" - ) - self.additional_tfvar_files.append(self.backend_config) - - def project_vars(self): - ret = super().project_vars() - ret.update({"account_name": self.account_name}) - return ret - - def copy_build_resources_from_package(self): - super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package("provider_registry.tf") - self.copy_build_resource_file_from_package("aws_provider.tf") - self.copy_build_resource_file_from_package("aws_backend_properties_vars.tf") - self.copy_build_resource_file_from_package("aws_backend_with_properties.tf") - - def copy_local_state(self): - pass - - def rescue_local_state(self): - pass - - def init_client(self): - terraform = Terraform( - working_dir=self.build_path(), - terraform_semantic_version=self.terraform_semantic_version, - ) - terraform.init(backend_config=self.backend_config) - self.print_terraform_command(terraform) - if self.use_workspace: - try: - terraform.workspace("select", self.stage) - self.print_terraform_command(terraform) - except: - terraform.workspace("new", self.stage) - self.print_terraform_command(terraform) - return terraform diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index b281a61..fdb1784 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -5,6 +5,7 @@ from .c4k import C4k from .terraform import TerraformDomain from .provider_digitalocean import Digitalocean from .provider_hetzner import Hetzner +from .provider_aws import Aws from .provs_k3s import K3s from .release import Release from .credentials import Credentials, CredentialMapping, GopassType diff --git a/src/main/python/ddadevops/domain/common.py b/src/main/python/ddadevops/domain/common.py index 01dc29d..fc32311 100644 --- a/src/main/python/ddadevops/domain/common.py +++ b/src/main/python/ddadevops/domain/common.py @@ -16,6 +16,7 @@ class BuildType(Enum): class ProviderType(Enum): DIGITALOCEAN = 0 HETZNER = 1 + AWS = 2 class MixinType(Enum): diff --git a/src/main/python/ddadevops/domain/provider_aws.py b/src/main/python/ddadevops/domain/provider_aws.py new file mode 100644 index 0000000..e0ef793 --- /dev/null +++ b/src/main/python/ddadevops/domain/provider_aws.py @@ -0,0 +1,80 @@ +from typing import List, Dict, Set, Any +from .common import Validateable, CredentialMappingDefault + + +class Aws(Validateable, CredentialMappingDefault): + def __init__( + self, + inp: dict, + ): + self.stage = inp.get("stage") + self.module = inp.get("module") + self.aws_bucket = inp.get("aws_bucket") + self.aws_bucket_kms_key_id = inp.get("aws_bucket_kms_key_id") + self.aws_account_name = inp.get("aws_account_name", self.stage) + self.aws_bucket_key = inp.get("aws_bucket_key", self.module) + self.aws_as_backend = inp.get("aws_as_backend", False) + self.aws_region = inp.get("aws_region", "eu-central-1") + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("stage") + result += self.__validate_is_not_empty__("module") + result += self.__validate_is_not_empty__("aws_account_name") + result += self.__validate_is_not_empty__("aws_as_backend") + if self.aws_as_backend: + result += self.__validate_is_not_empty__("aws_bucket") + result += self.__validate_is_not_empty__("aws_bucket_key") + result += self.__validate_is_not_empty__("aws_bucket_kms_key_id") + result += self.__validate_is_not_empty__("aws_region") + return result + + def backend_config(self) -> Dict[str, Any]: + result = {} + if self.aws_as_backend: + result = { + "bucket": self.aws_bucket, + "key": self.__bucket_key__(), + "region": self.aws_region, + } + if self.aws_bucket_kms_key_id: + result["kms_key_id"] = self.aws_bucket_kms_key_id + return result + + def resources_from_package(self) -> Set[str]: + result = {"provider_registry.tf", "aws_provider.tf"} + if self.aws_as_backend: + result.update( + {"aws_backend_properties_vars.tf", "aws_backend_with_properties.tf"} + ) + return result + + def project_vars(self): + result = {} + if self.aws_as_backend: + result.update( + { + "account_name": self.aws_account_name, + "bucket": self.aws_bucket, + "key": self.__bucket_key__(), + "kms_key_id": self.aws_bucket_kms_key_id, + "region": self.aws_region, + } + ) + return result + + def is_local_state(self): + return not self.aws_as_backend + + def __bucket_key__(self): + result = "" + if self.aws_as_backend: + if self.aws_account_name and self.aws_bucket_key: + result = f"{self.aws_account_name}/{self.aws_bucket_key}" + else: + result = f"{self.stage}/{self.module}" + return result + + @classmethod + def get_mapping_default(cls) -> List[Dict[str, str]]: + return [] diff --git a/src/main/python/ddadevops/domain/provider_digitalocean.py b/src/main/python/ddadevops/domain/provider_digitalocean.py index 491d93b..e00ce4f 100644 --- a/src/main/python/ddadevops/domain/provider_digitalocean.py +++ b/src/main/python/ddadevops/domain/provider_digitalocean.py @@ -35,6 +35,19 @@ class Digitalocean(Validateable, CredentialMappingDefault): result += self.__validate_is_not_empty__("do_region") return result + def backend_config(self) -> Dict[str, Any]: + result = {} + if self.do_as_backend: + result = { + "access_key": self.do_spaces_access_id, + "secret_key": self.do_spaces_secret_key, + "endpoint": self.do_endpoint, + "bucket": self.do_bucket, + "key": self.__bucket_key__(), + "region": self.do_region, + } + return result + def resources_from_package(self) -> Set[str]: result = {"provider_registry.tf", "do_provider.tf", "do_mixin_vars.tf"} if self.do_as_backend: @@ -61,19 +74,6 @@ class Digitalocean(Validateable, CredentialMappingDefault): ) return result - def backend_config(self) -> Dict[str, Any]: - result = {} - if self.do_as_backend: - result = { - "access_key": self.do_spaces_access_id, - "secret_key": self.do_spaces_secret_key, - "endpoint": self.do_endpoint, - "bucket": self.do_bucket, - "key": self.__bucket_key__(), - "region": self.do_region, - } - return result - def is_local_state(self): return not self.do_as_backend diff --git a/src/main/python/ddadevops/domain/terraform.py b/src/main/python/ddadevops/domain/terraform.py index 8df506f..c50f8d4 100644 --- a/src/main/python/ddadevops/domain/terraform.py +++ b/src/main/python/ddadevops/domain/terraform.py @@ -7,6 +7,7 @@ from .common import ( ) from .provider_digitalocean import Digitalocean from .provider_hetzner import Hetzner +from .provider_aws import Aws class TerraformDomain(Validateable): @@ -39,6 +40,8 @@ class TerraformDomain(Validateable): self.providers[ProviderType.DIGITALOCEAN] = Digitalocean(inp) if ProviderType.HETZNER in provider_types: self.providers[ProviderType.HETZNER] = Hetzner(inp) + if ProviderType.AWS in provider_types: + self.providers[ProviderType.AWS] = Aws(inp) def validate(self) -> List[str]: result = [] diff --git a/src/main/resources/terraform/aws_backend_properties_vars.tf b/src/main/resources/terraform/aws_backend_properties_vars.tf index d261cf7..08abc4b 100644 --- a/src/main/resources/terraform/aws_backend_properties_vars.tf +++ b/src/main/resources/terraform/aws_backend_properties_vars.tf @@ -1,5 +1,5 @@ +variable "account_name" {} variable "bucket" {} variable "key" {} variable "kms_key_id" {} -variable "region" {} -variable "account_name" {} \ No newline at end of file +variable "region" {} \ No newline at end of file diff --git a/src/main/resources/terraform/do_backend_properties_vars.tf b/src/main/resources/terraform/do_backend_properties_vars.tf index bb24162..8b722b2 100644 --- a/src/main/resources/terraform/do_backend_properties_vars.tf +++ b/src/main/resources/terraform/do_backend_properties_vars.tf @@ -1,5 +1,5 @@ +variable "account_name" {} variable "endpoint" {} variable "bucket" {} variable "key" {} -variable "region" {} -variable "account_name" {} \ No newline at end of file +variable "region" {} \ No newline at end of file diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index df1a4a1..53b48b5 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -24,7 +24,7 @@ def devops_config(overrides: dict) -> dict: "k3s_letsencrypt_endpoint": "k3s_letsencrypt_endpoint", "k3s_enable_echo": "false", "k3s_app_filename_to_provision": "k3s_app.yaml", - "tf_provider_types": ["DIGITALOCEAN", "HETZNER"], + "tf_provider_types": ["DIGITALOCEAN", "HETZNER", "AWS"], "tf_additional_vars": [], "tf_output_json_name": "the_out.json", "tf_use_workspace": None, @@ -45,6 +45,10 @@ def devops_config(overrides: dict) -> dict: "do_bucket": "bucket", "do_region": "region", "hetzner_api_key": "hetzner_api_key", + "aws_as_backend": True, + "aws_bucket": "bucket", + "aws_region": "region", + "aws_bucket_kms_key_id": "aws_bucket_kms_key_id", "release_type": "NONE", "release_main_branch": "main", "release_current_branch": "my_feature", diff --git a/src/test/python/domain/test_provider_aws.py b/src/test/python/domain/test_provider_aws.py new file mode 100644 index 0000000..7334039 --- /dev/null +++ b/src/test/python/domain/test_provider_aws.py @@ -0,0 +1,77 @@ +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain import ( + BuildType, + Aws, +) +from .helper import devops_config + + +def test_aws_creation(): + sut = Aws( + { + "module": "module", + "stage": "test", + "aws_account_name": "aws_account_name", + } + ) + assert sut is not None + assert sut.is_valid() + + sut = Aws( + { + "module": "module", + "stage": "test", + "aws_as_backend": True, + "aws_bucket": "bucket", + "aws_bucket_kms_key_id": "aws_bucket_kms_key_id", + } + ) + assert sut is not None + assert sut.is_valid() + + +def test_should_calculate_backend_config(): + sut = Aws( + devops_config( + { + "module": "dns_aws", + "stage": "prod", + "aws_bucket": "meissa-configuration", + "aws_bucket_kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration", + "aws_region": "eu-central-1", + } + ) + ) + assert { + "bucket": "meissa-configuration", + "key": "prod/dns_aws", + "kms_key_id": "arn:aws:kms:eu-central-1:907507348333:alias/meissa-configuration", + "region": "eu-central-1", + } == sut.backend_config() + + +def test_should_calculate_project_vars(): + sut = Aws( + devops_config( + { + "aws_as_backend": False, + } + ) + ) + assert {} == sut.project_vars() + + sut = Aws( + devops_config( + { + "aws_as_backend": True, + } + ) + ) + assert { + "account_name": "test", + "bucket": "bucket", + "key": "test/module", + "kms_key_id": "aws_bucket_kms_key_id", + "region": "region", + } == sut.project_vars() diff --git a/src/test/python/domain/test_terraform.py b/src/test/python/domain/test_terraform.py index 5633cd1..30ab62a 100644 --- a/src/test/python/domain/test_terraform.py +++ b/src/test/python/domain/test_terraform.py @@ -16,6 +16,7 @@ def test_creation(): assert sut assert sut.providers[ProviderType.DIGITALOCEAN] assert sut.providers[ProviderType.HETZNER] + assert sut.providers[ProviderType.AWS] def test_should_calculate_output_json_name(): @@ -42,6 +43,14 @@ def test_should_validate(): sut = TerraformDomain(config) assert not sut.is_valid() + config = devops_config( + { + "aws_account_name": "", + } + ) + sut = TerraformDomain(config) + assert not sut.is_valid() + def test_should_calculate_terraform_build_commons_path(): config = devops_config({}) @@ -64,7 +73,12 @@ def test_should_calculate_project_vars(): sut = TerraformDomain(config) assert {"module": "module", "stage": "test"} == sut.project_vars() - config = devops_config({"do_as_backend": False,}) + config = devops_config( + { + "do_as_backend": False, + "aws_as_backend": False, + } + ) sut = TerraformDomain(config) assert { "module": "module", @@ -144,6 +158,9 @@ def test_should_calculate_resources_from_package(): "provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf", + "aws_backend_with_properties.tf", + "aws_provider.tf", + "aws_backend_properties_vars.tf", "my.file", } == sut.resources_from_package() From 158d8a3b378ba234ebba2163a3ae01967588682e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 17:03:56 +0200 Subject: [PATCH 161/173] fix lint --- src/main/python/ddadevops/aws_mfa_mixin.py | 138 ++++++++++----------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/src/main/python/ddadevops/aws_mfa_mixin.py b/src/main/python/ddadevops/aws_mfa_mixin.py index 9f18a6b..b581f00 100644 --- a/src/main/python/ddadevops/aws_mfa_mixin.py +++ b/src/main/python/ddadevops/aws_mfa_mixin.py @@ -1,81 +1,81 @@ -from boto3 import Session -from .python_util import execute -from .aws_backend_properties_mixin import AwsBackendPropertiesMixin +# from boto3 import Session +# from .python_util import execute +# from .aws_backend_properties_mixin import AwsBackendPropertiesMixin -def add_aws_mfa_mixin_config(config, account_id, region, - mfa_role='developer', mfa_account_prefix='', - mfa_login_account_suffix='main'): - config.update({'AwsMfaMixin': - {'account_id': account_id, - 'region': region, - 'mfa_role': mfa_role, - 'mfa_account_prefix': mfa_account_prefix, - 'mfa_login_account_suffix': mfa_login_account_suffix}}) - return config +# def add_aws_mfa_mixin_config(config, account_id, region, +# mfa_role='developer', mfa_account_prefix='', +# mfa_login_account_suffix='main'): +# config.update({'AwsMfaMixin': +# {'account_id': account_id, +# 'region': region, +# 'mfa_role': mfa_role, +# 'mfa_account_prefix': mfa_account_prefix, +# 'mfa_login_account_suffix': mfa_login_account_suffix}}) +# return config -class AwsMfaMixin(AwsBackendPropertiesMixin): +# class AwsMfaMixin(AwsBackendPropertiesMixin): - def __init__(self, project, config): - super().__init__(project, config) - project.build_depends_on('boto3') - project.build_depends_on('mfa') - aws_mfa_mixin_config = config['AwsMfaMixin'] - self.account_id = aws_mfa_mixin_config['account_id'] - self.region = aws_mfa_mixin_config['region'] - self.mfa_role = aws_mfa_mixin_config['mfa_role'] - self.mfa_account_prefix = aws_mfa_mixin_config['mfa_account_prefix'] - self.mfa_login_account_suffix = aws_mfa_mixin_config['mfa_login_account_suffix'] +# def __init__(self, project, config): +# super().__init__(project, config) +# project.build_depends_on('boto3') +# project.build_depends_on('mfa') +# aws_mfa_mixin_config = config['AwsMfaMixin'] +# self.account_id = aws_mfa_mixin_config['account_id'] +# self.region = aws_mfa_mixin_config['region'] +# self.mfa_role = aws_mfa_mixin_config['mfa_role'] +# self.mfa_account_prefix = aws_mfa_mixin_config['mfa_account_prefix'] +# self.mfa_login_account_suffix = aws_mfa_mixin_config['mfa_login_account_suffix'] - def project_vars(self): - ret = super().project_vars() - ret.update({'account_name': self.account_name, - 'account_id': self.account_id, - 'region': self.region, - 'mfa_role': self.mfa_role, - 'mfa_account_prefix': self.mfa_account_prefix, - 'mfa_login_account_suffix': self.mfa_login_account_suffix}) - return ret +# def project_vars(self): +# ret = super().project_vars() +# ret.update({'account_name': self.account_name, +# 'account_id': self.account_id, +# 'region': self.region, +# 'mfa_role': self.mfa_role, +# 'mfa_account_prefix': self.mfa_account_prefix, +# 'mfa_login_account_suffix': self.mfa_login_account_suffix}) +# return ret - def get_username_from_account(self, p_account_name): - login_id = execute(r'cat ~/.aws/accounts | grep -A 2 "\[' + p_account_name + - r'\]" | grep username | awk -F= \'{print $2}\'', shell=True) - return login_id +# def get_username_from_account(self, p_account_name): +# login_id = execute(r'cat ~/.aws/accounts | grep -A 2 "\[' + p_account_name + +# r'\]" | grep username | awk -F= \'{print $2}\'', shell=True) +# return login_id - def get_account_id_from_account(self, p_account_name): - account_id = execute(r'cat ~/.aws/accounts | grep -A 2 "\[' + p_account_name + - r'\]" | grep account | awk -F= \'{print $2}\'', shell=True) - return account_id +# def get_account_id_from_account(self, p_account_name): +# account_id = execute(r'cat ~/.aws/accounts | grep -A 2 "\[' + p_account_name + +# r'\]" | grep account | awk -F= \'{print $2}\'', shell=True) +# return account_id - def get_mfa(self, mfa_path='aws'): - mfa_token = execute('mfa otp ' + mfa_path, shell=True) - return mfa_token +# def get_mfa(self, mfa_path='aws'): +# mfa_token = execute('mfa otp ' + mfa_path, shell=True) +# return mfa_token - def write_aws_config(self, to_profile, key, secret): - execute('aws configure --profile ' + to_profile + - ' set ' + key + ' ' + secret, shell=True) +# def write_aws_config(self, to_profile, key, secret): +# execute('aws configure --profile ' + to_profile + +# ' set ' + key + ' ' + secret, shell=True) - def get_mfa_session(self): - from_account_name = self.mfa_account_prefix + self.mfa_login_account_suffix - from_account_id = self.get_account_id_from_account(from_account_name) - to_account_name = self.mfa_account_prefix + self.account_name - to_account_id = self.get_account_id_from_account(to_account_name) - login_id = self.get_username_from_account(from_account_name) - mfa_token = self.get_mfa() - ses = Session(profile_name=from_account_name) - sts_client = ses.client('sts') - response = sts_client.assume_role( - RoleArn='arn:aws:iam::' + to_account_id + ':role/' + self.mfa_role, - RoleSessionName=to_account_id + '-' + self.account_name + '-' + self.mfa_role, - SerialNumber='arn:aws:iam::' + from_account_id + ':mfa/' + login_id, - TokenCode=mfa_token - ) +# def get_mfa_session(self): +# from_account_name = self.mfa_account_prefix + self.mfa_login_account_suffix +# from_account_id = self.get_account_id_from_account(from_account_name) +# to_account_name = self.mfa_account_prefix + self.account_name +# to_account_id = self.get_account_id_from_account(to_account_name) +# login_id = self.get_username_from_account(from_account_name) +# mfa_token = self.get_mfa() +# ses = Session(profile_name=from_account_name) +# sts_client = ses.client('sts') +# response = sts_client.assume_role( +# RoleArn='arn:aws:iam::' + to_account_id + ':role/' + self.mfa_role, +# RoleSessionName=to_account_id + '-' + self.account_name + '-' + self.mfa_role, +# SerialNumber='arn:aws:iam::' + from_account_id + ':mfa/' + login_id, +# TokenCode=mfa_token +# ) - self.write_aws_config(to_account_name, 'aws_access_key_id', - response['Credentials']['AccessKeyId']) - self.write_aws_config(to_account_name, 'aws_secret_access_key', - response['Credentials']['SecretAccessKey']) - self.write_aws_config(to_account_name, 'aws_session_token', - response['Credentials']['SessionToken']) - print('got token') +# self.write_aws_config(to_account_name, 'aws_access_key_id', +# response['Credentials']['AccessKeyId']) +# self.write_aws_config(to_account_name, 'aws_secret_access_key', +# response['Credentials']['SecretAccessKey']) +# self.write_aws_config(to_account_name, 'aws_session_token', +# response['Credentials']['SessionToken']) +# print('got token') From cf605d419857a8a4ca029ac52dd08492b23c3745 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 26 May 2023 17:12:38 +0200 Subject: [PATCH 162/173] minor fixes for hetzner --- build.py | 2 +- src/main/python/ddadevops/domain/provider_hetzner.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 3de1f42..1500532 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev61" +version = "4.0.0-dev63" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py index bd532ab..a23d602 100644 --- a/src/main/python/ddadevops/domain/provider_hetzner.py +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Set +from typing import List, Dict, Set, Any from .common import Validateable, CredentialMappingDefault @@ -14,6 +14,10 @@ class Hetzner(Validateable, CredentialMappingDefault): result += self.__validate_is_not_empty__("hetzner_api_key") return result + def backend_config(self) -> Dict[str, Any]: + result = {} + return result + def resources_from_package(self) -> Set[str]: return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"} From 7ba01f3ee4f84345f86bcd2a4cf744be6295d80a Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 27 May 2023 11:42:04 +0200 Subject: [PATCH 163/173] minor fix --- build.py | 2 +- src/main/python/ddadevops/__init__.py | 9 +++++++-- src/main/python/ddadevops/domain/__init__.py | 11 ++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/build.py b/build.py index 1500532..6d5a160 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev63" +version = "4.0.0-dev64" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index c58913a..b9a224f 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -4,12 +4,17 @@ terraform, dda-pallet, aws & hetzner-cloud. """ +from .domain import DnsRecord, BuildType, MixinType, ReleaseType, ProviderType from .provs_k3s_build import ProvsK3sBuild -#from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config + +# from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .c4k_build import C4kBuild from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config from .devops_image_build import DevopsImageBuild -from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config +from .devops_terraform_build import ( + DevopsTerraformBuild, + create_devops_terraform_build_config, +) from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build from .credential import gopass_password_from_path, gopass_field_from_path from .release_mixin import ReleaseMixin diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index fdb1784..234c48b 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -1,4 +1,13 @@ -from .common import Validateable, CredentialMappingDefault, DnsRecord, Devops, BuildType, MixinType, ReleaseType, ProviderType +from .common import ( + Validateable, + CredentialMappingDefault, + DnsRecord, + Devops, + BuildType, + MixinType, + ReleaseType, + ProviderType, +) from .devops_factory import DevopsFactory from .image import Image from .c4k import C4k From 0103080bbdb77335164bb6b5e6b4b112ea03efb9 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sat, 27 May 2023 14:37:23 +0200 Subject: [PATCH 164/173] fix update runtime --- build.py | 2 +- src/main/python/ddadevops/c4k_build.py | 1 + src/main/python/ddadevops/devops_build.py | 5 ++++- src/main/python/ddadevops/provs_k3s_build.py | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 6d5a160..7240009 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev64" +version = "4.0.0-dev66" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/c4k_build.py b/src/main/python/ddadevops/c4k_build.py index 4b95c63..8a0ee32 100644 --- a/src/main/python/ddadevops/c4k_build.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -53,6 +53,7 @@ class C4kBuild(DevopsBuild): raise ValueError("C4kBuild requires BuildType.C4K") def update_runtime_config(self, dns_record: DnsRecord): + super().update_runtime_config(dns_record) devops = self.devops_repo.get_devops(self.project) devops.specialized_builds[BuildType.C4K].update_runtime_config(dns_record) self.devops_repo.set_devops(self.project, devops) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index 40441b1..c840147 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,5 +1,5 @@ import deprecation -from .domain import InitService +from .domain import InitService, DnsRecord from .infrastructure import DevopsRepository, FileApi @@ -40,3 +40,6 @@ class DevopsBuild: def initialize_build_dir(self): devops = self.devops_repo.get_devops(self.project) self.file_api.clean_dir(devops.build_path()) + + def update_runtime_config(self, dns_record: DnsRecord): + pass diff --git a/src/main/python/ddadevops/provs_k3s_build.py b/src/main/python/ddadevops/provs_k3s_build.py index a6baabb..ecbc216 100644 --- a/src/main/python/ddadevops/provs_k3s_build.py +++ b/src/main/python/ddadevops/provs_k3s_build.py @@ -19,6 +19,7 @@ class ProvsK3sBuild(DevopsBuild): raise ValueError("K3SBuild requires BuildType.K3S") def update_runtime_config(self, dns_record: DnsRecord): + super().update_runtime_config(dns_record) devops = self.devops_repo.get_devops(self.project) devops.specialized_builds[BuildType.K3S].update_runtime_config(dns_record) self.devops_repo.set_devops(self.project, devops) From b0b3f1bf55a6b68d7a3e5d84ef627c63a89b6b6e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Sun, 28 May 2023 17:15:25 +0200 Subject: [PATCH 165/173] fixed last missing pieces --- .gitlab-ci.yml | 2 +- build.py | 2 +- src/main/python/ddadevops/application/terraform_service.py | 4 +--- src/main/python/ddadevops/domain/provider_hetzner.py | 3 +-- src/main/python/ddadevops/infrastructure/infrastructure.py | 4 ++-- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dff5793..4a0dd96 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ flake8: script: - pip install -r dev_requirements.txt - flake8 --max-line-length=120 --count --select=E9,F63,F7,F82 --show-source --statistics src/main/python/ddadevops/ - - flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics src/main/python/ddadevops/ + - flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --per-file-ignores="__init__.py:F401" --ignore=E722,W503 --statistics src/main/python/ddadevops/ mypy: stage: lint&test diff --git a/build.py b/build.py index 7240009..870d56c 100644 --- a/build.py +++ b/build.py @@ -28,7 +28,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev66" +version = "4.0.0-dev67" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index 815a62f..fe01938 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -76,9 +76,7 @@ class TerraformService: # TODO: internal? def init_client(self, devops: Devops): - terraform_domain = devops.specialized_builds[ - BuildType.TERRAFORM - ] + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] terraform = Terraform( working_dir=devops.build_path(), terraform_semantic_version=terraform_domain.tf_terraform_semantic_version, diff --git a/src/main/python/ddadevops/domain/provider_hetzner.py b/src/main/python/ddadevops/domain/provider_hetzner.py index a23d602..bb0b786 100644 --- a/src/main/python/ddadevops/domain/provider_hetzner.py +++ b/src/main/python/ddadevops/domain/provider_hetzner.py @@ -15,8 +15,7 @@ class Hetzner(Validateable, CredentialMappingDefault): return result def backend_config(self) -> Dict[str, Any]: - result = {} - return result + return {} def resources_from_package(self) -> Set[str]: return {"provider_registry.tf", "hetzner_provider.tf", "hetzner_mixin_vars.tf"} diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 18edca1..ae8b5ce 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -114,11 +114,11 @@ class ExecutionApi: output = output.rstrip() return output - def execute_live(self, command, dry_run=False): + def execute_live(self, command, dry_run=False, shell=True): if dry_run: print(command) else: - process = Popen(command, stdout=PIPE) + process = Popen(command, stdout=PIPE, shell=shell) for line in iter(process.stdout.readline, b""): print(line.decode("utf-8"), end="") process.stdout.close() From 63baaaf411a2a6bda98919190b1b4afca05dabc4 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 31 May 2023 08:36:12 +0200 Subject: [PATCH 166/173] remove deprecated functions --- src/main/python/ddadevops/__init__.py | 9 ++-- src/main/python/ddadevops/c4k_build.py | 41 ------------------- src/main/python/ddadevops/credential.py | 16 +++----- src/main/python/ddadevops/devops_build.py | 12 ------ .../python/ddadevops/devops_image_build.py | 30 +------------- .../ddadevops/devops_terraform_build.py | 38 +---------------- src/main/python/ddadevops/hetzner_mixin.py | 19 +++------ src/main/python/ddadevops/python_util.py | 22 ---------- src/test/python/test_c4k_build.py | 2 +- 9 files changed, 17 insertions(+), 172 deletions(-) delete mode 100644 src/main/python/ddadevops/python_util.py diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index b9a224f..ee8d6ca 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -9,13 +9,10 @@ from .provs_k3s_build import ProvsK3sBuild # from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .c4k_build import C4kBuild -from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config +from .hetzner_mixin import HetznerMixin from .devops_image_build import DevopsImageBuild -from .devops_terraform_build import ( - DevopsTerraformBuild, - create_devops_terraform_build_config, -) -from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build +from .devops_terraform_build import DevopsTerraformBuild +from .devops_build import DevopsBuild, get_devops_build from .credential import gopass_password_from_path, gopass_field_from_path from .release_mixin import ReleaseMixin diff --git a/src/main/python/ddadevops/c4k_build.py b/src/main/python/ddadevops/c4k_build.py index 8a0ee32..b9df490 100644 --- a/src/main/python/ddadevops/c4k_build.py +++ b/src/main/python/ddadevops/c4k_build.py @@ -1,49 +1,8 @@ -import deprecation from .domain import BuildType, DnsRecord from .devops_build import DevopsBuild -from .credential import gopass_field_from_path, gopass_password_from_path from .infrastructure import ExecutionApi -@deprecation.deprecated(deprecated_in="3.2", details="use direct dict instead") -def add_c4k_mixin_config( - config, - c4k_config_dict, - c4k_auth_dict, - executable_name=None, - grafana_cloud_user=None, - grafana_cloud_password=None, - grafana_cloud_url="https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push", -): - if not grafana_cloud_user: - grafana_cloud_user = gopass_field_from_path( - "server/meissa/grafana-cloud", "grafana-cloud-user" - ) - if not grafana_cloud_password: - grafana_cloud_password = gopass_password_from_path( - "server/meissa/grafana-cloud" - ) - c4k_auth_dict.update( - { - "mon-auth": { - "grafana-cloud-user": grafana_cloud_user, - "grafana-cloud-password": grafana_cloud_password, - } - } - ) - c4k_config_dict.update({"mon-cfg": {"grafana-cloud-url": grafana_cloud_url}}) - config.update( - { - "C4kMixin": { - "executable_name": executable_name, - "config": c4k_config_dict, - "auth": c4k_auth_dict, - } - } - ) - return config - - class C4kBuild(DevopsBuild): def __init__(self, project, config): super().__init__(project, config) diff --git a/src/main/python/ddadevops/credential.py b/src/main/python/ddadevops/credential.py index 12c6735..0e41f69 100644 --- a/src/main/python/ddadevops/credential.py +++ b/src/main/python/ddadevops/credential.py @@ -1,24 +1,18 @@ import deprecation -from .python_util import execute +from .infrastructure import CredentialsApi @deprecation.deprecated( deprecated_in="3.2", details="use infrastructure.CredentialsApi instead" ) def gopass_field_from_path(path, field): - credential = None - if path and field: - print("get field for: " + path + ", " + field) - credential = execute(["gopass", "show", path, field]) - return credential + api = CredentialsApi() + return api.gopass_field_from_path(path, field) @deprecation.deprecated( deprecated_in="3.2", details="use infrastructure.CredentialsApi instead" ) def gopass_password_from_path(path): - credential = None - if path: - print("get password for: " + path) - credential = execute(["gopass", "show", "--password", path]) - return credential + api = CredentialsApi() + return api.gopass_password_from_path(path) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index c840147..6b7b1fa 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -3,18 +3,6 @@ from .domain import InitService, DnsRecord from .infrastructure import DevopsRepository, FileApi -@deprecation.deprecated(deprecated_in="3.2", details="create objects direct instead") -def create_devops_build_config( - stage, project_root_path, module, build_dir_name="target" -): - return { - "stage": stage, - "project_root_path": project_root_path, - "module": module, - "build_dir_name": build_dir_name, - } - - def get_devops_build(project): return project.get_property("build") diff --git a/src/main/python/ddadevops/devops_image_build.py b/src/main/python/ddadevops/devops_image_build.py index 4dd3863..badafd8 100644 --- a/src/main/python/ddadevops/devops_image_build.py +++ b/src/main/python/ddadevops/devops_image_build.py @@ -1,34 +1,6 @@ -import deprecation from .domain import BuildType from .application import ImageBuildService -from .devops_build import DevopsBuild, create_devops_build_config - - -@deprecation.deprecated(deprecated_in="3.2", details="use direct dict instead") -def create_devops_docker_build_config( - stage, - project_root_path, - module, - dockerhub_user, - dockerhub_password, - build_dir_name="target", - use_package_common_files=True, - build_commons_path=None, - docker_build_commons_dir_name="docker", - docker_publish_tag=None, -): - ret = create_devops_build_config(stage, project_root_path, module, build_dir_name) - ret.update( - { - "dockerhub_user": dockerhub_user, - "dockerhub_password": dockerhub_password, - "use_package_common_files": use_package_common_files, - "docker_build_commons_dir_name": docker_build_commons_dir_name, - "build_commons_path": build_commons_path, - "docker_publish_tag": docker_publish_tag, - } - ) - return ret +from .devops_build import DevopsBuild class DevopsImageBuild(DevopsBuild): diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index b475358..3360cb0 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -1,43 +1,7 @@ -from .devops_build import DevopsBuild, create_devops_build_config +from .devops_build import DevopsBuild from .application import TerraformService -def create_devops_terraform_build_config( - stage, - project_root_path, - module, - additional_vars, - build_dir_name="target", - output_json_name=None, - use_workspace=True, - use_package_common_files=True, - build_commons_path=None, - terraform_build_commons_dir_name="terraform", - debug_print_terraform_command=False, - additional_tfvar_files=None, - terraform_semantic_version="1.0.8", -): - if not output_json_name: - output_json_name = "out_" + module + ".json" - if not additional_tfvar_files: - additional_tfvar_files = [] - ret = create_devops_build_config(stage, project_root_path, module, build_dir_name) - ret.update( - { - "additional_vars": additional_vars, - "output_json_name": output_json_name, - "use_workspace": use_workspace, - "use_package_common_files": use_package_common_files, - "build_commons_path": build_commons_path, - "terraform_build_commons_dir_name": terraform_build_commons_dir_name, - "debug_print_terraform_command": debug_print_terraform_command, - "additional_tfvar_files": additional_tfvar_files, - "terraform_semantic_version": terraform_semantic_version, - } - ) - return ret - - class DevopsTerraformBuild(DevopsBuild): def __init__(self, project, config): inp = config.copy() diff --git a/src/main/python/ddadevops/hetzner_mixin.py b/src/main/python/ddadevops/hetzner_mixin.py index 3cb7f6c..684334b 100644 --- a/src/main/python/ddadevops/hetzner_mixin.py +++ b/src/main/python/ddadevops/hetzner_mixin.py @@ -1,27 +1,20 @@ from .devops_terraform_build import DevopsTerraformBuild -def add_hetzner_mixin_config(config, hetzner_api_key): - config.update({'HetznerMixin': - {'hetzner_api_key': hetzner_api_key}}) - return config - - class HetznerMixin(DevopsTerraformBuild): - def __init__(self, project, config): super().__init__(project, config) - hetzner_mixin_config = config['HetznerMixin'] - self.hetzner_api_key = hetzner_mixin_config['hetzner_api_key'] + hetzner_mixin_config = config["HetznerMixin"] + self.hetzner_api_key = hetzner_mixin_config["hetzner_api_key"] def project_vars(self): ret = super().project_vars() if self.hetzner_api_key: - ret['hetzner_api_key'] = self.hetzner_api_key + ret["hetzner_api_key"] = self.hetzner_api_key return ret def copy_build_resources_from_package(self): super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package('provider_registry.tf') - self.copy_build_resource_file_from_package('hetzner_provider.tf') - self.copy_build_resource_file_from_package('hetzner_mixin_vars.tf') + self.copy_build_resource_file_from_package("provider_registry.tf") + self.copy_build_resource_file_from_package("hetzner_provider.tf") + self.copy_build_resource_file_from_package("hetzner_mixin_vars.tf") diff --git a/src/main/python/ddadevops/python_util.py b/src/main/python/ddadevops/python_util.py deleted file mode 100644 index e872ad6..0000000 --- a/src/main/python/ddadevops/python_util.py +++ /dev/null @@ -1,22 +0,0 @@ -from subprocess import check_output, Popen, PIPE -import deprecation - - -@deprecation.deprecated(deprecated_in="3.2", details="use ExecutionApi instead") -def execute(cmd, shell=False): - output = check_output(cmd, encoding="UTF-8", shell=shell) - return output.rstrip() - - -@deprecation.deprecated(deprecated_in="3.2", details="use ExecutionApi instead") -def execute_live(cmd): - process = Popen(cmd, stdout=PIPE) - for line in iter(process.stdout.readline, b""): - print(line.decode("utf-8"), end="") - process.stdout.close() - process.wait() - - -@deprecation.deprecated(deprecated_in="3.2", details="use domain.filter_none instead") -def filter_none(list_to_filter): - return [x for x in list_to_filter if x is not None] diff --git a/src/test/python/test_c4k_build.py b/src/test/python/test_c4k_build.py index f5a3a46..a60e744 100644 --- a/src/test/python/test_c4k_build.py +++ b/src/test/python/test_c4k_build.py @@ -1,7 +1,7 @@ import os from pybuilder.core import Project from src.main.python.ddadevops.domain import DnsRecord -from src.main.python.ddadevops.c4k_build import C4kBuild, add_c4k_mixin_config +from src.main.python.ddadevops.c4k_build import C4kBuild from .domain.helper import ( CredentialsApiMock, devops_config, From 003296112a24d8589719984c74fbd8d9ce02787c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 31 May 2023 08:37:44 +0200 Subject: [PATCH 167/173] fix linting --- src/main/python/ddadevops/devops_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index 6b7b1fa..7aa3639 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,4 +1,3 @@ -import deprecation from .domain import InitService, DnsRecord from .infrastructure import DevopsRepository, FileApi From 74027d5d6fa9cc1ef498b1eb5c47980bc1bbf901 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 31 May 2023 08:40:39 +0200 Subject: [PATCH 168/173] adjust version & homepage --- build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.py b/build.py index 870d56c..4b016ef 100644 --- a/build.py +++ b/build.py @@ -28,12 +28,12 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "4.0.0-dev67" +version = "4.0.0-dev68" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] -url = "https://github.com/DomainDrivenArchitecture/dda-devops-build" -requires_python = ">=3.8" # CHECK IF NEW VERSION EXISTS +url = "https://repo.prod.meissa.de/meissa/dda-devops-build" +requires_python = ">=3.10" # CHECK IF NEW VERSION EXISTS license = "Apache Software License" @init From e2fd3b1eb9778332a73a4cb7ccf14991a5b7e717 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 31 May 2023 08:52:19 +0200 Subject: [PATCH 169/173] make local methods more private --- doc/architecture/BuildAndMixins.md | 6 - .../application/terraform_service.py | 153 +++++++++--------- .../ddadevops/devops_terraform_build.py | 2 +- 3 files changed, 77 insertions(+), 84 deletions(-) diff --git a/doc/architecture/BuildAndMixins.md b/doc/architecture/BuildAndMixins.md index b4280cf..6418255 100644 --- a/doc/architecture/BuildAndMixins.md +++ b/doc/architecture/BuildAndMixins.md @@ -18,12 +18,8 @@ classDiagram copy_build_resource_file_from_package(name) copy_build_resources_from_package() copy_build_resources_from_dir() - copy_local_state() - rescue_local_state() initialize_build_dir() post_build() - init_client() - write_output(terraform) read_output_json() plan() plan_fail_on_diff() @@ -49,13 +45,11 @@ classDiagram class AwsBackendPropertiesMixin { def project_vars() copy_build_resources_from_package() - init_client() } class DigitaloceanBackendPropertiesMixin { project_vars(self) copy_build_resources_from_package(self) - init_client(self) } class DevopsImageBuild { diff --git a/src/main/python/ddadevops/application/terraform_service.py b/src/main/python/ddadevops/application/terraform_service.py index fe01938..94d2a2b 100644 --- a/src/main/python/ddadevops/application/terraform_service.py +++ b/src/main/python/ddadevops/application/terraform_service.py @@ -23,87 +23,18 @@ class TerraformService: TerraformApi(), ) - def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): - data = self.resource_api.read_resource( - f"src/main/resources/terraform/{resource_name}" - ) - self.file_api.write_data_to_file( - Path(f"{devops.build_path()}/{resource_name}"), data - ) - - def __copy_build_resources_from_package__(self, devops: Devops): - terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - for resource in terraform_domain.resources_from_package(): - self.__copy_build_resource_file_from_package__(resource, devops) - - def __copy_build_resources_from_dir__(self, devops: Devops): - terraform = devops.specialized_builds[BuildType.TERRAFORM] - self.file_api.cp_force( - f"{terraform.build_commons_path()}/*", devops.build_path() - ) - - def __print_terraform_command__(self, terraform: Terraform, devops: Devops): - terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - if terraform_domain.tf_debug_print_terraform_command: - output = f"cd {devops.build_path()} && {terraform.latest_cmd()}" - print(output) - - # TODO: internal? - def copy_local_state(self, devops: Devops): - terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - if terraform_domain.is_local_state(): - self.file_api.cp("terraform.tfstate", devops.build_path(), check=False) - - # TODO: internal? - def rescue_local_state(self, devops: Devops): - terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - if terraform_domain.is_local_state(): - self.file_api.cp( - f"{devops.build_path()}/terraform.tfstate", ".", check=False - ) - def initialize_build_dir(self, devops: Devops): terraform = devops.specialized_builds[BuildType.TERRAFORM] if terraform.tf_use_package_common_files: self.__copy_build_resources_from_package__(devops) else: self.__copy_build_resources_from_dir__(devops) - self.copy_local_state(devops) + self.__copy_local_state__(devops) self.file_api.cp("*.tf", devops.build_path(), check=False) self.file_api.cp("*.properties", devops.build_path(), check=False) self.file_api.cp("*.tfvars", devops.build_path(), check=False) self.file_api.cp_recursive("scripts", devops.build_path(), check=False) - # TODO: internal? - def init_client(self, devops: Devops): - terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - terraform = Terraform( - working_dir=devops.build_path(), - terraform_semantic_version=terraform_domain.tf_terraform_semantic_version, - ) - if terraform_domain.is_local_state(): - terraform.init() - else: - terraform.init(backend_config=terraform_domain.backend_config()) - self.__print_terraform_command__(terraform, devops) - if terraform_domain.tf_use_workspace: - try: - terraform.workspace("select", devops.stage) - self.__print_terraform_command__(terraform, devops) - except: - terraform.workspace("new", devops.stage) - self.__print_terraform_command__(terraform, devops) - return terraform - - # TODO: internal? - def write_output(self, terraform, devops: Devops): - terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - result = terraform.output(json=IsFlagged) - self.__print_terraform_command__(terraform, devops) - self.file_api.write_json_to_file( - Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}"), result - ) - def read_output(self, devops: Devops) -> map: terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] return self.file_api.read_json_fro_file( @@ -116,7 +47,7 @@ class TerraformService: detailed_exitcode = IsFlagged else: detailed_exitcode = None - terraform = self.init_client(devops) + terraform = self.__init_client__(devops) return_code, _, stderr = terraform.plan( detailed_exitcode=detailed_exitcode, capture_output=False, @@ -136,7 +67,7 @@ class TerraformService: auto_approve_flag = IsFlagged else: auto_approve_flag = None - terraform = self.init_client(devops) + terraform = self.__init_client__(devops) if version.parse( terraform_domain.tf_terraform_semantic_version ) >= version.parse("1.0.0"): @@ -158,11 +89,11 @@ class TerraformService: self.__print_terraform_command__(terraform, devops) if return_code > 0: raise RuntimeError(return_code, "terraform error:", stderr) - self.write_output(terraform, devops) + self.__write_output__(terraform, devops) def refresh(self, devops: Devops): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - terraform = self.init_client(devops) + terraform = self.__init_client__(devops) return_code, _, stderr = terraform.refresh( var=terraform_domain.project_vars(), var_file=terraform_domain.tf_additional_tfvar_files, @@ -170,7 +101,7 @@ class TerraformService: self.__print_terraform_command__(terraform, devops) if return_code > 0: raise RuntimeError(return_code, "terraform error:", stderr) - self.write_output(terraform, devops) + self.__write_output__(terraform, devops) def destroy(self, devops: Devops, auto_approve=False): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] @@ -178,7 +109,7 @@ class TerraformService: auto_approve_flag = IsFlagged else: auto_approve_flag = None - terraform = self.init_client(devops) + terraform = self.__init_client__(devops) if version.parse( terraform_domain.tf_terraform_semantic_version ) >= version.parse("1.0.0"): @@ -208,7 +139,7 @@ class TerraformService: tf_import_resource, ): terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] - terraform = self.init_client(devops) + terraform = self.__init_client__(devops) return_code, _, stderr = terraform.import_cmd( tf_import_name, tf_import_resource, @@ -220,3 +151,71 @@ class TerraformService: self.__print_terraform_command__(terraform, devops) if return_code > 0: raise RuntimeError(return_code, "terraform error:", stderr) + + def post_build(self, devops: Devops): + self.__rescue_local_state__(devops) + + def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops): + data = self.resource_api.read_resource( + f"src/main/resources/terraform/{resource_name}" + ) + self.file_api.write_data_to_file( + Path(f"{devops.build_path()}/{resource_name}"), data + ) + + def __copy_build_resources_from_package__(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + for resource in terraform_domain.resources_from_package(): + self.__copy_build_resource_file_from_package__(resource, devops) + + def __copy_build_resources_from_dir__(self, devops: Devops): + terraform = devops.specialized_builds[BuildType.TERRAFORM] + self.file_api.cp_force( + f"{terraform.build_commons_path()}/*", devops.build_path() + ) + + def __print_terraform_command__(self, terraform: Terraform, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if terraform_domain.tf_debug_print_terraform_command: + output = f"cd {devops.build_path()} && {terraform.latest_cmd()}" + print(output) + + def __copy_local_state__(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if terraform_domain.is_local_state(): + self.file_api.cp("terraform.tfstate", devops.build_path(), check=False) + + def __rescue_local_state__(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + if terraform_domain.is_local_state(): + self.file_api.cp( + f"{devops.build_path()}/terraform.tfstate", ".", check=False + ) + + def __init_client__(self, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + terraform = Terraform( + working_dir=devops.build_path(), + terraform_semantic_version=terraform_domain.tf_terraform_semantic_version, + ) + if terraform_domain.is_local_state(): + terraform.init() + else: + terraform.init(backend_config=terraform_domain.backend_config()) + self.__print_terraform_command__(terraform, devops) + if terraform_domain.tf_use_workspace: + try: + terraform.workspace("select", devops.stage) + self.__print_terraform_command__(terraform, devops) + except: + terraform.workspace("new", devops.stage) + self.__print_terraform_command__(terraform, devops) + return terraform + + def __write_output__(self, terraform, devops: Devops): + terraform_domain = devops.specialized_builds[BuildType.TERRAFORM] + result = terraform.output(json=IsFlagged) + self.__print_terraform_command__(terraform, devops) + self.file_api.write_json_to_file( + Path(f"{devops.build_path()}{terraform_domain.tf_output_json_name}"), result + ) diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 3360cb0..89901a0 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -22,7 +22,7 @@ class DevopsTerraformBuild(DevopsBuild): def post_build(self): devops = self.devops_repo.get_devops(self.project) - self.teraform_service.rescue_local_state(devops) + self.teraform_service.post_build(devops) def read_output_json(self) -> map: devops = self.devops_repo.get_devops(self.project) From 33609981756ff063f7fb5de4fe989cd63513a929 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 31 May 2023 09:24:54 +0200 Subject: [PATCH 170/173] not needed any more --- src/main/python/ddadevops/__init__.py | 1 - src/main/python/ddadevops/hetzner_mixin.py | 20 -------------------- 2 files changed, 21 deletions(-) delete mode 100644 src/main/python/ddadevops/hetzner_mixin.py diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index ee8d6ca..c9da925 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -9,7 +9,6 @@ from .provs_k3s_build import ProvsK3sBuild # from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .c4k_build import C4kBuild -from .hetzner_mixin import HetznerMixin from .devops_image_build import DevopsImageBuild from .devops_terraform_build import DevopsTerraformBuild from .devops_build import DevopsBuild, get_devops_build diff --git a/src/main/python/ddadevops/hetzner_mixin.py b/src/main/python/ddadevops/hetzner_mixin.py deleted file mode 100644 index 684334b..0000000 --- a/src/main/python/ddadevops/hetzner_mixin.py +++ /dev/null @@ -1,20 +0,0 @@ -from .devops_terraform_build import DevopsTerraformBuild - - -class HetznerMixin(DevopsTerraformBuild): - def __init__(self, project, config): - super().__init__(project, config) - hetzner_mixin_config = config["HetznerMixin"] - self.hetzner_api_key = hetzner_mixin_config["hetzner_api_key"] - - def project_vars(self): - ret = super().project_vars() - if self.hetzner_api_key: - ret["hetzner_api_key"] = self.hetzner_api_key - return ret - - def copy_build_resources_from_package(self): - super().copy_build_resources_from_package() - self.copy_build_resource_file_from_package("provider_registry.tf") - self.copy_build_resource_file_from_package("hetzner_provider.tf") - self.copy_build_resource_file_from_package("hetzner_mixin_vars.tf") From 0c0703c7ac678b6335351d13ba691f0c8a06e54e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 31 May 2023 09:25:10 +0200 Subject: [PATCH 171/173] write rationale --- README.md | 101 ++++++++++++++++++++++------- doc/architecture/BuildAndMixins.md | 98 ---------------------------- 2 files changed, 78 insertions(+), 121 deletions(-) delete mode 100644 doc/architecture/BuildAndMixins.md diff --git a/README.md b/README.md index 918b36c..aeccb26 100644 --- a/README.md +++ b/README.md @@ -4,36 +4,91 @@ ![release prod](https://github.com/DomainDrivenArchitecture/dda-devops-build/workflows/release%20prod/badge.svg) -dda-devops-build provide a environment to tie several DevOps tools together for easy interoperation. Supported tools are: -* aws with - * simple api-key auth - * mfa & assume-role auth -* hetzner with simple api-key auth -* terraform v0.11, v0.12 supporting - * local file backends - * s3 backends -* docker / dockerhub -* user / team credentials managed by gopass -* dda-pallet +dda-devops-build integrates all the tools we use to work with clouds & provide some nice functions around. + +Tools we support are + +* terraform: for setting up the plain infrastructure around. +* docker: for creating images +* c4k: for generating kubernetes manifests +* provs: for setting up small single-node k3s clusters +* gopass: for credential management on devops computers +* cloud providers: hetzner, digitalocean, aws + +In addition we provide a ReleaseMixin for release related tasks like tag / publish & version-bump + +```mermaid +classDiagram + class DevopsBuild { + name() + build_path() + initialize_build_dir() + } + + + class DevopsTerraformBuild { + terraform_build_commons_path() + project_vars() + initialize_build_dir() + post_build() + read_output_json() + plan() + plan_fail_on_diff() + apply(auto_approve=False) + refresh() + destroy(auto_approve=False) + tf_import(tf_import_name, tf_import_resource,) + print_terraform_command(terraform) + } + + class DevopsImageBuild { + def initialize_build_dir() + image() + drun() + dockerhub_login() + dockerhub_publish() + test() + } + + class ReleaseMixin { + prepare_release() + tag_and_push_release() + } + + class ProvsK3sBuild { + def update_runtime_config(dns_record) + write_provs_config() + provs_apply(dry_run=False) + } + + class C4kBuild { + def update_runtime_config(dns_record) + def write_c4k_config() + def write_c4k_auth() + c4k_apply(dry_run=False) + } + + DevopsBuild <|-- DevopsImageBuild + DevopsBuild <|-- DevopsTerraformBuild + DevopsBuild <|-- ReleaseMixin + DevopsBuild <|-- ProvsK3sBuild + DevopsBuild <|-- C4kBuild + +``` + +Principles we follow are: + +* Seperate build artefacts from version controlled code +* Domain Driven Design - in order to stay sustainable # Setup -Ensure that yout python3 version is at least Python 3.7! +Ensure that yout python3 version is at least Python 3.10 ``` sudo apt install python3-pip -pip3 install pip3 --upgrade -pip3 install pybuilder ddadevops deprecation +pip3 install -r requirements.txt export PATH=$PATH:~/.local/bin - -# in case of using terraform -pip3 install dda-python-terraform packaging - -# in case of using AwsMixin -pip3 install boto3 - -# in case of using AwsMfaMixin -pip3 install boto3 mfa ``` # Example Build diff --git a/doc/architecture/BuildAndMixins.md b/doc/architecture/BuildAndMixins.md deleted file mode 100644 index 6418255..0000000 --- a/doc/architecture/BuildAndMixins.md +++ /dev/null @@ -1,98 +0,0 @@ -# Overview of Build and Mixins - -* Build can be used standalone -* Mixin can be added to Build - -```mermaid -classDiagram - class DevopsBuild { - name() - build_path() - initialize_build_dir() - } - - - class DevopsTerraformBuild { - terraform_build_commons_path() - project_vars() - copy_build_resource_file_from_package(name) - copy_build_resources_from_package() - copy_build_resources_from_dir() - initialize_build_dir() - post_build() - read_output_json() - plan() - plan_fail_on_diff() - apply(auto_approve=False) - refresh() - destroy(auto_approve=False) - tf_import(tf_import_name, tf_import_resource,) - print_terraform_command(terraform) - } - - class HetznerMixin { - // HetznerMixin -> HetznerTerraformBuild - project_vars() - copy_build_resources_from_package() - } - - class ExoscaleMixin { - // ExoscaleMixin -> ExoscaleTerraformBuild - project_vars() - copy_build_resources_from_package() - } - - class AwsBackendPropertiesMixin { - def project_vars() - copy_build_resources_from_package() - } - - class DigitaloceanBackendPropertiesMixin { - project_vars(self) - copy_build_resources_from_package(self) - } - - class DevopsImageBuild { - def initialize_build_dir() - image() - drun() - dockerhub_login() - dockerhub_publish() - test() - } - - class ReleaseMixin { - prepare_release() - tag_and_push_release() - } - - class ProvsK3sBuild { - // ProvsK3sBuild -> ProvsK3sBuild - def update_runtime_config(fqdn, ipv4, ipv6=None) - write_provs_config() - provs_apply(dry_run=False) - } - - class C4kMixin { - // C4kMixin -> C4k - def write_c4k_config() - def write_c4k_auth() - c4k_apply(dry_run=False) - } - - DevopsBuild <|-- DevopsImageBuild - DevopsBuild <|-- DevopsTerraformBuild - DevopsBuild <|-- AwsRdsPgMixin - DevopsBuild <|-- ReleaseMixin - - DevopsTerraformBuild <|-- AwsBackendPropertiesMixin - DevopsTerraformBuild <|-- DigitaloceanTerraformBuild - DevopsTerraformBuild <|--ExoscaleMixin - DevopsTerraformBuild <|--HetznerMixin - DevopsBuild <|-- ProvsK3sBuild - DigitaloceanTerraformBuild <|-- DigitaloceanBackendPropertiesMixin - AwsBackendPropertiesMixin <|-- AwsMfaMixin - - DevopsBuild <|-- C4kMixin - -``` From 3af2d9c2c26e032ffc4adc0fd060861fffd85c2b Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Thu, 1 Jun 2023 09:28:43 +0200 Subject: [PATCH 172/173] add more doc --- README.md | 185 +++++++++++---------------------------------- doc/DevopsBuild.md | 61 +++++++++++++++ 2 files changed, 104 insertions(+), 142 deletions(-) create mode 100644 doc/DevopsBuild.md diff --git a/README.md b/README.md index aeccb26..284045e 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ classDiagram DevopsBuild <|-- ProvsK3sBuild DevopsBuild <|-- C4kBuild + link DevopsBuild "./doc/DevopsBuild.md" + ``` Principles we follow are: @@ -81,7 +83,7 @@ Principles we follow are: * Seperate build artefacts from version controlled code * Domain Driven Design - in order to stay sustainable -# Setup +## Installation Ensure that yout python3 version is at least Python 3.10 @@ -91,7 +93,13 @@ pip3 install -r requirements.txt export PATH=$PATH:~/.local/bin ``` -# Example Build +## Reference + +* [DevopsBuild](./doc/DevopsBuild.md) + + + +## Example Build lets assume the following project structure @@ -105,7 +113,7 @@ my-project | | -> ... ``` -``` +```python from pybuilder.core import task, init from ddadevops import * @@ -113,22 +121,36 @@ name = 'my-project' MODULE = 'my-module' PROJECT_ROOT_PATH = '..' -class MyBuild(DevopsTerraformBuild): - pass - @init def initialize(project): - project.build_depends_on('ddadevops>=0.5.0') - account_name = 'my-aws-account-name' - account_id = 'my-aws-account-id' - stage = 'my stage i.e. dev|test|prod' - additional_vars = {'var_to_use_insied_terraform': '...'} - additional_var_files = ['variable-' + account_name + '-' + stage + '.tfvars'] - config = create_devops_terraform_build_config(stage, PROJECT_ROOT_PATH, - MODULE, additional_vars, - additional_tfvar_files=additional_var_files) - build = MyBuild(project, config) + project.build_depends_on("ddadevops>=4.0.0-dev") + + config = { + "credentials_mapping": [ + { + "gopass_path": environ.get("DIGITALOCEAN_TOKEN_KEY_PATH", None), + "name": "do_api_key", + }, + { + "gopass_path": environ.get("HETZNER_API_KEY_PATH", None), + "name": "hetzner_api_key", + }, + ], + "name": name, + "module": MODULE, + "stage": environ["STAGE"], + "project_root_path": PROJECT_ROOT_PATH, + "build_types": ["TERRAFORM"], + "mixin_types": [], + "tf_provider_types": ["DIGITALOCEAN", "HETZNER"], + "tf_use_workspace": False, + "tf_terraform_semantic_version": "1.4.2", + "do_as_backend": True, + "do_bucket": "your-bucket", + } + + build = DevopsTerraformBuild(project, config) build.initialize_build_dir() @@ -141,137 +163,16 @@ def plan(project): @task def apply(project): build = get_devops_build(project) - build.apply() + build.apply(True) + @task def destroy(project): build = get_devops_build(project) - build.destroy() + build.destroy(True) -@task -def tf_import(project): - build = get_devops_build(project) - build.tf_import('aws_resource.choosen_name', 'the_aws_id') ``` -## Feature aws-backend - -Will use a file `backend.dev.live.properties` where dev is the [account-name], live is the [stage]. - -the backend.dev.live.properties file content: -``` -key = ".." -region = "the aws region" -profile = "the profile used for aws" -bucket = "the s3 bucket name" -kms_key_id = "the aws key id" -``` - -the build.py file content: -``` -class MyBuild(AwsBackendPropertiesMixin, DevopsTerraformBuild): - pass - - -@init -def initialize(project): - project.build_depends_on('ddadevops>=1.0') - account_name = 'my-aws-account-name' - account_id = 'my-aws-account-id' - stage = 'my stage i.e. dev|test|prod' - additional_vars = {} - config = create_devops_terraform_build_config(stage, PROJECT_ROOT_PATH, - MODULE, additional_vars) - config = add_aws_backend_properties_mixin_config(config, account_name) - build = MyBuild(project, config) - build.initialize_build_dir() -``` - -## Feature aws-mfa-assume-role - -In order to use aws assume role in combination with the mfa-tool (`pip install mfa`): - -the build.py file content: -``` -class MyBuild(class MyBuild(AwsMfaMixin, DevopsTerraformBuild): - pass - - -@init -def initialize(project): - project.build_depends_on('ddadevops>=1.0') - account_name = 'my-aws-account-name' - account_id = 'my-aws-account-id' - stage = 'my stage i.e. dev|test|prod' - additional_vars = {} - config = create_devops_terraform_build_config(stage, PROJECT_ROOT_PATH, - MODULE, additional_vars) - config = add_aws_backend_properties_mixin_config(config, account_name) - config = add_aws_mfa_mixin_config(config, account_id, 'eu-central-1', - mfa_role='my_developer_role', - mfa_account_prefix='company-', - mfa_login_account_suffix='users_are_defined_here') - build = MyBuild(project, config) - build.initialize_build_dir() - -@task -def access(project): - build = get_devops_build(project) - build.get_mfa_session() -``` - -## Feature DdaImageBuild - -The docker build supports image building, tagging, testing and login to dockerhost. -For bash based builds we support often used script-parts as predefined functions [see install_functions.sh](src/main/resources/docker/image/resources/install_functions.sh). - -A full working example: [doc/example/50_docker_module](doc/example/50_docker_module) - -## Feature AwsRdsPgMixin - -The AwsRdsPgMixin provides -* execute_pg_rds_sql - function will optionally resolve dns-c-names for trusted ssl-handshakes -* alter_db_user_password -* add_new_user -* deactivate_user - -the build.py file content: -``` -class MyBuild(..., AwsRdsPgMixin): - pass - - -@init -def initialize(project): - project.build_depends_on('ddadevops>=1.0') - - ... - config = add_aws_rds_pg_mixin_config(config, - stage + "-db.bcsimport.kauf." + account_name + ".breuni.de", - "kauf_bcsimport", - rds_resolve_dns=True,) - build = MyBuild(project, config) - build.initialize_build_dir() - -@task -def rotate_credentials_in(project): - build = get_devops_build(project) - build.alter_db_user_password('/postgres/support') - build.alter_db_user_password('/postgres/superuser') - build.add_new_user('/postgres/superuser', '/postgres/app', 'pg_group_role') - - -@task -def rotate_credentials_out(project): - build = get_devops_build(project) - build.deactivate_user('/postgres/superuser', 'old_user_name') -``` - -# Releasing and updating -## Publish snapshot - -1. every push will be published as dev-dependency - ## Release ``` @@ -285,7 +186,7 @@ git push pip3 install --upgrade --user ddadevops ``` -# License +## License -Copyright © 2021 meissa GmbH +Copyright © 2023 meissa GmbH Licensed under the [Apache License, Version 2.0](LICENSE) (the "License") diff --git a/doc/DevopsBuild.md b/doc/DevopsBuild.md new file mode 100644 index 0000000..16e63d2 --- /dev/null +++ b/doc/DevopsBuild.md @@ -0,0 +1,61 @@ +# DevopsBuild + +DevopsBuild stellt die build Grundlagen zur Verfügung. + +```mermaid +classDiagram + class DevopsBuild { + name() - thename of build + build_path() - the build dir in target + initialize_build_dir() - copy current directory & additional files to target + } + +``` + +## Input + +| name | description | default | +| ----------------- | -------------------------------------------------------------------------------------------------- | ------- | +| name | dedicated name of the build | module | +| module | module name - may result in a hierarchy like name/module | | +| stage | sth. like test, int, acc or prod | | +| project_root_path | relative path to projects root. Is used to locate the target dir | | +| build_dir_name | name of dir, build is executed in | target | +| build_types | list of special builds used. Valid values are ["IMAGE", "C4K", "K3S", "TERRAFORM"] | [] | +| mixin_types | mixins are orthoganl to builds and represent additional capabilities. Valid Values are ["RELEASE"] | [] | + + +## Example Usage + + +```python +from subprocess import run +from pybuilder.core import task, init +from ddadevops import * + +name = 'my-project' +MODULE = 'my-module' +PROJECT_ROOT_PATH = '..' + +@init +def initialize(project): + project.build_depends_on("ddadevops>=4.0.0") + + config = { + "name": name, + "module": MODULE, + "stage": environ["STAGE"], + "project_root_path": PROJECT_ROOT_PATH, + "build_types": [], + "mixin_types": [], + } + + build = DevopsTerraformBuild(project, config) + build.initialize_build_dir() + + +@task +def list_build_dir(project): + build = get_devops_build(project) + run(f"ls -la {build.build_path()}") +``` \ No newline at end of file From ba26f5aa6a779a7f8aec1432134de12c04f01759 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 2 Jun 2023 12:47:56 +0200 Subject: [PATCH 173/173] minor updates --- doc/DevopsBuild.md | 2 +- doc/architecture/Architecture.md | 1 + doc/architecture/Domain.md | 3 ++- src/main/python/ddadevops/domain/build_file.py | 6 ++++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/DevopsBuild.md b/doc/DevopsBuild.md index 16e63d2..f753a39 100644 --- a/doc/DevopsBuild.md +++ b/doc/DevopsBuild.md @@ -5,7 +5,7 @@ DevopsBuild stellt die build Grundlagen zur Verfügung. ```mermaid classDiagram class DevopsBuild { - name() - thename of build + name() - the name of build build_path() - the build dir in target initialize_build_dir() - copy current directory & additional files to target } diff --git a/doc/architecture/Architecture.md b/doc/architecture/Architecture.md index 3412691..a4922b2 100644 --- a/doc/architecture/Architecture.md +++ b/doc/architecture/Architecture.md @@ -14,6 +14,7 @@ Rel(buildAndMixin,dom, "use") Rel(app, dom, "use") Rel(app, infra, "use") + Rel(infra, dom, "use") UpdateLayoutConfig($c4ShapeInRow="2", $c4BoundaryInRow="1") diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 778f1dd..8148756 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -126,7 +126,8 @@ classDiagram BuildFile *-- "1" Version C4k *-- DnsRecord: dns_record ProvsK3s *-- DnsRecord: provision_dns - Credentials *-- "0..n" CredentialMapping: mappings + Credentials *-- "0..n" CredentialMapping: mappings[name] + Credentials *-- "0..n" CredentialMapping: default_mappings ``` diff --git a/src/main/python/ddadevops/domain/build_file.py b/src/main/python/ddadevops/domain/build_file.py index c7537a8..018fba3 100644 --- a/src/main/python/ddadevops/domain/build_file.py +++ b/src/main/python/ddadevops/domain/build_file.py @@ -113,3 +113,9 @@ class BuildFile(Validateable): self.content = substitute except: raise Exception(f"Version not found in file {self.file_path}") + + def __eq__(self, other): + return other and self.file_path == other.file_path + + def __hash__(self) -> int: + return self.file_path.__hash__()