From c15503b7a0fd970c6ebcd31b24880de3e8b73d59 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 3 Aug 2023 12:15:18 +0200 Subject: [PATCH 01/57] Update artifact deployment mixin --- src/main/python/ddadevops/artifact_deployment_mixin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/artifact_deployment_mixin.py b/src/main/python/ddadevops/artifact_deployment_mixin.py index 144fe53..d48e91c 100644 --- a/src/main/python/ddadevops/artifact_deployment_mixin.py +++ b/src/main/python/ddadevops/artifact_deployment_mixin.py @@ -1,5 +1,6 @@ from pybuilder.core import Project from .devops_build import DevopsBuild +from .domain import MixinType # """ # Functional Req: @@ -46,7 +47,7 @@ from .devops_build import DevopsBuild # generate sha256 sums & generate sha512 sums of results of [-1] # [6] -# push results of [-1] & [5] to [0]/[4] +# push results of [-1] & [5] to [0]/[4]/assets # """ @@ -54,3 +55,7 @@ from .devops_build import DevopsBuild class ArtifactDeploymentMixin(DevopsBuild): def __init__(self, project: Project, inp: dict): super().__init__(project, inp) + devops = self.devops_repo.get_devops(self.project) + if MixinType.ARTIFACT_DEPLOYMENT not in devops.mixins: # TODO: Check for Release mixin as well + raise ValueError("ArtifactDeploymentMixin requires MixinType.ARTIFACT_DEPLOYMENT") + self.base_url = 'https://repo.prod.meissa.de/api/v1/repos/' From e6450b796fe8a2b7c9141561bb9a16a2d78fc252 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 3 Aug 2023 12:15:37 +0200 Subject: [PATCH 02/57] Add basic artifact service --- .../application/artifact_deployment_service.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/main/python/ddadevops/application/artifact_deployment_service.py diff --git a/src/main/python/ddadevops/application/artifact_deployment_service.py b/src/main/python/ddadevops/application/artifact_deployment_service.py new file mode 100644 index 0000000..f90847a --- /dev/null +++ b/src/main/python/ddadevops/application/artifact_deployment_service.py @@ -0,0 +1,15 @@ +from ..infrastructure import GitApi + + +class ArtifactDeploymentService: + def __init__(self, git_api: GitApi): + self.git_api = git_api + + @classmethod + def prod(cls): + return cls( + GitApi(), + ) + +# def __get_base_artifact_release_url(self, base_url: str, organization: str, name: str) -> str: +# return f"{base_url}/{organization}/{name}/releases" From 6dbbb8f2a11c06abf3b8ffeba24c871ce0364bd1 Mon Sep 17 00:00:00 2001 From: erik Date: Thu, 3 Aug 2023 12:45:53 +0200 Subject: [PATCH 03/57] [Skip-CI] Add ArtifactDeploymentApi --- .../infrastructure/infrastructure.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 99ec45c..2c27fb1 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -206,3 +206,23 @@ class GitApi: class TerraformApi: pass + + +class ArtifactDeploymentApi: + def __init__(self): + self.execution_api = ExecutionApi() + + def post_release(self, target_url: str, token: str, tag: str): + self.execution_api.execute_secure(f"curl -X 'POST' '{target_url}' -H 'accept: application/json' -H 'Content-Type: application/json' -d " + + "'{ \"body\": \"Provides files for release " + tag + "\\nAttention: The \\\"Source Code\\\"-files below are not up-to-date!\", " + f"\"tag_name\": \"{tag}\"" + "}'" + + f"-H \"Authorization: token {token}\"") + + def post_asset(self, target_url: str, release_id: str, token: str, attachment: str, type: str): + self.execution_api.execute_secure(f"curl -X 'POST' '{target_url}/{release_id}/assets' -H 'accept: application/json' -H \"Authorization: token {token}\" -H 'Content-Type: multipart/form-data' -F 'attachment=@{attachment};type={type}'") + + def calculate_checksums(self, build_path: str): + self.execution_api.execute("find " + build_path + " -type f -exec sha256sum {} \\; | sort > " + build_path + "sha256sum.lst") + self.execution_api.execute("find " + build_path + " -type f -exec sha512sum {} \\; | sort > " + build_path + "sha512sum.lst") + + + \ No newline at end of file From 09bf3f5d4478a89dc8e74bf2c096b5b9680bd5b2 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 3 Aug 2023 13:33:57 +0200 Subject: [PATCH 04/57] Switch some quotes and brackets --- .../infrastructure/infrastructure.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 2c27fb1..95b5105 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -213,16 +213,24 @@ class ArtifactDeploymentApi: self.execution_api = ExecutionApi() def post_release(self, target_url: str, token: str, tag: str): - self.execution_api.execute_secure(f"curl -X 'POST' '{target_url}' -H 'accept: application/json' -H 'Content-Type: application/json' -d " + - "'{ \"body\": \"Provides files for release " + tag + "\\nAttention: The \\\"Source Code\\\"-files below are not up-to-date!\", " + f"\"tag_name\": \"{tag}\"" + "}'" + - f"-H \"Authorization: token {token}\"") + self.execution_api.execute_secure(f'curl -X "POST" "{target_url}" ' + + '-H "accept: application/json" -H "Content-Type: application/json" ' + + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501 + + f'-H "Authorization: token {token}"', + sanitized_command=f'curl -X "POST" "{target_url}" ' + + '-H "accept: application/json" -H "Content-Type: application/json" ' + + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ') # noqa: E501 + + def post_asset(self, target_url: str, release_id: str, token: str, attachment: str, attachment_type: str): + self.execution_api.execute_secure(f'curl -X "POST" "{target_url}/{release_id}/assets" ' + + f'-H "accept: application/json" -H "Authorization: token {token}" ' + + '-H "Content-Type: multipart/form-data" ' + + f'-F "attachment=@{attachment};type={attachment_type}"', + sanitized_command=f'curl -X "POST" "{target_url}/{release_id}/assets" ' + + '-H "accept: application/json" ' + + '-H "Content-Type: multipart/form-data" ' + + f'-F "attachment=@{attachment};type={attachment_type}"') - def post_asset(self, target_url: str, release_id: str, token: str, attachment: str, type: str): - self.execution_api.execute_secure(f"curl -X 'POST' '{target_url}/{release_id}/assets' -H 'accept: application/json' -H \"Authorization: token {token}\" -H 'Content-Type: multipart/form-data' -F 'attachment=@{attachment};type={type}'") - def calculate_checksums(self, build_path: str): - self.execution_api.execute("find " + build_path + " -type f -exec sha256sum {} \\; | sort > " + build_path + "sha256sum.lst") - self.execution_api.execute("find " + build_path + " -type f -exec sha512sum {} \\; | sort > " + build_path + "sha512sum.lst") - - - \ No newline at end of file + self.execution_api.execute(f"find {build_path} -type f -exec sha256sum {{}}; | sort > {build_path} sha256sum.lst") + self.execution_api.execute(f"find {build_path} -type f -exec sha512sum {{}}; | sort > {build_path} sha512sum.lst") From 8d4921ea704bd13d35446c488ad46f526b6e9cf7 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 4 Aug 2023 11:47:32 +0200 Subject: [PATCH 05/57] Switch order of function parameters --- .../python/ddadevops/infrastructure/infrastructure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 95b5105..3279430 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -212,7 +212,7 @@ class ArtifactDeploymentApi: def __init__(self): self.execution_api = ExecutionApi() - def post_release(self, target_url: str, token: str, tag: str): + def post_release(self, target_url: str, tag: str, token: str): self.execution_api.execute_secure(f'curl -X "POST" "{target_url}" ' + '-H "accept: application/json" -H "Content-Type: application/json" ' + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501 @@ -221,12 +221,12 @@ class ArtifactDeploymentApi: + '-H "accept: application/json" -H "Content-Type: application/json" ' + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ') # noqa: E501 - def post_asset(self, target_url: str, release_id: str, token: str, attachment: str, attachment_type: str): - self.execution_api.execute_secure(f'curl -X "POST" "{target_url}/{release_id}/assets" ' + def post_asset(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): + self.execution_api.execute_secure(f'curl -X "POST" "{target_url}/{release_id}/assets" ' # {target_url}/{release_id}/assets move to Domain + f'-H "accept: application/json" -H "Authorization: token {token}" ' + '-H "Content-Type: multipart/form-data" ' + f'-F "attachment=@{attachment};type={attachment_type}"', - sanitized_command=f'curl -X "POST" "{target_url}/{release_id}/assets" ' + sanitized_command=f'curl -X "POST" "{target_url}/{release_id}/assets" ' # see above + '-H "accept: application/json" ' + '-H "Content-Type: multipart/form-data" ' + f'-F "attachment=@{attachment};type={attachment_type}"') From c02440ac65eee037d971bc5eff561038f1f7abf4 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 4 Aug 2023 11:47:55 +0200 Subject: [PATCH 06/57] Return values of post requests --- src/main/python/ddadevops/infrastructure/infrastructure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 3279430..40fee1b 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -213,7 +213,7 @@ class ArtifactDeploymentApi: self.execution_api = ExecutionApi() def post_release(self, target_url: str, tag: str, token: str): - self.execution_api.execute_secure(f'curl -X "POST" "{target_url}" ' + return self.execution_api.execute_secure(f'curl -X "POST" "{target_url}" ' + '-H "accept: application/json" -H "Content-Type: application/json" ' + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501 + f'-H "Authorization: token {token}"', @@ -222,7 +222,7 @@ class ArtifactDeploymentApi: + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ') # noqa: E501 def post_asset(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): - self.execution_api.execute_secure(f'curl -X "POST" "{target_url}/{release_id}/assets" ' # {target_url}/{release_id}/assets move to Domain + return self.execution_api.execute_secure(f'curl -X "POST" "{target_url}/{release_id}/assets" ' # {target_url}/{release_id}/assets move to Domain + f'-H "accept: application/json" -H "Authorization: token {token}" ' + '-H "Content-Type: multipart/form-data" ' + f'-F "attachment=@{attachment};type={attachment_type}"', From f3bf8cb335f33c51ae36fc80733bcd4f57ff3c2e Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 4 Aug 2023 11:48:16 +0200 Subject: [PATCH 07/57] Add skeleton for ArtifactDeploymentService --- .../artifact_deployment_service.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/python/ddadevops/application/artifact_deployment_service.py b/src/main/python/ddadevops/application/artifact_deployment_service.py index f90847a..e7e3c61 100644 --- a/src/main/python/ddadevops/application/artifact_deployment_service.py +++ b/src/main/python/ddadevops/application/artifact_deployment_service.py @@ -1,15 +1,27 @@ -from ..infrastructure import GitApi - +from ..infrastructure import GitApi, ArtifactDeploymentApi +from ..domain import Credentials class ArtifactDeploymentService: - def __init__(self, git_api: GitApi): + def __init__(self, git_api: GitApi, artifact_deployment_api: ArtifactDeploymentApi): self.git_api = git_api + self.artifact_deployment_api = artifact_deployment_api @classmethod def prod(cls): return cls( GitApi(), + ArtifactDeploymentApi(), ) -# def __get_base_artifact_release_url(self, base_url: str, organization: str, name: str) -> str: -# return f"{base_url}/{organization}/{name}/releases" + def __get_base_artifact_release_url(self, base_url: str, organization: str, name: str) -> str: + return f"{base_url}/{organization}/{name}/releases" + + def post_release(self, target_url: str, tag: str, credentials: Credentials): + response = self.artifact_deployment_api.post_release(target_url, tag, credentials.mappings["token"]) + # Get release id from response + + def post_asset(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): + self.artifact_deployment_api.post_asset(target_url, release_id, attachment, attachment_type, token) + + def calculate_checksums(self, build_path: str): + self.artifact_deployment_api.calculate_checksums(build_path) From 911158f882d8099aefb341ec9b07e85183657a2c Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 4 Aug 2023 12:08:23 +0200 Subject: [PATCH 08/57] Remove __get_base_artifact_release_url() --- .../ddadevops/application/artifact_deployment_service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/application/artifact_deployment_service.py b/src/main/python/ddadevops/application/artifact_deployment_service.py index e7e3c61..a53a9cf 100644 --- a/src/main/python/ddadevops/application/artifact_deployment_service.py +++ b/src/main/python/ddadevops/application/artifact_deployment_service.py @@ -12,9 +12,6 @@ class ArtifactDeploymentService: GitApi(), ArtifactDeploymentApi(), ) - - def __get_base_artifact_release_url(self, base_url: str, organization: str, name: str) -> str: - return f"{base_url}/{organization}/{name}/releases" def post_release(self, target_url: str, tag: str, credentials: Credentials): response = self.artifact_deployment_api.post_release(target_url, tag, credentials.mappings["token"]) @@ -25,3 +22,5 @@ class ArtifactDeploymentService: def calculate_checksums(self, build_path: str): self.artifact_deployment_api.calculate_checksums(build_path) + + # ToDo: Update release Id as in release_mixin_services.py From 0952ec57a87c07bbeade33cc43dd1fad816c74a1 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 4 Aug 2023 12:09:09 +0200 Subject: [PATCH 09/57] [Skip-CI] Add initial artifact deployment domain object --- .../ddadevops/domain/artifact_deployment.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/main/python/ddadevops/domain/artifact_deployment.py diff --git a/src/main/python/ddadevops/domain/artifact_deployment.py b/src/main/python/ddadevops/domain/artifact_deployment.py new file mode 100644 index 0000000..64905cc --- /dev/null +++ b/src/main/python/ddadevops/domain/artifact_deployment.py @@ -0,0 +1,40 @@ +from typing import List, Dict, Optional +from .common import ( + Validateable, + CredentialMappingDefault, + DnsRecord, + Devops, +) + + +class ArtifactDeployment(Validateable, CredentialMappingDefault): + def __init__(self, inp: dict): + self.name = inp.get("name") + self.artifact_base_url = inp.get("artifact_base_url") + self.organization = inp.get("organization") + + def get_artifact_release_url(self) -> str: + return f"{self.artifact_base_url}/{self.organization}/{self.name}/releases" + + def get_artifact_asset_url(self, release_id: str) -> str: + return f"{self.get_artifact_release_url}/{release_id}/assets" + + def update_runtime_config(self, dns_record: DnsRecord): + self.dns_record = dns_record + self.throw_if_invalid() + + def validate(self) -> List[str]: + result = [] + result += self.__validate_is_not_empty__("name") + result += self.__validate_is_not_empty__("artifact_base_url") + return result + + @classmethod + def get_mapping_default(cls) -> List[Dict[str, str]]: + return [ # ToDo: Adapt for token + { + "gopass_path": "server/meissa/grafana-cloud", + "gopass_field": "grafana-cloud-user", + "name": "c4k_grafana_cloud_user", + } + ] From 2fc59f105b90fc8205cc1ee905983de544626443 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 14:08:46 +0200 Subject: [PATCH 10/57] artifact publish ist part of release now --- build.py | 1 + .../python/ddadevops/application/release_mixin_services.py | 6 ++++++ src/main/python/ddadevops/release_mixin.py | 5 +++++ 3 files changed, 12 insertions(+) diff --git a/build.py b/build.py index a1fddfb..c0377b4 100644 --- a/build.py +++ b/build.py @@ -176,6 +176,7 @@ def prepare(project): def tag(project): build = get_devops_build(project) build.tag_bump_and_push_release() + #TODO: build.publish_artifacts() def release(project): diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index c9ab06e..f686c3f 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -53,6 +53,12 @@ class ReleaseService: ) self.git_api.push_follow_tags() + def publish_artifacts(self, release: Release): + # calculate checksum + # create release + # add artifacts to release + pass + def __set_version_and_commit__( self, version: Version, build_file_ids: List[str], message: str ): diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 3d7defd..5ece524 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -26,3 +26,8 @@ class ReleaseMixin(DevopsBuild): devops = self.devops_repo.get_devops(self.project) release = devops.mixins[MixinType.RELEASE] self.release_service.tag_bump_and_push_release(release) + + def publish_artifacts(self): + devops = self.devops_repo.get_devops(self.project) + release = devops.mixins[MixinType.RELEASE] + self.release_service.publish_artifacts(release) From 88253f49ac3e587710ebf6c91e564172447e99b2 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 14:20:27 +0200 Subject: [PATCH 11/57] use new api in release-service --- .../application/release_mixin_services.py | 11 +++++++++-- .../python/ddadevops/infrastructure/__init__.py | 1 + .../application/test_release_mixin_services.py | 3 ++- src/test/python/domain/helper.py | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index f686c3f..f9aeb53 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,18 +1,25 @@ from typing import List from pathlib import Path -from ..infrastructure import GitApi, BuildFileRepository +from ..infrastructure import GitApi, ArtifactDeploymentApi, BuildFileRepository from ..domain import Version, Release, ReleaseType class ReleaseService: - def __init__(self, git_api: GitApi, build_file_repository: BuildFileRepository): + def __init__( + self, + git_api: GitApi, + build_file_repository: BuildFileRepository, + artifact_deployment_api: ArtifactDeploymentApi, + ): self.git_api = git_api + self.artifact_deployment_api = artifact_deployment_api self.build_file_repository = build_file_repository @classmethod def prod(cls, base_dir: str): return cls( GitApi(), + ArtifactDeploymentApi(), BuildFileRepository(base_dir), ) diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index 1bb11b5..1a520ed 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -7,5 +7,6 @@ from .infrastructure import ( CredentialsApi, GitApi, TerraformApi, + ArtifactDeploymentApi, ) from .repository import DevopsRepository, BuildFileRepository diff --git a/src/test/python/application/test_release_mixin_services.py b/src/test/python/application/test_release_mixin_services.py index 40f6553..b6fbe02 100644 --- a/src/test/python/application/test_release_mixin_services.py +++ b/src/test/python/application/test_release_mixin_services.py @@ -7,12 +7,13 @@ from src.main.python.ddadevops.domain import ( from src.test.python.domain.helper import ( BuildFileRepositoryMock, GitApiMock, + ArtifactDeploymentApiMock, build_devops, ) from src.main.python.ddadevops.application import ReleaseService def test_sould_update_release_type(): - sut = ReleaseService(GitApiMock(), BuildFileRepositoryMock("build.py")) + sut = ReleaseService(GitApiMock(), ArtifactDeploymentApiMock(), BuildFileRepositoryMock("build.py")) devops = build_devops({}) release = devops.mixins[MixinType.RELEASE] sut.update_release_type(release, "MAJOR") diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 05161d0..74a67d5 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -150,3 +150,17 @@ class GitApiMock: def checkout(self, branch: str): pass + + +class ArtifactDeploymentApiMock: + def __init__(self): + pass + + def post_release(self, target_url: str, tag: str, token: str): + pass + + def post_asset(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): + pass + + def calculate_checksums(self, build_path: str): + pass From 074e77196e320f74a9ec48d83b4c940f2f9c2cc8 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 14:24:11 +0200 Subject: [PATCH 12/57] wip calculate-sha [skip-ci] --- src/main/python/ddadevops/application/release_mixin_services.py | 2 +- src/main/python/ddadevops/infrastructure/infrastructure.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index f9aeb53..84c5533 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -61,7 +61,7 @@ class ReleaseService: self.git_api.push_follow_tags() def publish_artifacts(self, release: Release): - # calculate checksum + self.artifact_deployment_api.calculate_checksums(artifact_path=) # create release # add artifacts to release pass diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 40fee1b..878581c 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -231,6 +231,6 @@ class ArtifactDeploymentApi: + '-H "Content-Type: multipart/form-data" ' + f'-F "attachment=@{attachment};type={attachment_type}"') - def calculate_checksums(self, build_path: str): + def calculate_checksums(self, artifact_path: str): self.execution_api.execute(f"find {build_path} -type f -exec sha256sum {{}}; | sort > {build_path} sha256sum.lst") self.execution_api.execute(f"find {build_path} -type f -exec sha512sum {{}}; | sort > {build_path} sha512sum.lst") From 369f62ff38a3b242a117d2c5d5223e888d2b069c Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 11 Aug 2023 14:25:12 +0200 Subject: [PATCH 13/57] Correct var names --- src/main/python/ddadevops/infrastructure/infrastructure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 878581c..44054bb 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -232,5 +232,5 @@ class ArtifactDeploymentApi: + f'-F "attachment=@{attachment};type={attachment_type}"') def calculate_checksums(self, artifact_path: str): - self.execution_api.execute(f"find {build_path} -type f -exec sha256sum {{}}; | sort > {build_path} sha256sum.lst") - self.execution_api.execute(f"find {build_path} -type f -exec sha512sum {{}}; | sort > {build_path} sha512sum.lst") + self.execution_api.execute(f"find {artifact_path} -type f -exec sha256sum {{}}; | sort > {artifact_path} sha256sum.lst") + self.execution_api.execute(f"find {artifact_path} -type f -exec sha512sum {{}}; | sort > {artifact_path} sha512sum.lst") From 935baa9932a41c6e985ab9c1ad7ad276f002c041 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 11 Aug 2023 14:35:52 +0200 Subject: [PATCH 14/57] Simpler sha calc command --- src/main/python/ddadevops/infrastructure/infrastructure.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 44054bb..5a7ab9a 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -232,5 +232,6 @@ class ArtifactDeploymentApi: + f'-F "attachment=@{attachment};type={attachment_type}"') def calculate_checksums(self, artifact_path: str): - self.execution_api.execute(f"find {artifact_path} -type f -exec sha256sum {{}}; | sort > {artifact_path} sha256sum.lst") - self.execution_api.execute(f"find {artifact_path} -type f -exec sha512sum {{}}; | sort > {artifact_path} sha512sum.lst") + # self.execution_api.execute(f"find {artifact_path} -type f -exec sha256sum {{}}; | sort > {artifact_path} sha256sum.lst") relevant für provs + self.execution_api(f"sha256sum {artifact_path} > {artifact_path}.sha256",) + self.execution_api(f"sha512sum {artifact_path} > {artifact_path}.sha512",) From 337a7900444f8486387ed23cb1d33dfa0819367d Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 11 Aug 2023 14:38:41 +0200 Subject: [PATCH 15/57] Implement checksum calculation --- doc/architecture/Domain.md | 1 + .../python/ddadevops/application/release_mixin_services.py | 3 ++- src/main/python/ddadevops/domain/release.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index afb8378..a865a1e 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -88,6 +88,7 @@ classDiagram release_type release_main_branch release_current_branch + release_artifacts } class Credentials { <> diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 84c5533..6b5624b 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -61,7 +61,8 @@ class ReleaseService: self.git_api.push_follow_tags() def publish_artifacts(self, release: Release): - self.artifact_deployment_api.calculate_checksums(artifact_path=) + for artifact_path in release.release_artifacts: + self.artifact_deployment_api.calculate_checksums(artifact_path) # create release # add artifacts to release pass diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index e8a6db5..ae3720b 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -17,6 +17,9 @@ class Release(Validateable): self.release_primary_build_file = inp.get( "release_primary_build_file", "./project.clj" ) + self.release_artifacts = inp.get( + "release_artifacts", [] + ) self.release_secondary_build_files = inp.get( "release_secondary_build_files", [] ) From 39591c8aa99332364e2a055b107a08c07974f3ef Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 11 Aug 2023 14:45:43 +0200 Subject: [PATCH 16/57] [Skip-CI] WIP Implement release publishing --- src/main/python/ddadevops/application/release_mixin_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 6b5624b..b15f260 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -63,7 +63,7 @@ class ReleaseService: def publish_artifacts(self, release: Release): for artifact_path in release.release_artifacts: self.artifact_deployment_api.calculate_checksums(artifact_path) - # create release + self.artifact_deployment_api.post_release(release) # create release # add artifacts to release pass From 2469fd2f5bacfc944df9463c4869a92091cbfb29 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 11 Aug 2023 14:46:25 +0200 Subject: [PATCH 17/57] [Skip-CI] Rename function --- src/main/python/ddadevops/infrastructure/infrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 5a7ab9a..0a41221 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -212,7 +212,7 @@ class ArtifactDeploymentApi: def __init__(self): self.execution_api = ExecutionApi() - def post_release(self, target_url: str, tag: str, token: str): + def create_forgejo_release(self, target_url: str, tag: str, token: str): return self.execution_api.execute_secure(f'curl -X "POST" "{target_url}" ' + '-H "accept: application/json" -H "Content-Type: application/json" ' + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501 From 5f5743354ca34865be0228b1980c9e5566a64820 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 15:08:21 +0200 Subject: [PATCH 18/57] wip: calculate forgejo_release_api_endpoint --- src/main/python/ddadevops/domain/release.py | 8 ++++ src/test/python/domain/helper.py | 4 ++ src/test/python/domain/test_release.py | 46 +++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index ae3720b..b8c64ec 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -24,6 +24,9 @@ class Release(Validateable): "release_secondary_build_files", [] ) self.version = version + self.release_artifact_server_url = inp.get("release_artifact_server_url") + self.release_organisation = inp.get("release_organisation") + self.release_repository_name = inp.get("release_repository_name") def update_release_type(self, release_type: ReleaseType): self.release_type = release_type @@ -63,3 +66,8 @@ class Release(Validateable): result = [self.release_primary_build_file] result += self.release_secondary_build_files return result + + def forgejo_release_api_endpoint(self): + if self.release_artifact_server_url == None or self.release_organisation == None or self.release_repository_name == None: + raise RuntimeError("when doing artifact release, release_artifact_server_url, release_organisation, release_repository_name may not be None.") + return f"{self.release_artifact_server_url}/api/v1/repos/{self.release_organisation}/{self.release_repository_name}/releases" diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 74a67d5..31a8911 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -53,6 +53,10 @@ def devops_config(overrides: dict) -> dict: "release_current_branch": "my_feature", "release_primary_build_file": "./package.json", "release_secondary_build_file": [], + "release_artifacts": [], + "release_artifact_server_url": None, + "release_organisation": None, + "release_repository_name": None, "credentials_mappings": [ { "gopass_path": "a/path", diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index c23bc39..48b520a 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -1,3 +1,4 @@ +import pytest from pybuilder.core import Project from pathlib import Path from src.main.python.ddadevops.domain import ( @@ -61,3 +62,48 @@ def test_sould_calculate_build_files(): Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), ) assert ["project.clj", "package.json"] == sut.build_files() + + +def test_should_calculate_forgejo_release_api_endpoint(): + sut = Release( + devops_config( + { + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + assert ( + "https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases" + == sut.forgejo_release_api_endpoint() + ) + + sut = Release( + devops_config( + { + "release_artifact_server_url": "https://repo.prod.meissa.de/", + "release_organisation": "/meissa/", + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + assert ( + "https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases" + == sut.forgejo_release_api_endpoint() + ) + + with pytest.raises(Exception): + sut = Release( + devops_config( + { + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": None, + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + sut.forgejo_release_api_endpoint() From f4be6e0c8ba9fd39d18610b213913d9fa7306703 Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 11 Aug 2023 15:26:59 +0200 Subject: [PATCH 19/57] Implement calculation of forgejo release api endpoint url --- .../python/ddadevops/application/release_mixin_services.py | 2 +- src/main/python/ddadevops/domain/release.py | 6 +++++- src/test/python/domain/test_release.py | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index b15f260..19e167e 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -63,7 +63,7 @@ class ReleaseService: def publish_artifacts(self, release: Release): for artifact_path in release.release_artifacts: self.artifact_deployment_api.calculate_checksums(artifact_path) - self.artifact_deployment_api.post_release(release) # create release + self.artifact_deployment_api.create_forgejo_release(release.forgejo_release_api_endpoint) # create release # add artifacts to release pass diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index b8c64ec..7679ff8 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -70,4 +70,8 @@ class Release(Validateable): def forgejo_release_api_endpoint(self): if self.release_artifact_server_url == None or self.release_organisation == None or self.release_repository_name == None: raise RuntimeError("when doing artifact release, release_artifact_server_url, release_organisation, release_repository_name may not be None.") - return f"{self.release_artifact_server_url}/api/v1/repos/{self.release_organisation}/{self.release_repository_name}/releases" + + server_url = self.release_artifact_server_url.removeprefix("/").removesuffix("/") + organisation = self.release_organisation.removeprefix("/").removesuffix("/") + repository = self.release_repository_name.removeprefix("/").removesuffix("/") + return f"{server_url}/api/v1/repos/{organisation}/{repository}/releases" diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index 48b520a..ffd1ffb 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -94,6 +94,10 @@ def test_should_calculate_forgejo_release_api_endpoint(): "https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases" == sut.forgejo_release_api_endpoint() ) + assert( + "/meissa/" + == sut.release_organisation + ) with pytest.raises(Exception): sut = Release( From db8b41be19983632ec845a04bb6b156c0d36a02f Mon Sep 17 00:00:00 2001 From: erik Date: Fri, 11 Aug 2023 15:29:07 +0200 Subject: [PATCH 20/57] Add comments for moving functionality --- .../python/ddadevops/application/artifact_deployment_service.py | 1 + src/main/python/ddadevops/artifact_deployment_mixin.py | 2 +- src/main/python/ddadevops/domain/artifact_deployment.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/application/artifact_deployment_service.py b/src/main/python/ddadevops/application/artifact_deployment_service.py index a53a9cf..3d8a3b4 100644 --- a/src/main/python/ddadevops/application/artifact_deployment_service.py +++ b/src/main/python/ddadevops/application/artifact_deployment_service.py @@ -1,6 +1,7 @@ from ..infrastructure import GitApi, ArtifactDeploymentApi from ..domain import Credentials +# This will be moved to release mixin class ArtifactDeploymentService: def __init__(self, git_api: GitApi, artifact_deployment_api: ArtifactDeploymentApi): self.git_api = git_api diff --git a/src/main/python/ddadevops/artifact_deployment_mixin.py b/src/main/python/ddadevops/artifact_deployment_mixin.py index d48e91c..77a373c 100644 --- a/src/main/python/ddadevops/artifact_deployment_mixin.py +++ b/src/main/python/ddadevops/artifact_deployment_mixin.py @@ -51,7 +51,7 @@ from .domain import MixinType # """ - +# This will be moved to release mixin class ArtifactDeploymentMixin(DevopsBuild): def __init__(self, project: Project, inp: dict): super().__init__(project, inp) diff --git a/src/main/python/ddadevops/domain/artifact_deployment.py b/src/main/python/ddadevops/domain/artifact_deployment.py index 64905cc..4d0976f 100644 --- a/src/main/python/ddadevops/domain/artifact_deployment.py +++ b/src/main/python/ddadevops/domain/artifact_deployment.py @@ -6,7 +6,7 @@ from .common import ( Devops, ) - +# This will be moved to release mixin class ArtifactDeployment(Validateable, CredentialMappingDefault): def __init__(self, inp: dict): self.name = inp.get("name") From cb41ad0719f1816cadc9deffa87653746bcceb2a Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 15:37:07 +0200 Subject: [PATCH 21/57] update domain doc --- doc/architecture/Domain.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index a865a1e..1f4bb88 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -89,6 +89,9 @@ classDiagram release_main_branch release_current_branch release_artifacts + release_artifact_server_url + release_organisation + release_repository_name } class Credentials { <> From 5d2596bd3f33a0eaa636a8e448cfdd9897a6901d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 15:49:25 +0200 Subject: [PATCH 22/57] improve some linting --- .../application/release_mixin_services.py | 7 +- src/main/python/ddadevops/domain/release.py | 25 ++++-- .../infrastructure/infrastructure.py | 77 ++++++++++++------- 3 files changed, 71 insertions(+), 38 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 19e167e..db0a8f8 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -8,8 +8,8 @@ class ReleaseService: def __init__( self, git_api: GitApi, - build_file_repository: BuildFileRepository, artifact_deployment_api: ArtifactDeploymentApi, + build_file_repository: BuildFileRepository, ): self.git_api = git_api self.artifact_deployment_api = artifact_deployment_api @@ -63,7 +63,10 @@ class ReleaseService: def publish_artifacts(self, release: Release): for artifact_path in release.release_artifacts: self.artifact_deployment_api.calculate_checksums(artifact_path) - self.artifact_deployment_api.create_forgejo_release(release.forgejo_release_api_endpoint) # create release + self.artifact_deployment_api.create_forgejo_release( + release.forgejo_release_api_endpoint(), + ) + # create release # add artifacts to release pass diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 7679ff8..8d50eec 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -17,9 +17,7 @@ class Release(Validateable): self.release_primary_build_file = inp.get( "release_primary_build_file", "./project.clj" ) - self.release_artifacts = inp.get( - "release_artifacts", [] - ) + self.release_artifacts = inp.get("release_artifacts", []) self.release_secondary_build_files = inp.get( "release_secondary_build_files", [] ) @@ -59,7 +57,9 @@ class Release(Validateable): and self.release_type != ReleaseType.NONE and self.release_main_branch != self.release_current_branch ): - result.append(f"Releases are allowed only on {self.release_main_branch}") + result.append( + f"Releases are allowed only on {self.release_main_branch}" + ) return result def build_files(self) -> List[str]: @@ -68,10 +68,19 @@ class Release(Validateable): return result def forgejo_release_api_endpoint(self): - if self.release_artifact_server_url == None or self.release_organisation == None or self.release_repository_name == None: - raise RuntimeError("when doing artifact release, release_artifact_server_url, release_organisation, release_repository_name may not be None.") - - server_url = self.release_artifact_server_url.removeprefix("/").removesuffix("/") + if ( + self.release_artifact_server_url is None + or self.release_organisation is None + or self.release_repository_name is None + ): + raise RuntimeError( + "when doing artifact release, release_artifact_server_url, "+ + "release_organisation, release_repository_name may not be None." + ) + + server_url = self.release_artifact_server_url.removeprefix("/").removesuffix( + "/" + ) organisation = self.release_organisation.removeprefix("/").removesuffix("/") repository = self.release_repository_name.removeprefix("/").removesuffix("/") return f"{server_url}/api/v1/repos/{organisation}/{repository}/releases" diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 0a41221..5aaf88e 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -65,16 +65,12 @@ class ImageApi: def dockerhub_login(self, username: str, password: str): self.execution_api.execute_secure( f"docker login --username {username} --password {password}", - "docker login --username ***** --password *****" + "docker login --username ***** --password *****", ) def dockerhub_publish(self, name: str, username: str, tag: str): - self.execution_api.execute_live( - f"docker tag {name} {username}/{name}:{tag}" - ) - self.execution_api.execute_live( - f"docker push {username}/{name}:{tag}" - ) + self.execution_api.execute_live(f"docker tag {name} {username}/{name}:{tag}") + self.execution_api.execute_live(f"docker push {username}/{name}:{tag}") def test(self, name: str, path: Path): self.execution_api.execute_live( @@ -95,14 +91,24 @@ class ExecutionApi: check=check, stdout=PIPE, stderr=PIPE, - text=True).stdout + text=True, + ).stdout output = output.rstrip() except CalledProcessError as exc: - print(f"Command failed with code: {exc.returncode} and message: {exc.stderr}") + print( + f"Command failed with code: {exc.returncode} and message: {exc.stderr}" + ) raise exc return output - def execute_secure(self, command: str, sanitized_command: str, dry_run=False, shell=True, check=True): + def execute_secure( + self, + command: str, + sanitized_command: str, + dry_run=False, + shell=True, + check=True, + ): try: output = self.execute(command, dry_run, shell, check) return output @@ -213,25 +219,40 @@ class ArtifactDeploymentApi: self.execution_api = ExecutionApi() def create_forgejo_release(self, target_url: str, tag: str, token: str): - return self.execution_api.execute_secure(f'curl -X "POST" "{target_url}" ' - + '-H "accept: application/json" -H "Content-Type: application/json" ' - + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501 - + f'-H "Authorization: token {token}"', - sanitized_command=f'curl -X "POST" "{target_url}" ' - + '-H "accept: application/json" -H "Content-Type: application/json" ' - + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ') # noqa: E501 + return self.execution_api.execute_secure( + f'curl -X "POST" "{target_url}" ' + + '-H "accept: application/json" -H "Content-Type: application/json" ' + + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501 + + f'-H "Authorization: token {token}"', + sanitized_command=f'curl -X "POST" "{target_url}" ' + + '-H "accept: application/json" -H "Content-Type: application/json" ' + + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ', + ) # noqa: E501 - def post_asset(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): - return self.execution_api.execute_secure(f'curl -X "POST" "{target_url}/{release_id}/assets" ' # {target_url}/{release_id}/assets move to Domain - + f'-H "accept: application/json" -H "Authorization: token {token}" ' - + '-H "Content-Type: multipart/form-data" ' - + f'-F "attachment=@{attachment};type={attachment_type}"', - sanitized_command=f'curl -X "POST" "{target_url}/{release_id}/assets" ' # see above - + '-H "accept: application/json" ' - + '-H "Content-Type: multipart/form-data" ' - + f'-F "attachment=@{attachment};type={attachment_type}"') + def post_asset( + self, + target_url: str, + release_id: str, + attachment: str, + attachment_type: str, + token: str, + ): + return self.execution_api.execute_secure( + f'curl -X "POST" "{target_url}/{release_id}/assets" ' # {target_url}/{release_id}/assets move to Domain + + f'-H "accept: application/json" -H "Authorization: token {token}" ' + + '-H "Content-Type: multipart/form-data" ' + + f'-F "attachment=@{attachment};type={attachment_type}"', + sanitized_command=f'curl -X "POST" "{target_url}/{release_id}/assets" ' # see above + + '-H "accept: application/json" ' + + '-H "Content-Type: multipart/form-data" ' + + f'-F "attachment=@{attachment};type={attachment_type}"', + ) def calculate_checksums(self, artifact_path: str): # self.execution_api.execute(f"find {artifact_path} -type f -exec sha256sum {{}}; | sort > {artifact_path} sha256sum.lst") relevant für provs - self.execution_api(f"sha256sum {artifact_path} > {artifact_path}.sha256",) - self.execution_api(f"sha512sum {artifact_path} > {artifact_path}.sha512",) + self.execution_api( + f"sha256sum {artifact_path} > {artifact_path}.sha256", + ) + self.execution_api( + f"sha512sum {artifact_path} > {artifact_path}.sha512", + ) From 3cfb453454f7d1df32f3d1136b522dca7abefe0c Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 15:54:15 +0200 Subject: [PATCH 23/57] improve some linting --- src/main/python/ddadevops/domain/release.py | 4 ++-- src/main/python/ddadevops/infrastructure/infrastructure.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 8d50eec..b28e5da 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -74,8 +74,8 @@ class Release(Validateable): or self.release_repository_name is None ): raise RuntimeError( - "when doing artifact release, release_artifact_server_url, "+ - "release_organisation, release_repository_name may not be None." + "when doing artifact release, release_artifact_server_url, " + + "release_organisation, release_repository_name may not be None." ) server_url = self.release_artifact_server_url.removeprefix("/").removesuffix( diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 5aaf88e..1da56bb 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -249,7 +249,9 @@ class ArtifactDeploymentApi: ) def calculate_checksums(self, artifact_path: str): - # self.execution_api.execute(f"find {artifact_path} -type f -exec sha256sum {{}}; | sort > {artifact_path} sha256sum.lst") relevant für provs + # self.execution_api.execute(f"find {artifact_path} -type f + # -exec sha256sum {{}}; | sort > {artifact_path} sha256sum.lst") + # relevant für provs self.execution_api( f"sha256sum {artifact_path} > {artifact_path}.sha256", ) From 2217e5c8d1e5d2b2d64818cb9b5177d2e1ab7fa8 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 16:16:03 +0200 Subject: [PATCH 24/57] add token & improve artifact validation --- src/main/python/ddadevops/domain/release.py | 33 ++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index b28e5da..29f78c1 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import Optional, List, Dict from pathlib import Path from .common import ( Validateable, @@ -25,6 +25,7 @@ class Release(Validateable): self.release_artifact_server_url = inp.get("release_artifact_server_url") self.release_organisation = inp.get("release_organisation") self.release_repository_name = inp.get("release_repository_name") + self.release_artifact_token = inp.get("release_artifact_token") def update_release_type(self, release_type: ReleaseType): self.release_type = release_type @@ -62,21 +63,24 @@ class Release(Validateable): ) return result + def validate_for_artifact(self): + result = [] + result += self.__validate_is_not_empty__("release_artifact_server_url") + result += self.__validate_is_not_empty__("release_organisation") + result += self.__validate_is_not_empty__("release_repository_name") + result += self.__validate_is_not_empty__("release_artifacts") + result += self.__validate_is_not_empty__("release_artifact_token") + return result + def build_files(self) -> List[str]: result = [self.release_primary_build_file] result += self.release_secondary_build_files return result def forgejo_release_api_endpoint(self): - if ( - self.release_artifact_server_url is None - or self.release_organisation is None - or self.release_repository_name is None - ): - raise RuntimeError( - "when doing artifact release, release_artifact_server_url, " - + "release_organisation, release_repository_name may not be None." - ) + validation = self.validate_for_artifact() + if validation != []: + raise RuntimeError(f"not valid for creating artifacts: {validation}") server_url = self.release_artifact_server_url.removeprefix("/").removesuffix( "/" @@ -84,3 +88,12 @@ class Release(Validateable): organisation = self.release_organisation.removeprefix("/").removesuffix("/") repository = self.release_repository_name.removeprefix("/").removesuffix("/") return f"{server_url}/api/v1/repos/{organisation}/{repository}/releases" + + @classmethod + def get_mapping_default(cls) -> List[Dict[str, str]]: + return [ + { + "gopass_path": "server/meissa/repo/buero-rw", + "name": "release_artifact_token", + } + ] From f30609288a412be1f2e121f7d073c3ebc7b34671 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 16:16:25 +0200 Subject: [PATCH 25/57] add token & improve artifact validation --- src/test/python/domain/test_release.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index ffd1ffb..42f8699 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -68,6 +68,8 @@ def test_should_calculate_forgejo_release_api_endpoint(): sut = Release( devops_config( { + "release_artifacts": ["x"], + "release_artifact_token": "y", "release_artifact_server_url": "https://repo.prod.meissa.de", "release_organisation": "meissa", "release_repository_name": "provs", @@ -83,6 +85,8 @@ def test_should_calculate_forgejo_release_api_endpoint(): sut = Release( devops_config( { + "release_artifacts": ["x"], + "release_artifact_token": "y", "release_artifact_server_url": "https://repo.prod.meissa.de/", "release_organisation": "/meissa/", "release_repository_name": "provs", From ee58151a8df2a900ae1161211175320b7d6ae711 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 16:17:07 +0200 Subject: [PATCH 26/57] initialize default credentials --- src/main/python/ddadevops/domain/init_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 2d5ab56..1afa1f3 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -8,7 +8,7 @@ from .provider_digitalocean import Digitalocean from .provider_hetzner import Hetzner from .c4k import C4k from .image import Image -from .release import ReleaseType +from .release import ReleaseType, Release from ..infrastructure import BuildFileRepository, CredentialsApi, EnvironmentApi, GitApi @@ -69,6 +69,7 @@ class InitService: Path(primary_build_file_id) ) version = primary_build_file.get_version() + default_mappings += Release.get_mapping_default() credentials = Credentials(inp, default_mappings) authorization = self.authorization(credentials) From 700a0a2f4f6d4e09991c2f2db9fef444dc415cc8 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Fri, 11 Aug 2023 16:17:42 +0200 Subject: [PATCH 27/57] add missing informations for creating release --- src/main/python/ddadevops/application/release_mixin_services.py | 2 ++ src/test/python/domain/helper.py | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index db0a8f8..5e657cc 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -65,6 +65,8 @@ class ReleaseService: self.artifact_deployment_api.calculate_checksums(artifact_path) self.artifact_deployment_api.create_forgejo_release( release.forgejo_release_api_endpoint(), + release.version.to_string(), + release.release_artifact_token ) # create release # add artifacts to release diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 31a8911..5311703 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -54,6 +54,7 @@ def devops_config(overrides: dict) -> dict: "release_primary_build_file": "./package.json", "release_secondary_build_file": [], "release_artifacts": [], + "release_artifact_token": "release_artifact_token", "release_artifact_server_url": None, "release_organisation": None, "release_repository_name": None, From b861087e9db33f7067b45cc395f1a397d8c73fc6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 08:24:42 +0200 Subject: [PATCH 28/57] fix test no longer needs gopass --- src/test/python/test_release_mixin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py index 291eb73..a2539a6 100644 --- a/src/test/python/test_release_mixin.py +++ b/src/test/python/test_release_mixin.py @@ -14,6 +14,8 @@ def test_release_mixin(tmp_path): copy_resource(Path("package.json"), tmp_path) project = Project(str_tmp_path, name="name") + os.environ["RELEASE_ARTIFACT_TOKEN"] = "ratoken" + sut = ReleaseMixin( project, devops_config( From e0150e6fcced43e06c9739b11632524dce757b3f Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 08:59:45 +0200 Subject: [PATCH 29/57] parse the release_id from create response --- .../application/release_mixin_services.py | 23 ++++++++++++------- .../infrastructure/infrastructure.py | 2 +- .../test_release_mixin_services.py | 23 +++++++++++++++++-- src/test/python/domain/helper.py | 16 +++++++++---- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 5e657cc..2b094a8 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,3 +1,4 @@ +import json from typing import List from pathlib import Path from ..infrastructure import GitApi, ArtifactDeploymentApi, BuildFileRepository @@ -61,16 +62,22 @@ class ReleaseService: self.git_api.push_follow_tags() def publish_artifacts(self, release: Release): + release_response = json.loads( + self.artifact_deployment_api.create_forgejo_release( + release.forgejo_release_api_endpoint(), + release.version.to_string(), + release.release_artifact_token, + ) + ) for artifact_path in release.release_artifacts: self.artifact_deployment_api.calculate_checksums(artifact_path) - self.artifact_deployment_api.create_forgejo_release( - release.forgejo_release_api_endpoint(), - release.version.to_string(), - release.release_artifact_token - ) - # create release - # add artifacts to release - pass + self.artifact_deployment_api.add_asset_to_release( + "todo", + release_response["id"], + "todo", + "todo", + release.release_artifact_token, + ) def __set_version_and_commit__( self, version: Version, build_file_ids: List[str], message: str diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 1da56bb..cd5f946 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -229,7 +229,7 @@ class ArtifactDeploymentApi: + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ', ) # noqa: E501 - def post_asset( + def add_asset_to_release( self, target_url: str, release_id: str, diff --git a/src/test/python/application/test_release_mixin_services.py b/src/test/python/application/test_release_mixin_services.py index b6fbe02..8e1d786 100644 --- a/src/test/python/application/test_release_mixin_services.py +++ b/src/test/python/application/test_release_mixin_services.py @@ -1,7 +1,7 @@ import pytest from pathlib import Path from src.main.python.ddadevops.domain import ( - ReleaseType, + ReleaseType, MixinType, ) from src.test.python.domain.helper import ( @@ -12,8 +12,11 @@ from src.test.python.domain.helper import ( ) from src.main.python.ddadevops.application import ReleaseService + def test_sould_update_release_type(): - sut = ReleaseService(GitApiMock(), ArtifactDeploymentApiMock(), BuildFileRepositoryMock("build.py")) + sut = ReleaseService( + GitApiMock(), ArtifactDeploymentApiMock(), BuildFileRepositoryMock("build.py") + ) devops = build_devops({}) release = devops.mixins[MixinType.RELEASE] sut.update_release_type(release, "MAJOR") @@ -21,3 +24,19 @@ def test_sould_update_release_type(): with pytest.raises(Exception): sut.update_release_type(release, "NOT_EXISTING") + + +def test_sould_publish_artifacts(): + mock = ArtifactDeploymentApiMock(release='{"id": 2345}') + sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock()) + devops = build_devops( + { + "release_artifacts": ["target/art"], + "release_artifact_server_url": "http://repo.test/", + "release_organisation": "orga", + "release_repository_name": "repo", + } + ) + release = devops.mixins[MixinType.RELEASE] + sut.publish_artifacts(release) + assert 2345 == mock.add_asset_to_release_id diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 5311703..321e6a7 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -158,13 +158,19 @@ class GitApiMock: class ArtifactDeploymentApiMock: - def __init__(self): - pass + def __init__(self, release=""): + self.release = release + self.create_forgejo_release_count = 0 + self.add_asset_to_release_count = 0 + self.add_asset_to_release_id = "" - def post_release(self, target_url: str, tag: str, token: str): - pass + def create_forgejo_release(self, target_url: str, tag: str, token: str): + self.create_forgejo_release_count += 1 + return self.release - def post_asset(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): + def add_asset_to_release(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): + self.add_asset_to_release_count += 1 + self.add_asset_to_release_id = release_id pass def calculate_checksums(self, build_path: str): From a876fdc799b89938f6ae3d559b645992bb4d7478 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 09:10:31 +0200 Subject: [PATCH 30/57] add error handling for release_id parsing --- .../application/release_mixin_services.py | 8 +++++-- .../infrastructure/infrastructure.py | 6 +++--- .../test_release_mixin_services.py | 21 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 2b094a8..546d93e 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -62,7 +62,7 @@ class ReleaseService: self.git_api.push_follow_tags() def publish_artifacts(self, release: Release): - release_response = json.loads( + release_id = self.__parse_forgejo_release_id__( self.artifact_deployment_api.create_forgejo_release( release.forgejo_release_api_endpoint(), release.version.to_string(), @@ -73,12 +73,16 @@ class ReleaseService: self.artifact_deployment_api.calculate_checksums(artifact_path) self.artifact_deployment_api.add_asset_to_release( "todo", - release_response["id"], + release_id, "todo", "todo", release.release_artifact_token, ) + def __parse_forgejo_release_id__(self, release_response: str) -> int: + parsed = json.loads(release_response) + return parsed["id"] + def __set_version_and_commit__( self, version: Version, build_file_ids: List[str], message: str ): diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index cd5f946..d86e029 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -218,13 +218,13 @@ class ArtifactDeploymentApi: def __init__(self): self.execution_api = ExecutionApi() - def create_forgejo_release(self, target_url: str, tag: str, token: str): + def create_forgejo_release(self, api_endpoint_url: str, tag: str, token: str): return self.execution_api.execute_secure( - f'curl -X "POST" "{target_url}" ' + f'curl -X "POST" "{api_endpoint_url}" ' + '-H "accept: application/json" -H "Content-Type: application/json" ' + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501 + f'-H "Authorization: token {token}"', - sanitized_command=f'curl -X "POST" "{target_url}" ' + sanitized_command=f'curl -X "POST" "{api_endpoint_url}" ' + '-H "accept: application/json" -H "Content-Type: application/json" ' + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ', ) # noqa: E501 diff --git a/src/test/python/application/test_release_mixin_services.py b/src/test/python/application/test_release_mixin_services.py index 8e1d786..56f9ddf 100644 --- a/src/test/python/application/test_release_mixin_services.py +++ b/src/test/python/application/test_release_mixin_services.py @@ -40,3 +40,24 @@ def test_sould_publish_artifacts(): release = devops.mixins[MixinType.RELEASE] sut.publish_artifacts(release) assert 2345 == mock.add_asset_to_release_id + +def test_sould_throw_exception_if_there_was_an_error_in_publish_artifacts(): + devops = build_devops( + { + "release_artifacts": ["target/art"], + "release_artifact_server_url": "http://repo.test/", + "release_organisation": "orga", + "release_repository_name": "repo", + } + ) + release = devops.mixins[MixinType.RELEASE] + + with pytest.raises(Exception): + mock = ArtifactDeploymentApiMock(release='') + sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock()) + sut.publish_artifacts(release) + + with pytest.raises(Exception): + mock = ArtifactDeploymentApiMock(release='{"message": "there was an error", "url":"some-url"}') + sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock()) + sut.publish_artifacts(release) From caaf804fd32f643e687453170930c1e93bd5a886 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 09:15:46 +0200 Subject: [PATCH 31/57] add release-endpoint-calculation to domain object --- src/main/python/ddadevops/domain/release.py | 5 ++++- src/test/python/domain/test_release.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 29f78c1..f3516cd 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -77,7 +77,7 @@ class Release(Validateable): result += self.release_secondary_build_files return result - def forgejo_release_api_endpoint(self): + def forgejo_release_api_endpoint(self) -> str: validation = self.validate_for_artifact() if validation != []: raise RuntimeError(f"not valid for creating artifacts: {validation}") @@ -89,6 +89,9 @@ class Release(Validateable): repository = self.release_repository_name.removeprefix("/").removesuffix("/") return f"{server_url}/api/v1/repos/{organisation}/{repository}/releases" + def forgejo_release_asset_api_endpoint(self, id: int) -> str: + return f"{self.forgejo_release_api_endpoint()}/{id}/assets" + @classmethod def get_mapping_default(cls) -> List[Dict[str, str]]: return [ diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index 42f8699..efa82a6 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -115,3 +115,21 @@ def test_should_calculate_forgejo_release_api_endpoint(): Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), ) sut.forgejo_release_api_endpoint() + +def test_should_calculate_forgejo_release_asset_api_endpoint(): + sut = Release( + devops_config( + { + "release_artifacts": ["x"], + "release_artifact_token": "y", + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + assert ( + "https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases/123/assets" + == sut.forgejo_release_asset_api_endpoint(123) + ) From bfcdcbb78a08dced67dcdaeaa3e558959cde7476 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 09:31:01 +0200 Subject: [PATCH 32/57] wip: prepare ataching artifacts --- .../application/release_mixin_services.py | 22 +++++++++++++++++-- .../infrastructure/infrastructure.py | 7 +++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 546d93e..8dccb2d 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -69,11 +69,29 @@ class ReleaseService: release.release_artifact_token, ) ) + + artifacts_to_attach = [] for artifact_path in release.release_artifacts: self.artifact_deployment_api.calculate_checksums(artifact_path) + # TODO: make api more explizit to have the path calculation & decision which shas to take clear + artifacts_to_attach += [{ + "path": artifact_path, + # TODO: it will not always be a jar file -> move type to release input-side. + "type": "application/x-java-archive", + }, + { + "path": f"{artifact_path}.sha256", + "type": "text/plain", + }, + { + "path": f"{artifact_path}.sha512", + "type": "text/plain", + }] + + # TODO: use structure created above + for artifact in artifacts_to_attach: self.artifact_deployment_api.add_asset_to_release( - "todo", - release_id, + release.forgejo_release_asset_api_endpoint(release_id), "todo", "todo", release.release_artifact_token, diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index d86e029..9708cc6 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -231,18 +231,17 @@ class ArtifactDeploymentApi: def add_asset_to_release( self, - target_url: str, - release_id: str, + api_endpoint_url: str, attachment: str, attachment_type: str, token: str, ): return self.execution_api.execute_secure( - f'curl -X "POST" "{target_url}/{release_id}/assets" ' # {target_url}/{release_id}/assets move to Domain + f'curl -X "POST" "{api_endpoint_url}" ' + f'-H "accept: application/json" -H "Authorization: token {token}" ' + '-H "Content-Type: multipart/form-data" ' + f'-F "attachment=@{attachment};type={attachment_type}"', - sanitized_command=f'curl -X "POST" "{target_url}/{release_id}/assets" ' # see above + sanitized_command=f'curl -X "POST" "{api_endpoint_url}" ' + '-H "accept: application/json" ' + '-H "Content-Type: multipart/form-data" ' + f'-F "attachment=@{attachment};type={attachment_type}"', From 45884d10322531ad07b190d062594f98d05ee2ff Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 09:40:25 +0200 Subject: [PATCH 33/57] wip: prepare ataching artifacts - fix the test --- .../python/application/test_release_mixin_services.py | 2 +- src/test/python/domain/helper.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/python/application/test_release_mixin_services.py b/src/test/python/application/test_release_mixin_services.py index 56f9ddf..976d7c5 100644 --- a/src/test/python/application/test_release_mixin_services.py +++ b/src/test/python/application/test_release_mixin_services.py @@ -39,7 +39,7 @@ def test_sould_publish_artifacts(): ) release = devops.mixins[MixinType.RELEASE] sut.publish_artifacts(release) - assert 2345 == mock.add_asset_to_release_id + assert "http://repo.test/api/v1/repos/orga/repo/releases/2345/assets" == mock.add_asset_to_release_api_endpoint def test_sould_throw_exception_if_there_was_an_error_in_publish_artifacts(): devops = build_devops( diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 321e6a7..1740624 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -162,15 +162,15 @@ class ArtifactDeploymentApiMock: self.release = release self.create_forgejo_release_count = 0 self.add_asset_to_release_count = 0 - self.add_asset_to_release_id = "" + self.add_asset_to_release_api_endpoint = "" - def create_forgejo_release(self, target_url: str, tag: str, token: str): + def create_forgejo_release(self, api_endpoint: str, tag: str, token: str): self.create_forgejo_release_count += 1 return self.release - def add_asset_to_release(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): + def add_asset_to_release(self, api_endpoint: str, attachment: str, attachment_type: str, token: str): + self.add_asset_to_release_api_endpoint = api_endpoint self.add_asset_to_release_count += 1 - self.add_asset_to_release_id = release_id pass def calculate_checksums(self, build_path: str): From 7aa45910e9deb78e07e42c1e159820d52158d46d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 18:55:30 +0200 Subject: [PATCH 34/57] finish register artifacts --- .../python/ddadevops/application/release_mixin_services.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 8dccb2d..dd9d0d5 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -88,12 +88,11 @@ class ReleaseService: "type": "text/plain", }] - # TODO: use structure created above for artifact in artifacts_to_attach: self.artifact_deployment_api.add_asset_to_release( release.forgejo_release_asset_api_endpoint(release_id), - "todo", - "todo", + artifact["path"], + artifact["type"], release.release_artifact_token, ) From f9daf87262188ad4ec1b5d89c5b67b164ff2d5bc Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 18:58:51 +0200 Subject: [PATCH 35/57] remove no longer neede artifact deployment --- .../artifact_deployment_service.py | 27 -------- .../ddadevops/artifact_deployment_mixin.py | 61 ------------------- .../ddadevops/domain/artifact_deployment.py | 40 ------------ 3 files changed, 128 deletions(-) delete mode 100644 src/main/python/ddadevops/application/artifact_deployment_service.py delete mode 100644 src/main/python/ddadevops/artifact_deployment_mixin.py delete mode 100644 src/main/python/ddadevops/domain/artifact_deployment.py diff --git a/src/main/python/ddadevops/application/artifact_deployment_service.py b/src/main/python/ddadevops/application/artifact_deployment_service.py deleted file mode 100644 index 3d8a3b4..0000000 --- a/src/main/python/ddadevops/application/artifact_deployment_service.py +++ /dev/null @@ -1,27 +0,0 @@ -from ..infrastructure import GitApi, ArtifactDeploymentApi -from ..domain import Credentials - -# This will be moved to release mixin -class ArtifactDeploymentService: - def __init__(self, git_api: GitApi, artifact_deployment_api: ArtifactDeploymentApi): - self.git_api = git_api - self.artifact_deployment_api = artifact_deployment_api - - @classmethod - def prod(cls): - return cls( - GitApi(), - ArtifactDeploymentApi(), - ) - - def post_release(self, target_url: str, tag: str, credentials: Credentials): - response = self.artifact_deployment_api.post_release(target_url, tag, credentials.mappings["token"]) - # Get release id from response - - def post_asset(self, target_url: str, release_id: str, attachment: str, attachment_type: str, token: str): - self.artifact_deployment_api.post_asset(target_url, release_id, attachment, attachment_type, token) - - def calculate_checksums(self, build_path: str): - self.artifact_deployment_api.calculate_checksums(build_path) - - # ToDo: Update release Id as in release_mixin_services.py diff --git a/src/main/python/ddadevops/artifact_deployment_mixin.py b/src/main/python/ddadevops/artifact_deployment_mixin.py deleted file mode 100644 index 77a373c..0000000 --- a/src/main/python/ddadevops/artifact_deployment_mixin.py +++ /dev/null @@ -1,61 +0,0 @@ -from pybuilder.core import Project -from .devops_build import DevopsBuild -from .domain import MixinType - -# """ -# Functional Req: - -# General process for deploying prebuilt (meissa) binaries to our own repo server. - -# [-1] -# Building is handled by other entities -# is another pybuilder task -# the binary is reachable with devops.build_path() -# we might need to establish a "build" that does lein builds for us -# we might need to establish a "build" that does gradlew build for us -# same for all other projects that produce binaries -# currently the c4k_build.py just creates the auth and config yamls - -# [0] -# get artifact deployment url -# Base url: https://repo.prod.meissa.de/api/v1/repos/ -# Changeable: /meissa/provs/ -# persitent suffix to url: releases -# name is accessible from input - -# [1] -# get release token -# could be an api token for repo.prod.meissa.de -# credential mapping as described in the docs - -# [2] -# get release tag -# is the version of the project -# get from gitApi - -# [3] -# post a json message containting [2] to [0], watching stdout for answers -# authorized by [1] -# validate if [3] was successful by reading stdout -# or create error message containing ID of release - -# [4] -# get release-id from stdout of [3] -# print release-id - -# [5] -# generate sha256 sums & generate sha512 sums of results of [-1] - -# [6] -# push results of [-1] & [5] to [0]/[4]/assets - -# """ - -# This will be moved to release mixin -class ArtifactDeploymentMixin(DevopsBuild): - def __init__(self, project: Project, inp: dict): - super().__init__(project, inp) - devops = self.devops_repo.get_devops(self.project) - if MixinType.ARTIFACT_DEPLOYMENT not in devops.mixins: # TODO: Check for Release mixin as well - raise ValueError("ArtifactDeploymentMixin requires MixinType.ARTIFACT_DEPLOYMENT") - self.base_url = 'https://repo.prod.meissa.de/api/v1/repos/' diff --git a/src/main/python/ddadevops/domain/artifact_deployment.py b/src/main/python/ddadevops/domain/artifact_deployment.py deleted file mode 100644 index 4d0976f..0000000 --- a/src/main/python/ddadevops/domain/artifact_deployment.py +++ /dev/null @@ -1,40 +0,0 @@ -from typing import List, Dict, Optional -from .common import ( - Validateable, - CredentialMappingDefault, - DnsRecord, - Devops, -) - -# This will be moved to release mixin -class ArtifactDeployment(Validateable, CredentialMappingDefault): - def __init__(self, inp: dict): - self.name = inp.get("name") - self.artifact_base_url = inp.get("artifact_base_url") - self.organization = inp.get("organization") - - def get_artifact_release_url(self) -> str: - return f"{self.artifact_base_url}/{self.organization}/{self.name}/releases" - - def get_artifact_asset_url(self, release_id: str) -> str: - return f"{self.get_artifact_release_url}/{release_id}/assets" - - def update_runtime_config(self, dns_record: DnsRecord): - self.dns_record = dns_record - self.throw_if_invalid() - - def validate(self) -> List[str]: - result = [] - result += self.__validate_is_not_empty__("name") - result += self.__validate_is_not_empty__("artifact_base_url") - return result - - @classmethod - def get_mapping_default(cls) -> List[Dict[str, str]]: - return [ # ToDo: Adapt for token - { - "gopass_path": "server/meissa/grafana-cloud", - "gopass_field": "grafana-cloud-user", - "name": "c4k_grafana_cloud_user", - } - ] From d555b34eef9fa352925ab2fd77e2edd821e8b81a Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 19:06:57 +0200 Subject: [PATCH 36/57] add artifact to domain --- doc/architecture/Domain.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index 1f4bb88..fde909f 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -88,10 +88,14 @@ classDiagram release_type release_main_branch release_current_branch - release_artifacts release_artifact_server_url release_organisation release_repository_name + release_artifact_token + } + class Artifact { + release_artifact_path + release_artifact_type } class Credentials { <> @@ -134,6 +138,7 @@ classDiagram TerraformDomain *-- "0..1" ProviderAws: providers Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files + Release "1" *-- "0..n" Artifact: release_artifacts Release "1" *-- "1" Version: version BuildFile *-- "1" Version: version C4k *-- DnsRecord: dns_record From 2e24e79a4ca8a63d16596e668e49c6c7aca3f3b4 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 20:39:59 +0200 Subject: [PATCH 37/57] introduce artifact class --- doc/architecture/Domain.md | 5 ++- .../application/release_mixin_services.py | 33 ++++++-------- src/main/python/ddadevops/domain/__init__.py | 1 + src/main/python/ddadevops/domain/artifact.py | 43 +++++++++++++++++++ src/main/python/ddadevops/domain/release.py | 7 ++- src/main/python/ddadevops/domain/version.py | 12 +++--- .../infrastructure/infrastructure.py | 15 ++++--- src/test/python/domain/helper.py | 11 +++-- src/test/python/domain/test_artifact.py | 29 +++++++++++++ src/test/python/domain/test_devops_factory.py | 26 +++++++++++ 10 files changed, 143 insertions(+), 39 deletions(-) create mode 100644 src/main/python/ddadevops/domain/artifact.py create mode 100644 src/test/python/domain/test_artifact.py diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index fde909f..3ba7c0b 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -94,8 +94,9 @@ classDiagram release_artifact_token } class Artifact { - release_artifact_path - release_artifact_type + path_str + path() + type() } class Credentials { <> diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index dd9d0d5..0c40e98 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -2,7 +2,7 @@ import json from typing import List from pathlib import Path from ..infrastructure import GitApi, ArtifactDeploymentApi, BuildFileRepository -from ..domain import Version, Release, ReleaseType +from ..domain import Version, Release, ReleaseType, Artifact class ReleaseService: @@ -70,29 +70,20 @@ class ReleaseService: ) ) - artifacts_to_attach = [] - for artifact_path in release.release_artifacts: - self.artifact_deployment_api.calculate_checksums(artifact_path) - # TODO: make api more explizit to have the path calculation & decision which shas to take clear - artifacts_to_attach += [{ - "path": artifact_path, - # TODO: it will not always be a jar file -> move type to release input-side. - "type": "application/x-java-archive", - }, - { - "path": f"{artifact_path}.sha256", - "type": "text/plain", - }, - { - "path": f"{artifact_path}.sha512", - "type": "text/plain", - }] + artifacts_sums = [] + for artifact in release.release_artifacts: + sha256 = self.artifact_deployment_api.calculate_sha256(artifact.path()) + sha512 = self.artifact_deployment_api.calculate_sha512(artifact.path()) + artifacts_sums += [Artifact(sha256), Artifact(sha512)] - for artifact in artifacts_to_attach: + artifacts = release.release_artifacts + artifacts_sums + print(artifacts) + for artifact in artifacts: + print(str) self.artifact_deployment_api.add_asset_to_release( release.forgejo_release_asset_api_endpoint(release_id), - artifact["path"], - artifact["type"], + artifact.path(), + artifact.type(), release.release_artifact_token, ) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 234c48b..6bff258 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -17,6 +17,7 @@ from .provider_hetzner import Hetzner from .provider_aws import Aws from .provs_k3s import K3s from .release import Release +from .artifact import Artifact from .credentials import Credentials, CredentialMapping, GopassType from .version import Version from .build_file import BuildFileType, BuildFile diff --git a/src/main/python/ddadevops/domain/artifact.py b/src/main/python/ddadevops/domain/artifact.py new file mode 100644 index 0000000..fc1d221 --- /dev/null +++ b/src/main/python/ddadevops/domain/artifact.py @@ -0,0 +1,43 @@ +from enum import Enum +from pathlib import Path +from .common import ( + Validateable, +) + +class ArtifactType(Enum): + TEXT = 0 + JAR = 1 + +class Artifact(Validateable): + def __init__(self, path: str): + self.path_str = path + + def path(self) -> Path: + return Path(self.path_str) + + def type(self) -> str: + suffix = self.path().suffix + match suffix: + case ".jar": + return "application/x-java-archive" + case _: + return "text/plain" + + def validate(self): + result = [] + result += self.__validate_is_not_empty__("path_str") + try: + Path(self.path_str) + except Exception as e: + result += [f"path was not a valid: {e}"] + return result + + def __str__(self): + return str(self.path()) + + def __eq__(self, other): + return other and self.__str__() == other.__str__() + + def __hash__(self) -> int: + return self.__str__().__hash__() + diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index f3516cd..e08ae6e 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -7,6 +7,9 @@ from .common import ( from .version import ( Version, ) +from .artifact import ( + Artifact, +) class Release(Validateable): @@ -17,7 +20,6 @@ class Release(Validateable): self.release_primary_build_file = inp.get( "release_primary_build_file", "./project.clj" ) - self.release_artifacts = inp.get("release_artifacts", []) self.release_secondary_build_files = inp.get( "release_secondary_build_files", [] ) @@ -26,6 +28,9 @@ class Release(Validateable): self.release_organisation = inp.get("release_organisation") self.release_repository_name = inp.get("release_repository_name") self.release_artifact_token = inp.get("release_artifact_token") + self.release_artifacts = [] + for a in inp.get("release_artifacts", []): + self.release_artifacts.append(Artifact(a)) def update_release_type(self, release_type: ReleaseType): self.release_type = release_type diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 2f2096a..3c9ac0e 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -32,12 +32,6 @@ class Version(Validateable): self.snapshot_suffix = snapshot_suffix self.default_snapshot_suffix = default_snapshot_suffix - def __eq__(self, other): - return other and self.to_string() == other.to_string() - - def __hash__(self) -> int: - return self.to_string().__hash__() - def is_snapshot(self): return self.snapshot_suffix is not None @@ -139,3 +133,9 @@ class Version(Validateable): snapshot_suffix=None, version_str=None, ) + + def __eq__(self, other): + return other and self.to_string() == other.to_string() + + def __hash__(self) -> int: + return self.to_string().__hash__() diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 9708cc6..e94fab9 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -247,13 +247,16 @@ class ArtifactDeploymentApi: + f'-F "attachment=@{attachment};type={attachment_type}"', ) - def calculate_checksums(self, artifact_path: str): - # self.execution_api.execute(f"find {artifact_path} -type f - # -exec sha256sum {{}}; | sort > {artifact_path} sha256sum.lst") - # relevant für provs + def calculate_sha256(self, path: Path): + sum = f"{path}.sha256" self.execution_api( - f"sha256sum {artifact_path} > {artifact_path}.sha256", + f"sha256sum {path} > {sum}", ) + return sum + + def calculate_sha512(self, path: Path): + sum = f"{path}.sha512" self.execution_api( - f"sha512sum {artifact_path} > {artifact_path}.sha512", + f"sha512sum {path} > {sum}", ) + return sum diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 1740624..7855dc3 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -168,10 +168,15 @@ class ArtifactDeploymentApiMock: self.create_forgejo_release_count += 1 return self.release - def add_asset_to_release(self, api_endpoint: str, attachment: str, attachment_type: str, token: str): + def add_asset_to_release( + self, api_endpoint: str, attachment: str, attachment_type: str, token: str + ): self.add_asset_to_release_api_endpoint = api_endpoint self.add_asset_to_release_count += 1 pass - def calculate_checksums(self, build_path: str): - pass + def calculate_sha256(self, path: Path): + return f"{path}.sha256" + + def calculate_sha512(self, path: Path): + return f"{path}.sha512" diff --git a/src/test/python/domain/test_artifact.py b/src/test/python/domain/test_artifact.py new file mode 100644 index 0000000..e454c5c --- /dev/null +++ b/src/test/python/domain/test_artifact.py @@ -0,0 +1,29 @@ +import pytest +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain import ( + Validateable, + DnsRecord, + Devops, + BuildType, + MixinType, + Artifact, + Image, +) +from .helper import build_devops, devops_config + + +def test_sould_validate_release(): + sut = Artifact("x") + assert sut.is_valid() + + sut = Artifact(None) + assert not sut.is_valid() + +def test_should_calculate_type(): + sut = Artifact("x.jar") + assert "application/x-java-archive" == sut.type() + + sut = Artifact("x.jar.sha256") + assert "text/plain" == sut.type() + diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index b9f4008..26cc319 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -4,6 +4,7 @@ from src.main.python.ddadevops.domain import ( Version, BuildType, MixinType, + Artifact, ) @@ -50,6 +51,7 @@ def test_devops_creation(): assert sut is not None assert sut.specialized_builds[BuildType.C4K] is not None +def test_release_devops_creation(): sut = DevopsFactory().build_devops( { "stage": "test", @@ -67,6 +69,30 @@ def test_devops_creation(): assert sut is not None assert sut.mixins[MixinType.RELEASE] is not None + sut = DevopsFactory().build_devops( + { + "stage": "test", + "name": "mybuild", + "module": "test_image", + "project_root_path": "../../..", + "build_types": [], + "mixin_types": ["RELEASE"], + "release_main_branch": "main", + "release_current_branch": "my_feature", + "release_config_file": "project.clj", + "release_artifacts": ["x.jar"], + "release_artifact_token": "y", + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "provs", + }, + Version.from_str("1.0.0", "SNAPSHOT"), + ) + + release = sut.mixins[MixinType.RELEASE] + assert release is not None + assert Artifact("x.jar") == release.release_artifacts[0] + def test_on_merge_input_should_win(): sut = DevopsFactory() From 7d2d197cbb31b906e1ccaec159e40b4a7c723fa7 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Mon, 14 Aug 2023 20:41:04 +0200 Subject: [PATCH 38/57] fix some linting --- src/main/python/ddadevops/domain/artifact.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/domain/artifact.py b/src/main/python/ddadevops/domain/artifact.py index fc1d221..e6d5904 100644 --- a/src/main/python/ddadevops/domain/artifact.py +++ b/src/main/python/ddadevops/domain/artifact.py @@ -4,10 +4,12 @@ from .common import ( Validateable, ) + class ArtifactType(Enum): TEXT = 0 JAR = 1 + class Artifact(Validateable): def __init__(self, path: str): self.path_str = path @@ -22,7 +24,7 @@ class Artifact(Validateable): return "application/x-java-archive" case _: return "text/plain" - + def validate(self): result = [] result += self.__validate_is_not_empty__("path_str") @@ -40,4 +42,3 @@ class Artifact(Validateable): def __hash__(self) -> int: return self.__str__().__hash__() - From 2be0a44aa8168c5ee6ebafef84eeff74b73a7b8e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 15 Aug 2023 08:09:27 +0200 Subject: [PATCH 39/57] add js --- src/main/python/ddadevops/domain/artifact.py | 2 ++ src/test/python/domain/test_artifact.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/main/python/ddadevops/domain/artifact.py b/src/main/python/ddadevops/domain/artifact.py index e6d5904..f7ae720 100644 --- a/src/main/python/ddadevops/domain/artifact.py +++ b/src/main/python/ddadevops/domain/artifact.py @@ -22,6 +22,8 @@ class Artifact(Validateable): match suffix: case ".jar": return "application/x-java-archive" + case ".js": + return "application/x-javascript" case _: return "text/plain" diff --git a/src/test/python/domain/test_artifact.py b/src/test/python/domain/test_artifact.py index e454c5c..86e5324 100644 --- a/src/test/python/domain/test_artifact.py +++ b/src/test/python/domain/test_artifact.py @@ -24,6 +24,9 @@ def test_should_calculate_type(): sut = Artifact("x.jar") assert "application/x-java-archive" == sut.type() + sut = Artifact("x.js") + assert "application/x-javascript" == sut.type() + sut = Artifact("x.jar.sha256") assert "text/plain" == sut.type() From ab8bb7f400e57dae29428ba737ca9947e6c72a09 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 15 Aug 2023 08:21:06 +0200 Subject: [PATCH 40/57] allow empty artifacts --- .gitlab-ci.yml | 1 - build.py | 12 ++++++++++-- src/main/python/ddadevops/domain/release.py | 1 - src/test/python/domain/test_release.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6570fa..312e95a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,7 +59,6 @@ clj-image-publish: script: - cd infrastructure/clj && pyb image publish - python-image-publish: <<: *img <<: *tag_only diff --git a/build.py b/build.py index c0377b4..c7487bb 100644 --- a/build.py +++ b/build.py @@ -33,7 +33,7 @@ default_task = "dev" name = "ddadevops" MODULE = "not-used" PROJECT_ROOT_PATH = "." -version = "4.3.2-dev" +version = "4.3.2-dev1" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] @@ -103,6 +103,10 @@ def initialize(project): "infrastructure/clj-cljs/build.py", "infrastructure/clj/build.py", ], + "release_artifacts": [], + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "dda-devops-build", } build = ReleaseMixin(project, input) @@ -176,7 +180,11 @@ def prepare(project): def tag(project): build = get_devops_build(project) build.tag_bump_and_push_release() - #TODO: build.publish_artifacts() + + +@task +def publish_artifacts(project): + build.publish_artifacts() def release(project): diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index e08ae6e..99fa091 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -73,7 +73,6 @@ class Release(Validateable): result += self.__validate_is_not_empty__("release_artifact_server_url") result += self.__validate_is_not_empty__("release_organisation") result += self.__validate_is_not_empty__("release_repository_name") - result += self.__validate_is_not_empty__("release_artifacts") result += self.__validate_is_not_empty__("release_artifact_token") return result diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index efa82a6..968ce69 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -68,7 +68,7 @@ def test_should_calculate_forgejo_release_api_endpoint(): sut = Release( devops_config( { - "release_artifacts": ["x"], + "release_artifacts": [], "release_artifact_token": "y", "release_artifact_server_url": "https://repo.prod.meissa.de", "release_organisation": "meissa", From 1fff46986e4f5303373b5072715308d242745218 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 15 Aug 2023 08:23:00 +0200 Subject: [PATCH 41/57] create an empty release --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 312e95a..01a4e4a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ pypi-stable: <<: *tag_only stage: upload script: - - pyb -P version=$CI_COMMIT_TAG publish upload + - pyb -P version=$CI_COMMIT_TAG publish upload publish_artifacts clj-cljs-image-publish: <<: *img From 06a650fd7773dab300c069c55987a045e6a73a1d Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 15 Aug 2023 08:28:15 +0200 Subject: [PATCH 42/57] use new version for ci --- .gitlab-ci.yml | 2 +- infrastructure/python/image/Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01a4e4a..52ccd7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - image .py: &py - image: "domaindrivenarchitecture/ddadevops-python:4.1.0" + image: "domaindrivenarchitecture/ddadevops-python:4.3.2-dev2023-08-15-08-26-54" before_script: - python --version - pip install -r requirements.txt diff --git a/infrastructure/python/image/Dockerfile b/infrastructure/python/image/Dockerfile index 3d49141..57d756c 100644 --- a/infrastructure/python/image/Dockerfile +++ b/infrastructure/python/image/Dockerfile @@ -5,3 +5,4 @@ RUN apk add --no-cache build-base rust python3 python3-dev py3-pip py3-setuptool RUN python3 -m pip install -U pip; RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 pyyaml inflection; RUN pip3 install coverage flake8 flake8-polyfill mypy mypy-extensions pycodestyle pyflakes pylint pytest pytest-cov pytest-datafiles types-setuptools types-PyYAML; +RUN pip3 install --upgrade ddadevops --pre From dea2d7f3ed508e08e03dcd6a9d140e1bb0133c3f Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 10:30:58 +0200 Subject: [PATCH 43/57] Ignore line too long --- src/main/python/ddadevops/infrastructure/infrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index e94fab9..48959c2 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -226,7 +226,7 @@ class ArtifactDeploymentApi: + f'-H "Authorization: token {token}"', sanitized_command=f'curl -X "POST" "{api_endpoint_url}" ' + '-H "accept: application/json" -H "Content-Type: application/json" ' - + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ', + + f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ', # noqa: E501 ) # noqa: E501 def add_asset_to_release( From 46c8a1751c77672c77fa846ff3ac3a7fdff599a3 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 12:09:06 +0200 Subject: [PATCH 44/57] Rename var --- src/main/python/ddadevops/domain/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index 99fa091..490db9b 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -93,8 +93,8 @@ class Release(Validateable): repository = self.release_repository_name.removeprefix("/").removesuffix("/") return f"{server_url}/api/v1/repos/{organisation}/{repository}/releases" - def forgejo_release_asset_api_endpoint(self, id: int) -> str: - return f"{self.forgejo_release_api_endpoint()}/{id}/assets" + def forgejo_release_asset_api_endpoint(self, release_id: int) -> str: + return f"{self.forgejo_release_api_endpoint()}/{release_id}/assets" @classmethod def get_mapping_default(cls) -> List[Dict[str, str]]: From a9d01c59070249eecf8b971f1d8a774f07bee699 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 12:10:02 +0200 Subject: [PATCH 45/57] Assert that token is a string --- .../python/ddadevops/application/release_mixin_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index 0c40e98..bc3ab0a 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -66,7 +66,7 @@ class ReleaseService: self.artifact_deployment_api.create_forgejo_release( release.forgejo_release_api_endpoint(), release.version.to_string(), - release.release_artifact_token, + str(release.release_artifact_token), ) ) @@ -84,7 +84,7 @@ class ReleaseService: release.forgejo_release_asset_api_endpoint(release_id), artifact.path(), artifact.type(), - release.release_artifact_token, + str(release.release_artifact_token), ) def __parse_forgejo_release_id__(self, release_response: str) -> int: From ddb173af46c94c5d7abf39f0dcd6923f01696482 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 12:10:47 +0200 Subject: [PATCH 46/57] Change attachment type to path --- src/main/python/ddadevops/infrastructure/infrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 48959c2..ed80da0 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -232,7 +232,7 @@ class ArtifactDeploymentApi: def add_asset_to_release( self, api_endpoint_url: str, - attachment: str, + attachment: Path, attachment_type: str, token: str, ): From 57b88664d2da8c6685f3d4cf0555e4cc0880bf92 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 12:11:08 +0200 Subject: [PATCH 47/57] Rename vars --- .../ddadevops/infrastructure/infrastructure.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index ed80da0..fda7c90 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -248,15 +248,15 @@ class ArtifactDeploymentApi: ) def calculate_sha256(self, path: Path): - sum = f"{path}.sha256" - self.execution_api( - f"sha256sum {path} > {sum}", + shasum = f"{path}.sha256" + self.execution_api.execute( + f"sha256sum {path} > {shasum}", ) - return sum + return shasum def calculate_sha512(self, path: Path): - sum = f"{path}.sha512" - self.execution_api( - f"sha512sum {path} > {sum}", + shasum = f"{path}.sha512" + self.execution_api.execute( + f"sha512sum {path} > {shasum}", ) - return sum + return shasum From 25887841a2937a0ee1f564c922a29e85907fc891 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 12:48:37 +0200 Subject: [PATCH 48/57] Add missing var def --- build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build.py b/build.py index c7487bb..c36c659 100644 --- a/build.py +++ b/build.py @@ -184,6 +184,7 @@ def tag(project): @task def publish_artifacts(project): + build = get_devops_build(project) build.publish_artifacts() From dce22ba7b3fb87cb4e175a681aa9f900e0c91005 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 13:54:28 +0200 Subject: [PATCH 49/57] Remove publish_artifact task for test purpose --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52ccd7d..22a5221 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ pypi-stable: <<: *tag_only stage: upload script: - - pyb -P version=$CI_COMMIT_TAG publish upload publish_artifacts + - pyb -P version=$CI_COMMIT_TAG publish upload clj-cljs-image-publish: <<: *img From 66691c9ee9bb057c7cd7bb6982fb92d3e97531e7 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 13:57:15 +0200 Subject: [PATCH 50/57] Add task publish artifacts --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22a5221..52ccd7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ pypi-stable: <<: *tag_only stage: upload script: - - pyb -P version=$CI_COMMIT_TAG publish upload + - pyb -P version=$CI_COMMIT_TAG publish upload publish_artifacts clj-cljs-image-publish: <<: *img From 1f2e1d956902376055e7aeeb6487e0ce16209a2f Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 14:07:52 +0200 Subject: [PATCH 51/57] Use v4.2.0 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52ccd7d..1bd6609 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - image .py: &py - image: "domaindrivenarchitecture/ddadevops-python:4.3.2-dev2023-08-15-08-26-54" + image: "domaindrivenarchitecture/ddadevops-python:4.2.0" before_script: - python --version - pip install -r requirements.txt From d160c551e185dedeaeb63dfbeceeb0a2f9f5e917 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 14:13:15 +0200 Subject: [PATCH 52/57] Use v4.3.0 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1bd6609..4ea93f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - image .py: &py - image: "domaindrivenarchitecture/ddadevops-python:4.2.0" + image: "domaindrivenarchitecture/ddadevops-python:4.3.0" before_script: - python --version - pip install -r requirements.txt From 46b8172f11a4dba20d2b03040a1cbde6ac302ca3 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 16 Aug 2023 14:15:38 +0200 Subject: [PATCH 53/57] Use v4.3.1 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ea93f7..911e93c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - image .py: &py - image: "domaindrivenarchitecture/ddadevops-python:4.3.0" + image: "domaindrivenarchitecture/ddadevops-python:4.3.1" before_script: - python --version - pip install -r requirements.txt From b6b2e6d9b5e4faf0b09fd900dd075e7ee377a12f Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 16 Aug 2023 15:25:57 +0200 Subject: [PATCH 54/57] add env for all jobs --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 911e93c..61f75b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ stages: .py: &py image: "domaindrivenarchitecture/ddadevops-python:4.3.1" before_script: + - export RELEASE_ARTIFACT_TOKEN=$RELEASE_ARTIFACT_TOKEN - python --version - pip install -r requirements.txt From ec0844d53b7ecad8ba5483898d7dc15e4d3603dc Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 16 Aug 2023 15:42:59 +0200 Subject: [PATCH 55/57] use new ddadevops --- .gitlab-ci.yml | 2 +- build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61f75b3..2ac01a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - image .py: &py - image: "domaindrivenarchitecture/ddadevops-python:4.3.1" + image: "domaindrivenarchitecture/ddadevops-python:4.3.2-dev2023-08-16-15-37-46" before_script: - export RELEASE_ARTIFACT_TOKEN=$RELEASE_ARTIFACT_TOKEN - python --version diff --git a/build.py b/build.py index c36c659..f6db68d 100644 --- a/build.py +++ b/build.py @@ -33,7 +33,7 @@ default_task = "dev" name = "ddadevops" MODULE = "not-used" PROJECT_ROOT_PATH = "." -version = "4.3.2-dev1" +version = "4.3.2-dev2" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] From 17ae55f3c072ed8bcf5cb3e6afdfa21f991b84b6 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 16 Aug 2023 16:10:56 +0200 Subject: [PATCH 56/57] fix drun & improve env rekognition --- build.py | 2 +- src/main/python/ddadevops/domain/init_service.py | 5 ++--- .../python/ddadevops/infrastructure/infrastructure.py | 9 +++++++-- src/test/python/domain/helper.py | 3 +++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/build.py b/build.py index f6db68d..f32588d 100644 --- a/build.py +++ b/build.py @@ -33,7 +33,7 @@ default_task = "dev" name = "ddadevops" MODULE = "not-used" PROJECT_ROOT_PATH = "." -version = "4.3.2-dev2" +version = "4.3.2-dev6" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 1afa1f3..8529665 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -112,9 +112,8 @@ class InitService: result = {} for name in credentials.mappings.keys(): mapping = credentials.mappings[name] - env_value = self.environment_api.get(mapping.name_for_environment()) - if env_value: - result[name] = env_value + if self.environment_api.is_defined(mapping.name_for_environment()): + result[name] = self.environment_api.get(mapping.name_for_environment()) else: if mapping.gopass_type() == GopassType.FIELD: result[name] = self.credentials_api.gopass_field_from_path( diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index fda7c90..847bc57 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -58,8 +58,10 @@ class ImageApi: ) def drun(self, name: str): - self.execution_api.execute_live( - f'docker run -it --entrypoint="" {name} /bin/bash' + run( + f'docker run -it --entrypoint="" {name} /bin/bash', + shell=True, + check=True, ) def dockerhub_login(self, username: str, password: str): @@ -134,6 +136,9 @@ class EnvironmentApi: def get(self, key): return environ.get(key) + def is_defined(self, key): + return key in environ + class CredentialsApi: def __init__(self): diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 7855dc3..7c6df4c 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -104,6 +104,9 @@ class EnvironmentApiMock: def get(self, key): return self.mappings.get(key, None) + def is_defined(self, key): + return key in self.mappings + class CredentialsApiMock: def __init__(self, mappings): From 898b8e89004a9d0ed638a8cbd0d6d8d4691eb777 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 16 Aug 2023 16:21:12 +0200 Subject: [PATCH 57/57] use new version --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ac01a9..67cf964 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - image .py: &py - image: "domaindrivenarchitecture/ddadevops-python:4.3.2-dev2023-08-16-15-37-46" + image: "domaindrivenarchitecture/ddadevops-python:4.3.2-dev2023-08-16-16-16-48" before_script: - export RELEASE_ARTIFACT_TOKEN=$RELEASE_ARTIFACT_TOKEN - python --version