Merge branch 'artifact-mixin' into 'main'

Release creation on forgejo targets

See merge request domaindrivenarchitecture/dda-devops-build!18
This commit is contained in:
Michael Jerger 2023-08-16 14:28:27 +00:00
commit 62e3f58f81
20 changed files with 454 additions and 89 deletions

View file

@ -4,8 +4,9 @@ stages:
- image
.py: &py
image: "domaindrivenarchitecture/ddadevops-python:4.1.0"
image: "domaindrivenarchitecture/ddadevops-python:4.3.2-dev2023-08-16-16-16-48"
before_script:
- export RELEASE_ARTIFACT_TOKEN=$RELEASE_ARTIFACT_TOKEN
- python --version
- pip install -r requirements.txt
@ -43,7 +44,7 @@ pypi-stable:
<<: *tag_only
stage: upload
script:
- pyb -P version=$CI_COMMIT_TAG publish upload
- pyb -P version=$CI_COMMIT_TAG publish upload publish_artifacts
clj-cljs-image-publish:
<<: *img
@ -59,7 +60,6 @@ clj-image-publish:
script:
- cd infrastructure/clj && pyb image publish
python-image-publish:
<<: *img
<<: *tag_only

View file

@ -33,7 +33,7 @@ default_task = "dev"
name = "ddadevops"
MODULE = "not-used"
PROJECT_ROOT_PATH = "."
version = "4.3.2-dev"
version = "4.3.2-dev6"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]
@ -103,6 +103,10 @@ def initialize(project):
"infrastructure/clj-cljs/build.py",
"infrastructure/clj/build.py",
],
"release_artifacts": [],
"release_artifact_server_url": "https://repo.prod.meissa.de",
"release_organisation": "meissa",
"release_repository_name": "dda-devops-build",
}
build = ReleaseMixin(project, input)
@ -178,6 +182,12 @@ def tag(project):
build.tag_bump_and_push_release()
@task
def publish_artifacts(project):
build = get_devops_build(project)
build.publish_artifacts()
def release(project):
prepare(project)
tag(project)

View file

@ -88,6 +88,15 @@ classDiagram
release_type
release_main_branch
release_current_branch
release_artifact_server_url
release_organisation
release_repository_name
release_artifact_token
}
class Artifact {
path_str
path()
type()
}
class Credentials {
<<AggregateRoot>>
@ -130,6 +139,7 @@ classDiagram
TerraformDomain *-- "0..1" ProviderAws: providers
Release o-- "0..1" BuildFile: primary_build_file
Release o-- "0..n" BuildFile: secondary_build_files
Release "1" *-- "0..n" Artifact: release_artifacts
Release "1" *-- "1" Version: version
BuildFile *-- "1" Version: version
C4k *-- DnsRecord: dns_record

View file

@ -5,3 +5,4 @@ RUN apk add --no-cache build-base rust python3 python3-dev py3-pip py3-setuptool
RUN python3 -m pip install -U pip;
RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 pyyaml inflection;
RUN pip3 install coverage flake8 flake8-polyfill mypy mypy-extensions pycodestyle pyflakes pylint pytest pytest-cov pytest-datafiles types-setuptools types-PyYAML;
RUN pip3 install --upgrade ddadevops --pre

View file

@ -1,18 +1,26 @@
import json
from typing import List
from pathlib import Path
from ..infrastructure import GitApi, BuildFileRepository
from ..domain import Version, Release, ReleaseType
from ..infrastructure import GitApi, ArtifactDeploymentApi, BuildFileRepository
from ..domain import Version, Release, ReleaseType, Artifact
class ReleaseService:
def __init__(self, git_api: GitApi, build_file_repository: BuildFileRepository):
def __init__(
self,
git_api: GitApi,
artifact_deployment_api: ArtifactDeploymentApi,
build_file_repository: BuildFileRepository,
):
self.git_api = git_api
self.artifact_deployment_api = artifact_deployment_api
self.build_file_repository = build_file_repository
@classmethod
def prod(cls, base_dir: str):
return cls(
GitApi(),
ArtifactDeploymentApi(),
BuildFileRepository(base_dir),
)
@ -53,6 +61,36 @@ class ReleaseService:
)
self.git_api.push_follow_tags()
def publish_artifacts(self, release: Release):
release_id = self.__parse_forgejo_release_id__(
self.artifact_deployment_api.create_forgejo_release(
release.forgejo_release_api_endpoint(),
release.version.to_string(),
str(release.release_artifact_token),
)
)
artifacts_sums = []
for artifact in release.release_artifacts:
sha256 = self.artifact_deployment_api.calculate_sha256(artifact.path())
sha512 = self.artifact_deployment_api.calculate_sha512(artifact.path())
artifacts_sums += [Artifact(sha256), Artifact(sha512)]
artifacts = release.release_artifacts + artifacts_sums
print(artifacts)
for artifact in artifacts:
print(str)
self.artifact_deployment_api.add_asset_to_release(
release.forgejo_release_asset_api_endpoint(release_id),
artifact.path(),
artifact.type(),
str(release.release_artifact_token),
)
def __parse_forgejo_release_id__(self, release_response: str) -> int:
parsed = json.loads(release_response)
return parsed["id"]
def __set_version_and_commit__(
self, version: Version, build_file_ids: List[str], message: str
):

View file

@ -1,56 +0,0 @@
from pybuilder.core import Project
from .devops_build import DevopsBuild
# """
# Functional Req:
# General process for deploying prebuilt (meissa) binaries to our own repo server.
# [-1]
# Building is handled by other entities
# is another pybuilder task
# the binary is reachable with devops.build_path()
# we might need to establish a "build" that does lein builds for us
# we might need to establish a "build" that does gradlew build for us
# same for all other projects that produce binaries
# currently the c4k_build.py just creates the auth and config yamls
# [0]
# get artifact deployment url
# Base url: https://repo.prod.meissa.de/api/v1/repos/
# Changeable: /meissa/provs/
# persitent suffix to url: releases
# name is accessible from input
# [1]
# get release token
# could be an api token for repo.prod.meissa.de
# credential mapping as described in the docs
# [2]
# get release tag
# is the version of the project
# get from gitApi
# [3]
# post a json message containting [2] to [0], watching stdout for answers
# authorized by [1]
# validate if [3] was successful by reading stdout
# or create error message containing ID of release
# [4]
# get release-id from stdout of [3]
# print release-id
# [5]
# generate sha256 sums & generate sha512 sums of results of [-1]
# [6]
# push results of [-1] & [5] to [0]/[4]
# """
class ArtifactDeploymentMixin(DevopsBuild):
def __init__(self, project: Project, inp: dict):
super().__init__(project, inp)

View file

@ -17,6 +17,7 @@ from .provider_hetzner import Hetzner
from .provider_aws import Aws
from .provs_k3s import K3s
from .release import Release
from .artifact import Artifact
from .credentials import Credentials, CredentialMapping, GopassType
from .version import Version
from .build_file import BuildFileType, BuildFile

View file

@ -0,0 +1,46 @@
from enum import Enum
from pathlib import Path
from .common import (
Validateable,
)
class ArtifactType(Enum):
TEXT = 0
JAR = 1
class Artifact(Validateable):
def __init__(self, path: str):
self.path_str = path
def path(self) -> Path:
return Path(self.path_str)
def type(self) -> str:
suffix = self.path().suffix
match suffix:
case ".jar":
return "application/x-java-archive"
case ".js":
return "application/x-javascript"
case _:
return "text/plain"
def validate(self):
result = []
result += self.__validate_is_not_empty__("path_str")
try:
Path(self.path_str)
except Exception as e:
result += [f"path was not a valid: {e}"]
return result
def __str__(self):
return str(self.path())
def __eq__(self, other):
return other and self.__str__() == other.__str__()
def __hash__(self) -> int:
return self.__str__().__hash__()

View file

@ -8,7 +8,7 @@ from .provider_digitalocean import Digitalocean
from .provider_hetzner import Hetzner
from .c4k import C4k
from .image import Image
from .release import ReleaseType
from .release import ReleaseType, Release
from ..infrastructure import BuildFileRepository, CredentialsApi, EnvironmentApi, GitApi
@ -69,6 +69,7 @@ class InitService:
Path(primary_build_file_id)
)
version = primary_build_file.get_version()
default_mappings += Release.get_mapping_default()
credentials = Credentials(inp, default_mappings)
authorization = self.authorization(credentials)
@ -111,9 +112,8 @@ class InitService:
result = {}
for name in credentials.mappings.keys():
mapping = credentials.mappings[name]
env_value = self.environment_api.get(mapping.name_for_environment())
if env_value:
result[name] = env_value
if self.environment_api.is_defined(mapping.name_for_environment()):
result[name] = self.environment_api.get(mapping.name_for_environment())
else:
if mapping.gopass_type() == GopassType.FIELD:
result[name] = self.credentials_api.gopass_field_from_path(

View file

@ -1,4 +1,4 @@
from typing import Optional, List
from typing import Optional, List, Dict
from pathlib import Path
from .common import (
Validateable,
@ -7,6 +7,9 @@ from .common import (
from .version import (
Version,
)
from .artifact import (
Artifact,
)
class Release(Validateable):
@ -21,6 +24,13 @@ class Release(Validateable):
"release_secondary_build_files", []
)
self.version = version
self.release_artifact_server_url = inp.get("release_artifact_server_url")
self.release_organisation = inp.get("release_organisation")
self.release_repository_name = inp.get("release_repository_name")
self.release_artifact_token = inp.get("release_artifact_token")
self.release_artifacts = []
for a in inp.get("release_artifacts", []):
self.release_artifacts.append(Artifact(a))
def update_release_type(self, release_type: ReleaseType):
self.release_type = release_type
@ -53,10 +63,44 @@ class Release(Validateable):
and self.release_type != ReleaseType.NONE
and self.release_main_branch != self.release_current_branch
):
result.append(f"Releases are allowed only on {self.release_main_branch}")
result.append(
f"Releases are allowed only on {self.release_main_branch}"
)
return result
def validate_for_artifact(self):
result = []
result += self.__validate_is_not_empty__("release_artifact_server_url")
result += self.__validate_is_not_empty__("release_organisation")
result += self.__validate_is_not_empty__("release_repository_name")
result += self.__validate_is_not_empty__("release_artifact_token")
return result
def build_files(self) -> List[str]:
result = [self.release_primary_build_file]
result += self.release_secondary_build_files
return result
def forgejo_release_api_endpoint(self) -> str:
validation = self.validate_for_artifact()
if validation != []:
raise RuntimeError(f"not valid for creating artifacts: {validation}")
server_url = self.release_artifact_server_url.removeprefix("/").removesuffix(
"/"
)
organisation = self.release_organisation.removeprefix("/").removesuffix("/")
repository = self.release_repository_name.removeprefix("/").removesuffix("/")
return f"{server_url}/api/v1/repos/{organisation}/{repository}/releases"
def forgejo_release_asset_api_endpoint(self, release_id: int) -> str:
return f"{self.forgejo_release_api_endpoint()}/{release_id}/assets"
@classmethod
def get_mapping_default(cls) -> List[Dict[str, str]]:
return [
{
"gopass_path": "server/meissa/repo/buero-rw",
"name": "release_artifact_token",
}
]

View file

@ -32,12 +32,6 @@ class Version(Validateable):
self.snapshot_suffix = snapshot_suffix
self.default_snapshot_suffix = default_snapshot_suffix
def __eq__(self, other):
return other and self.to_string() == other.to_string()
def __hash__(self) -> int:
return self.to_string().__hash__()
def is_snapshot(self):
return self.snapshot_suffix is not None
@ -139,3 +133,9 @@ class Version(Validateable):
snapshot_suffix=None,
version_str=None,
)
def __eq__(self, other):
return other and self.to_string() == other.to_string()
def __hash__(self) -> int:
return self.to_string().__hash__()

View file

@ -7,5 +7,6 @@ from .infrastructure import (
CredentialsApi,
GitApi,
TerraformApi,
ArtifactDeploymentApi,
)
from .repository import DevopsRepository, BuildFileRepository

View file

@ -58,23 +58,21 @@ class ImageApi:
)
def drun(self, name: str):
self.execution_api.execute_live(
f'docker run -it --entrypoint="" {name} /bin/bash'
run(
f'docker run -it --entrypoint="" {name} /bin/bash',
shell=True,
check=True,
)
def dockerhub_login(self, username: str, password: str):
self.execution_api.execute_secure(
f"docker login --username {username} --password {password}",
"docker login --username ***** --password *****"
"docker login --username ***** --password *****",
)
def dockerhub_publish(self, name: str, username: str, tag: str):
self.execution_api.execute_live(
f"docker tag {name} {username}/{name}:{tag}"
)
self.execution_api.execute_live(
f"docker push {username}/{name}:{tag}"
)
self.execution_api.execute_live(f"docker tag {name} {username}/{name}:{tag}")
self.execution_api.execute_live(f"docker push {username}/{name}:{tag}")
def test(self, name: str, path: Path):
self.execution_api.execute_live(
@ -95,14 +93,24 @@ class ExecutionApi:
check=check,
stdout=PIPE,
stderr=PIPE,
text=True).stdout
text=True,
).stdout
output = output.rstrip()
except CalledProcessError as exc:
print(f"Command failed with code: {exc.returncode} and message: {exc.stderr}")
print(
f"Command failed with code: {exc.returncode} and message: {exc.stderr}"
)
raise exc
return output
def execute_secure(self, command: str, sanitized_command: str, dry_run=False, shell=True, check=True):
def execute_secure(
self,
command: str,
sanitized_command: str,
dry_run=False,
shell=True,
check=True,
):
try:
output = self.execute(command, dry_run, shell, check)
return output
@ -128,6 +136,9 @@ class EnvironmentApi:
def get(self, key):
return environ.get(key)
def is_defined(self, key):
return key in environ
class CredentialsApi:
def __init__(self):
@ -206,3 +217,51 @@ class GitApi:
class TerraformApi:
pass
class ArtifactDeploymentApi:
def __init__(self):
self.execution_api = ExecutionApi()
def create_forgejo_release(self, api_endpoint_url: str, tag: str, token: str):
return self.execution_api.execute_secure(
f'curl -X "POST" "{api_endpoint_url}" '
+ '-H "accept: application/json" -H "Content-Type: application/json" '
+ f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ' # noqa: E501
+ f'-H "Authorization: token {token}"',
sanitized_command=f'curl -X "POST" "{api_endpoint_url}" '
+ '-H "accept: application/json" -H "Content-Type: application/json" '
+ f'-d "{{ "body": "Provides files for release {tag} Attention: The "Source Code"-files below are not up-to-date!", "tag_name": "{tag}"}}" ', # noqa: E501
) # noqa: E501
def add_asset_to_release(
self,
api_endpoint_url: str,
attachment: Path,
attachment_type: str,
token: str,
):
return self.execution_api.execute_secure(
f'curl -X "POST" "{api_endpoint_url}" '
+ f'-H "accept: application/json" -H "Authorization: token {token}" '
+ '-H "Content-Type: multipart/form-data" '
+ f'-F "attachment=@{attachment};type={attachment_type}"',
sanitized_command=f'curl -X "POST" "{api_endpoint_url}" '
+ '-H "accept: application/json" '
+ '-H "Content-Type: multipart/form-data" '
+ f'-F "attachment=@{attachment};type={attachment_type}"',
)
def calculate_sha256(self, path: Path):
shasum = f"{path}.sha256"
self.execution_api.execute(
f"sha256sum {path} > {shasum}",
)
return shasum
def calculate_sha512(self, path: Path):
shasum = f"{path}.sha512"
self.execution_api.execute(
f"sha512sum {path} > {shasum}",
)
return shasum

View file

@ -26,3 +26,8 @@ class ReleaseMixin(DevopsBuild):
devops = self.devops_repo.get_devops(self.project)
release = devops.mixins[MixinType.RELEASE]
self.release_service.tag_bump_and_push_release(release)
def publish_artifacts(self):
devops = self.devops_repo.get_devops(self.project)
release = devops.mixins[MixinType.RELEASE]
self.release_service.publish_artifacts(release)

View file

@ -1,18 +1,22 @@
import pytest
from pathlib import Path
from src.main.python.ddadevops.domain import (
ReleaseType,
ReleaseType,
MixinType,
)
from src.test.python.domain.helper import (
BuildFileRepositoryMock,
GitApiMock,
ArtifactDeploymentApiMock,
build_devops,
)
from src.main.python.ddadevops.application import ReleaseService
def test_sould_update_release_type():
sut = ReleaseService(GitApiMock(), BuildFileRepositoryMock("build.py"))
sut = ReleaseService(
GitApiMock(), ArtifactDeploymentApiMock(), BuildFileRepositoryMock("build.py")
)
devops = build_devops({})
release = devops.mixins[MixinType.RELEASE]
sut.update_release_type(release, "MAJOR")
@ -20,3 +24,40 @@ def test_sould_update_release_type():
with pytest.raises(Exception):
sut.update_release_type(release, "NOT_EXISTING")
def test_sould_publish_artifacts():
mock = ArtifactDeploymentApiMock(release='{"id": 2345}')
sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock())
devops = build_devops(
{
"release_artifacts": ["target/art"],
"release_artifact_server_url": "http://repo.test/",
"release_organisation": "orga",
"release_repository_name": "repo",
}
)
release = devops.mixins[MixinType.RELEASE]
sut.publish_artifacts(release)
assert "http://repo.test/api/v1/repos/orga/repo/releases/2345/assets" == mock.add_asset_to_release_api_endpoint
def test_sould_throw_exception_if_there_was_an_error_in_publish_artifacts():
devops = build_devops(
{
"release_artifacts": ["target/art"],
"release_artifact_server_url": "http://repo.test/",
"release_organisation": "orga",
"release_repository_name": "repo",
}
)
release = devops.mixins[MixinType.RELEASE]
with pytest.raises(Exception):
mock = ArtifactDeploymentApiMock(release='')
sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock())
sut.publish_artifacts(release)
with pytest.raises(Exception):
mock = ArtifactDeploymentApiMock(release='{"message": "there was an error", "url":"some-url"}')
sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock())
sut.publish_artifacts(release)

View file

@ -53,6 +53,11 @@ def devops_config(overrides: dict) -> dict:
"release_current_branch": "my_feature",
"release_primary_build_file": "./package.json",
"release_secondary_build_file": [],
"release_artifacts": [],
"release_artifact_token": "release_artifact_token",
"release_artifact_server_url": None,
"release_organisation": None,
"release_repository_name": None,
"credentials_mappings": [
{
"gopass_path": "a/path",
@ -99,6 +104,9 @@ class EnvironmentApiMock:
def get(self, key):
return self.mappings.get(key, None)
def is_defined(self, key):
return key in self.mappings
class CredentialsApiMock:
def __init__(self, mappings):
@ -150,3 +158,28 @@ class GitApiMock:
def checkout(self, branch: str):
pass
class ArtifactDeploymentApiMock:
def __init__(self, release=""):
self.release = release
self.create_forgejo_release_count = 0
self.add_asset_to_release_count = 0
self.add_asset_to_release_api_endpoint = ""
def create_forgejo_release(self, api_endpoint: str, tag: str, token: str):
self.create_forgejo_release_count += 1
return self.release
def add_asset_to_release(
self, api_endpoint: str, attachment: str, attachment_type: str, token: str
):
self.add_asset_to_release_api_endpoint = api_endpoint
self.add_asset_to_release_count += 1
pass
def calculate_sha256(self, path: Path):
return f"{path}.sha256"
def calculate_sha512(self, path: Path):
return f"{path}.sha512"

View file

@ -0,0 +1,32 @@
import pytest
from pybuilder.core import Project
from pathlib import Path
from src.main.python.ddadevops.domain import (
Validateable,
DnsRecord,
Devops,
BuildType,
MixinType,
Artifact,
Image,
)
from .helper import build_devops, devops_config
def test_sould_validate_release():
sut = Artifact("x")
assert sut.is_valid()
sut = Artifact(None)
assert not sut.is_valid()
def test_should_calculate_type():
sut = Artifact("x.jar")
assert "application/x-java-archive" == sut.type()
sut = Artifact("x.js")
assert "application/x-javascript" == sut.type()
sut = Artifact("x.jar.sha256")
assert "text/plain" == sut.type()

View file

@ -4,6 +4,7 @@ from src.main.python.ddadevops.domain import (
Version,
BuildType,
MixinType,
Artifact,
)
@ -50,6 +51,7 @@ def test_devops_creation():
assert sut is not None
assert sut.specialized_builds[BuildType.C4K] is not None
def test_release_devops_creation():
sut = DevopsFactory().build_devops(
{
"stage": "test",
@ -67,6 +69,30 @@ def test_devops_creation():
assert sut is not None
assert sut.mixins[MixinType.RELEASE] is not None
sut = DevopsFactory().build_devops(
{
"stage": "test",
"name": "mybuild",
"module": "test_image",
"project_root_path": "../../..",
"build_types": [],
"mixin_types": ["RELEASE"],
"release_main_branch": "main",
"release_current_branch": "my_feature",
"release_config_file": "project.clj",
"release_artifacts": ["x.jar"],
"release_artifact_token": "y",
"release_artifact_server_url": "https://repo.prod.meissa.de",
"release_organisation": "meissa",
"release_repository_name": "provs",
},
Version.from_str("1.0.0", "SNAPSHOT"),
)
release = sut.mixins[MixinType.RELEASE]
assert release is not None
assert Artifact("x.jar") == release.release_artifacts[0]
def test_on_merge_input_should_win():
sut = DevopsFactory()

View file

@ -1,3 +1,4 @@
import pytest
from pybuilder.core import Project
from pathlib import Path
from src.main.python.ddadevops.domain import (
@ -61,3 +62,74 @@ def test_sould_calculate_build_files():
Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"),
)
assert ["project.clj", "package.json"] == sut.build_files()
def test_should_calculate_forgejo_release_api_endpoint():
sut = Release(
devops_config(
{
"release_artifacts": [],
"release_artifact_token": "y",
"release_artifact_server_url": "https://repo.prod.meissa.de",
"release_organisation": "meissa",
"release_repository_name": "provs",
}
),
Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"),
)
assert (
"https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases"
== sut.forgejo_release_api_endpoint()
)
sut = Release(
devops_config(
{
"release_artifacts": ["x"],
"release_artifact_token": "y",
"release_artifact_server_url": "https://repo.prod.meissa.de/",
"release_organisation": "/meissa/",
"release_repository_name": "provs",
}
),
Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"),
)
assert (
"https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases"
== sut.forgejo_release_api_endpoint()
)
assert(
"/meissa/"
== sut.release_organisation
)
with pytest.raises(Exception):
sut = Release(
devops_config(
{
"release_artifact_server_url": "https://repo.prod.meissa.de",
"release_organisation": None,
"release_repository_name": "provs",
}
),
Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"),
)
sut.forgejo_release_api_endpoint()
def test_should_calculate_forgejo_release_asset_api_endpoint():
sut = Release(
devops_config(
{
"release_artifacts": ["x"],
"release_artifact_token": "y",
"release_artifact_server_url": "https://repo.prod.meissa.de",
"release_organisation": "meissa",
"release_repository_name": "provs",
}
),
Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"),
)
assert (
"https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases/123/assets"
== sut.forgejo_release_asset_api_endpoint(123)
)

View file

@ -14,6 +14,8 @@ def test_release_mixin(tmp_path):
copy_resource(Path("package.json"), tmp_path)
project = Project(str_tmp_path, name="name")
os.environ["RELEASE_ARTIFACT_TOKEN"] = "ratoken"
sut = ReleaseMixin(
project,
devops_config(