From b2e807d5b59dea93c71c6036eff3a802f682f1fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M=C2=AA=20Fern=C3=A1ndez?= Date: Tue, 5 Sep 2023 11:31:04 +0200 Subject: [PATCH] Removed code duplication between podman_container and docker_container. This work is needed to improve maintainability of both, as they share the same code to compress saved images on the fly, and decompress on the fly container images being loaded into the local registry. Also, switched container image compression from lzma to pgzip. Images are bigger, but the parallelized version of gzip library is much faster. --- mypy-stubs/pgzip/__init__.pyi | 8 + mypy-stubs/pgzip/pgzip.pyi | 93 ++++++ requirements.txt | 4 +- wfexs_backend/abstract_docker_container.py | 351 +++++++++++++++++++++ wfexs_backend/container.py | 42 --- wfexs_backend/docker_container.py | 173 +--------- wfexs_backend/podman_container.py | 196 +----------- 7 files changed, 466 insertions(+), 401 deletions(-) create mode 100644 mypy-stubs/pgzip/__init__.pyi create mode 100644 mypy-stubs/pgzip/pgzip.pyi create mode 100644 wfexs_backend/abstract_docker_container.py diff --git a/mypy-stubs/pgzip/__init__.pyi b/mypy-stubs/pgzip/__init__.pyi new file mode 100644 index 00000000..e7312be3 --- /dev/null +++ b/mypy-stubs/pgzip/__init__.pyi @@ -0,0 +1,8 @@ +from .pgzip import ( + PgzipFile, + compress as compress, + decompress as decompress, + open as open, +) + +GzipFile = PgzipFile diff --git a/mypy-stubs/pgzip/pgzip.pyi b/mypy-stubs/pgzip/pgzip.pyi new file mode 100644 index 00000000..042b1716 --- /dev/null +++ b/mypy-stubs/pgzip/pgzip.pyi @@ -0,0 +1,93 @@ +from concurrent.futures import ( + Future, + ThreadPoolExecutor, +) +from gzip import GzipFile, _GzipReader +from io import FileIO +from typing import ( + Any, + IO, + Iterable, + MutableSequence, + Optional, + Sequence, + Tuple, +) +from typing_extensions import ( + Buffer, + Literal, +) +import zlib + +SID: bytes + +def open( + filename: str, + mode: str = ..., + compresslevel: int = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + newline: Optional[str] = ..., + thread: Optional[int] = ..., + blocksize: int = ..., +) -> IO[bytes]: ... +def compress( + data: bytes, + compresslevel: int = ..., + thread: Optional[int] = ..., + blocksize: int = ..., +) -> bytes: ... +def decompress( + data: bytes, thread: Optional[int] = ..., blocksize: int = ... +) -> bytes: ... +def padded_file_seek(self: PgzipFile, off: int, whence: int = ...) -> int: ... + +class PgzipFile(GzipFile): + thread: int + read_blocks: Optional[Any] + mode: Literal[1, 2] + raw: _MulitGzipReader + name: str + index: MutableSequence[MutableSequence[int]] + compress: zlib._Compress + compresslevel: int + blocksize: int + pool: ThreadPoolExecutor + pool_result: MutableSequence[Future[Tuple[bytes, bytes, bytes, int, int]]] + small_buf: IO[bytes] + fileobj: IO[bytes] + def __init__( + self, + filename: Optional[str] = ..., + mode: Optional[str] = ..., + compresslevel: int = ..., + fileobj: Optional[IO[bytes]] = ..., + mtime: Optional[float] = ..., + thread: Optional[int] = ..., + blocksize: int = ..., + ) -> None: ... + def write(self, data: Buffer) -> int: ... + def get_index(self) -> MutableSequence[MutableSequence[int]]: ... + def show_index(self) -> None: ... + def build_index( + self, idx_file: Optional[str] = ... + ) -> MutableSequence[MutableSequence[int]]: ... + def load_index(self, idx_file: str) -> MutableSequence[MutableSequence[int]]: ... + def set_read_blocks(self, block_ids: Sequence[int]) -> None: ... + def set_read_blocks_by_name(self, block_names: Sequence[str]) -> None: ... + def clear_read_blocks(self) -> None: ... + myfileobj: FileIO + def close(self) -> None: ... + def flush(self, zlib_mode: int = ...) -> None: ... + +class _MulitGzipReader(_GzipReader): + memberidx: MutableSequence[Tuple[int, int]] + max_block_size: int + thread: int + block_start_iter: Optional[Iterable[int]] + def __init__( + self, fp: Any, thread: int = ..., max_block_size: int = ... + ) -> None: ... + def read(self, size: int = ...) -> bytes: ... + def set_block_iter(self, block_start_list: Sequence[int]) -> None: ... + def clear_block_iter(self) -> None: ... diff --git a/requirements.txt b/requirements.txt index 545fd38f..a06939ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,10 +17,10 @@ rocrate >= 0.7.0 boto3 botocore google-cloud-storage -typing_extensions pyncclient-ext >= 0.1.2 wiktionary-term-fetcher >= 0.1.1 funny-passphrase >= 0.2.3 pyxdg groovy-parser == 0.1.1 -data-url \ No newline at end of file +data-url +pgzip \ No newline at end of file diff --git a/wfexs_backend/abstract_docker_container.py b/wfexs_backend/abstract_docker_container.py new file mode 100644 index 00000000..2d448346 --- /dev/null +++ b/wfexs_backend/abstract_docker_container.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import + +import copy +import lzma +import os +import tempfile +import atexit +import platform +import shutil +import subprocess +import abc +import logging +import inspect + +from typing import ( + cast, + TYPE_CHECKING, +) + +import magic +import pgzip + +from .common import ( + AbstractWfExSException, +) + +if TYPE_CHECKING: + from types import ( + ModuleType, + ) + + from typing import ( + Any, + IO, + Mapping, + MutableMapping, + MutableSequence, + Optional, + Sequence, + Set, + Tuple, + Union, + ) + + from typing_extensions import ( + TypeAlias, + TypedDict, + Final, + ) + + from .common import ( + AbsPath, + AnyPath, + Container, + ContainerEngineVersionStr, + ContainerFileNamingMethod, + ContainerLocalConfig, + ContainerOperatingSystem, + ContainerTaggedName, + ExitVal, + Fingerprint, + ProcessorArchitecture, + RelPath, + ) + + DockerLikeManifest: TypeAlias = Mapping[str, Any] + MutableDockerLikeManifest: TypeAlias = MutableMapping[str, Any] + + class DockerManifestMetadata(TypedDict): + image_id: "Fingerprint" + image_signature: "Fingerprint" + manifests_signature: "Fingerprint" + manifests: "Sequence[DockerLikeManifest]" + + +from . import common +from .container import ( + ContainerFactory, + ContainerFactoryException, +) +from .utils.digests import ComputeDigestFromObject + + +class AbstractDockerContainerFactory(ContainerFactory): + ACCEPTED_CONTAINER_TYPES = set( + ( + common.ContainerType.Docker, + common.ContainerType.UDocker, + common.ContainerType.Podman, + ) + ) + + @classmethod + def AcceptsContainerType( + cls, container_type: "Union[common.ContainerType, Set[common.ContainerType]]" + ) -> "bool": + return not cls.ACCEPTED_CONTAINER_TYPES.isdisjoint( + container_type if isinstance(container_type, set) else (container_type,) + ) + + @classmethod + @abc.abstractmethod + def trimmable_manifest_keys(cls) -> "Sequence[str]": + pass + + def _gen_trimmed_manifests_signature( + self, manifests: "Sequence[DockerLikeManifest]" + ) -> "Fingerprint": + trimmed_manifests: "MutableSequence[DockerLikeManifest]" = [] + some_trimmed = False + for manifest in manifests: + # Copy the manifest + trimmed_manifest = cast("MutableDockerLikeManifest", copy.copy(manifest)) + # And trim the keys + for key in self.trimmable_manifest_keys(): + if key in trimmed_manifest: + del trimmed_manifest[key] + some_trimmed = True + + trimmed_manifests.append(trimmed_manifest) + + return ComputeDigestFromObject(trimmed_manifests if some_trimmed else manifests) + + @classmethod + @abc.abstractmethod + def variant_name(cls) -> "str": + pass + + def _images(self, matEnv: "Mapping[str, str]") -> "Tuple[ExitVal, str, str]": + with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: + self.logger.debug(f"querying available {self.variant_name()} containers") + d_retval = subprocess.Popen( + [self.runtime_cmd, "images"], + env=matEnv, + stdout=d_out, + stderr=d_err, + ).wait() + + self.logger.debug(f"{self.variant_name()} images retval: {d_retval}") + + with open(d_out.name, mode="rb") as c_stF: + d_out_v = c_stF.read().decode("utf-8", errors="continue") + with open(d_err.name, mode="rb") as c_stF: + d_err_v = c_stF.read().decode("utf-8", errors="continue") + + self.logger.debug(f"{self.variant_name()} inspect stdout: {d_out_v}") + + self.logger.debug(f"{self.variant_name()} inspect stderr: {d_err_v}") + + return cast("ExitVal", d_retval), d_out_v, d_err_v + + def _inspect( + self, dockerTag: "str", matEnv: "Mapping[str, str]" + ) -> "Tuple[ExitVal, str, str]": + with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: + self.logger.debug(f"querying {self.variant_name()} container {dockerTag}") + d_retval = subprocess.Popen( + [self.runtime_cmd, "inspect", dockerTag], + env=matEnv, + stdout=d_out, + stderr=d_err, + ).wait() + + self.logger.debug( + f"{self.variant_name()} inspect {dockerTag} retval: {d_retval}" + ) + + with open(d_out.name, mode="rb") as c_stF: + d_out_v = c_stF.read().decode("utf-8", errors="continue") + with open(d_err.name, mode="rb") as c_stF: + d_err_v = c_stF.read().decode("utf-8", errors="continue") + + self.logger.debug(f"{self.variant_name()} inspect stdout: {d_out_v}") + + self.logger.debug(f"{self.variant_name()} inspect stderr: {d_err_v}") + + return cast("ExitVal", d_retval), d_out_v, d_err_v + + def _pull( + self, dockerTag: "str", matEnv: "Mapping[str, str]" + ) -> "Tuple[ExitVal, str, str]": + with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: + self.logger.debug(f"pulling {self.variant_name()} container {dockerTag}") + d_retval = subprocess.Popen( + [self.runtime_cmd, "pull", dockerTag], + env=matEnv, + stdout=d_out, + stderr=d_err, + ).wait() + + self.logger.debug( + f"{self.variant_name()} pull {dockerTag} retval: {d_retval}" + ) + + with open(d_out.name, mode="r") as c_stF: + d_out_v = c_stF.read() + with open(d_err.name, "r") as c_stF: + d_err_v = c_stF.read() + + self.logger.debug(f"{self.variant_name()} pull stdout: {d_out_v}") + + self.logger.debug(f"{self.variant_name()} pull stderr: {d_err_v}") + + return cast("ExitVal", d_retval), d_out_v, d_err_v + + def _rmi( + self, dockerTag: "str", matEnv: "Mapping[str, str]" + ) -> "Tuple[ExitVal, str, str]": + with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: + self.logger.debug(f"removing {self.variant_name()} container {dockerTag}") + d_retval = subprocess.Popen( + [self.runtime_cmd, "rmi", dockerTag], + env=matEnv, + stdout=d_out, + stderr=d_err, + ).wait() + + self.logger.debug( + f"{self.variant_name()} rmi {dockerTag} retval: {d_retval}" + ) + + with open(d_out.name, mode="r") as c_stF: + d_out_v = c_stF.read() + with open(d_err.name, "r") as c_stF: + d_err_v = c_stF.read() + + self.logger.debug(f"{self.variant_name()} rmi stdout: {d_out_v}") + + self.logger.debug(f"{self.variant_name()} rmi stderr: {d_err_v}") + + return cast("ExitVal", d_retval), d_out_v, d_err_v + + def _load( + self, + archivefile: "AbsPath", + dockerTag: "str", + matEnv: "Mapping[str, str]", + ) -> "Tuple[ExitVal, str, str]": + foundmime = magic.from_file(archivefile, mime=True) + package: "ModuleType" + if foundmime == "application/x-xz": + package = lzma + elif foundmime == "application/gzip": + package = pgzip + else: + raise ContainerFactoryException( + f"Unknown {self.variant_name()} archive compression format: {foundmime}" + ) + + with package.open( + archivefile, mode="rb" + ) as d_in, tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: + self.logger.debug(f"loading {self.variant_name()} container {dockerTag}") + with subprocess.Popen( + [self.runtime_cmd, "load"], + env=matEnv, + stdin=d_in, + stdout=d_out, + stderr=d_err, + ) as sp: + d_retval = sp.wait() + + self.logger.debug( + f"{self.variant_name()} load {dockerTag} retval: {d_retval}" + ) + + with open(d_out.name, "r") as c_stF: + d_out_v = c_stF.read() + + self.logger.debug(f"{self.variant_name()} load stdout: {d_out_v}") + + with open(d_err.name, "r") as c_stF: + d_err_v = c_stF.read() + + self.logger.debug(f"{self.variant_name()} load stderr: {d_err_v}") + + return cast("ExitVal", d_retval), d_out_v, d_err_v + + def _save( + self, + dockerTag: "str", + destfile: "AbsPath", + matEnv: "Mapping[str, str]", + ) -> "Tuple[ExitVal, str]": + with pgzip.open( + destfile, mode="wb" + ) as d_out, tempfile.NamedTemporaryFile() as d_err: + self.logger.debug(f"saving {self.variant_name()} container {dockerTag}") + with subprocess.Popen( + [self.runtime_cmd, "save", dockerTag], + env=matEnv, + stdout=subprocess.PIPE, + stderr=d_err, + ) as sp: + if sp.stdout is not None: + shutil.copyfileobj( + cast("IO[str]", sp.stdout), d_out, length=1024 * 1024 + ) + d_retval = sp.wait() + + self.logger.debug( + f"{self.variant_name()} save {dockerTag} retval: {d_retval}" + ) + + with open(d_err.name, "r") as c_stF: + d_err_v = c_stF.read() + + self.logger.debug(f"{self.variant_name()} save stderr: {d_err_v}") + + return cast("ExitVal", d_retval), d_err_v + + def _version( + self, + ) -> "Tuple[ExitVal, str, str]": + with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: + self.logger.debug(f"querying {self.variant_name()} version and details") + d_retval = subprocess.Popen( + [self.runtime_cmd, "version", "--format", "{{json .}}"], + stdout=d_out, + stderr=d_err, + ).wait() + + self.logger.debug(f"{self.variant_name()} version retval: {d_retval}") + + with open(d_out.name, mode="r") as c_stF: + d_out_v = c_stF.read() + with open(d_err.name, "r") as c_stF: + d_err_v = c_stF.read() + + self.logger.debug(f"{self.variant_name()} version stdout: {d_out_v}") + + self.logger.debug(f"{self.variant_name()} version stderr: {d_err_v}") + + return cast("ExitVal", d_retval), d_out_v, d_err_v diff --git a/wfexs_backend/container.py b/wfexs_backend/container.py index b9f7a970..370ae016 100644 --- a/wfexs_backend/container.py +++ b/wfexs_backend/container.py @@ -81,7 +81,6 @@ class DockerManifestMetadata(TypedDict): from . import common -from .utils.digests import ComputeDigestFromObject class ContainerFactoryException(AbstractWfExSException): @@ -428,44 +427,3 @@ def deploySingleContainer( This is a no-op """ return False - - -class AbstractDockerContainerFactory(ContainerFactory): - ACCEPTED_CONTAINER_TYPES = set( - ( - common.ContainerType.Docker, - common.ContainerType.UDocker, - common.ContainerType.Podman, - ) - ) - - @classmethod - def AcceptsContainerType( - cls, container_type: "Union[common.ContainerType, Set[common.ContainerType]]" - ) -> "bool": - return not cls.ACCEPTED_CONTAINER_TYPES.isdisjoint( - container_type if isinstance(container_type, set) else (container_type,) - ) - - @classmethod - @abc.abstractmethod - def trimmable_manifest_keys(cls) -> "Sequence[str]": - pass - - def _gen_trimmed_manifests_signature( - self, manifests: "Sequence[DockerLikeManifest]" - ) -> "Fingerprint": - trimmed_manifests: "MutableSequence[DockerLikeManifest]" = [] - some_trimmed = False - for manifest in manifests: - # Copy the manifest - trimmed_manifest = cast("MutableDockerLikeManifest", copy.copy(manifest)) - # And trim the keys - for key in self.trimmable_manifest_keys(): - if key in trimmed_manifest: - del trimmed_manifest[key] - some_trimmed = True - - trimmed_manifests.append(trimmed_manifest) - - return ComputeDigestFromObject(trimmed_manifests if some_trimmed else manifests) diff --git a/wfexs_backend/docker_container.py b/wfexs_backend/docker_container.py index 3ac746d3..0db103cd 100644 --- a/wfexs_backend/docker_container.py +++ b/wfexs_backend/docker_container.py @@ -18,16 +18,11 @@ from __future__ import absolute_import import json -import lzma import os -import shutil -import subprocess -import tempfile from typing import ( cast, TYPE_CHECKING, ) -import uuid if TYPE_CHECKING: from typing import ( @@ -50,7 +45,6 @@ ContainerLocalConfig, ContainerOperatingSystem, ContainerTaggedName, - ExitVal, Fingerprint, ProcessorArchitecture, RelPath, @@ -68,10 +62,12 @@ ) from .container import ( - AbstractDockerContainerFactory, ContainerEngineException, ContainerFactoryException, ) +from .abstract_docker_container import ( + AbstractDockerContainerFactory, +) from .utils.contents import ( link_or_copy, real_unlink_if_exists, @@ -112,166 +108,9 @@ def __init__( def ContainerType(cls) -> "ContainerType": return ContainerType.Docker - def _inspect( - self, dockerTag: "str", matEnv: "Mapping[str, str]" - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"querying docker container {dockerTag}") - d_retval = subprocess.Popen( - [self.runtime_cmd, "inspect", dockerTag], - env=matEnv, - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"docker inspect {dockerTag} retval: {d_retval}") - - with open(d_out.name, mode="rb") as c_stF: - d_out_v = c_stF.read().decode("utf-8", errors="continue") - with open(d_err.name, mode="rb") as c_stF: - d_err_v = c_stF.read().decode("utf-8", errors="continue") - - self.logger.debug(f"docker inspect stdout: {d_out_v}") - - self.logger.debug(f"docker inspect stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _pull( - self, dockerTag: "str", matEnv: "Mapping[str, str]" - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"pulling docker container {dockerTag}") - d_retval = subprocess.Popen( - [self.runtime_cmd, "pull", dockerTag], - env=matEnv, - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"docker pull {dockerTag} retval: {d_retval}") - - with open(d_out.name, mode="r") as c_stF: - d_out_v = c_stF.read() - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"docker pull stdout: {d_out_v}") - - self.logger.debug(f"docker pull stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _rmi( - self, dockerTag: "str", matEnv: "Mapping[str, str]" - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"removing docker container {dockerTag}") - d_retval = subprocess.Popen( - [self.runtime_cmd, "rmi", dockerTag], - env=matEnv, - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"docker rmi {dockerTag} retval: {d_retval}") - - with open(d_out.name, mode="r") as c_stF: - d_out_v = c_stF.read() - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"docker rmi stdout: {d_out_v}") - - self.logger.debug(f"docker rmi stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _load( - self, - archivefile: "AbsPath", - dockerTag: "str", - matEnv: "Mapping[str, str]", - ) -> "Tuple[ExitVal, str, str]": - with lzma.open( - archivefile, mode="rb" - ) as d_in, tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"loading docker container {dockerTag}") - with subprocess.Popen( - [self.runtime_cmd, "load"], - env=matEnv, - stdin=d_in, - stdout=d_out, - stderr=d_err, - ) as sp: - d_retval = sp.wait() - - self.logger.debug(f"docker load {dockerTag} retval: {d_retval}") - - with open(d_out.name, "r") as c_stF: - d_out_v = c_stF.read() - - self.logger.debug(f"docker load stdout: {d_out_v}") - - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"docker load stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _save( - self, - dockerTag: "str", - destfile: "AbsPath", - matEnv: "Mapping[str, str]", - ) -> "Tuple[ExitVal, str]": - with lzma.open( - destfile, mode="wb" - ) as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"saving docker container {dockerTag}") - with subprocess.Popen( - [self.runtime_cmd, "save", dockerTag], - env=matEnv, - stdout=subprocess.PIPE, - stderr=d_err, - ) as sp: - if sp.stdout is not None: - shutil.copyfileobj(sp.stdout, d_out, 1024 * 1024) - d_retval = sp.wait() - - self.logger.debug(f"docker save {dockerTag} retval: {d_retval}") - - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"docker save stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_err_v - - def _version( - self, - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"querying docker version and details") - d_retval = subprocess.Popen( - [self.runtime_cmd, "version", "--format", "{{json .}}"], - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"docker version retval: {d_retval}") - - with open(d_out.name, mode="r") as c_stF: - d_out_v = c_stF.read() - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"docker version stdout: {d_out_v}") - - self.logger.debug(f"docker version stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v + @classmethod + def variant_name(self) -> "str": + return "docker" @property def architecture(self) -> "Tuple[ContainerOperatingSystem, ProcessorArchitecture]": diff --git a/wfexs_backend/podman_container.py b/wfexs_backend/podman_container.py index 2df40e13..c51c06a6 100644 --- a/wfexs_backend/podman_container.py +++ b/wfexs_backend/podman_container.py @@ -18,16 +18,11 @@ from __future__ import absolute_import import json -import lzma import os -import shutil -import subprocess -import tempfile from typing import ( cast, TYPE_CHECKING, ) -import uuid if TYPE_CHECKING: from typing import ( @@ -50,7 +45,6 @@ ContainerLocalConfig, ContainerOperatingSystem, ContainerTaggedName, - ExitVal, Fingerprint, ProcessorArchitecture, RelPath, @@ -67,10 +61,12 @@ ContainerType, ) from .container import ( - AbstractDockerContainerFactory, ContainerEngineException, ContainerFactoryException, ) +from .abstract_docker_container import ( + AbstractDockerContainerFactory, +) from .utils.contents import ( link_or_copy, real_unlink_if_exists, @@ -128,189 +124,9 @@ def __init__( def ContainerType(cls) -> "ContainerType": return ContainerType.Podman - def _images(self, matEnv: "Mapping[str, str]") -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug("querying available podman containers") - d_retval = subprocess.Popen( - [self.runtime_cmd, "images"], - env=matEnv, - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"podman images retval: {d_retval}") - - with open(d_out.name, mode="rb") as c_stF: - d_out_v = c_stF.read().decode("utf-8", errors="continue") - with open(d_err.name, mode="rb") as c_stF: - d_err_v = c_stF.read().decode("utf-8", errors="continue") - - self.logger.debug(f"podman inspect stdout: {d_out_v}") - - self.logger.debug(f"podman inspect stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _inspect( - self, dockerTag: "str", matEnv: "Mapping[str, str]" - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"querying podman container {dockerTag}") - d_retval = subprocess.Popen( - [self.runtime_cmd, "inspect", dockerTag], - env=matEnv, - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"podman inspect {dockerTag} retval: {d_retval}") - - with open(d_out.name, mode="rb") as c_stF: - d_out_v = c_stF.read().decode("utf-8", errors="continue") - with open(d_err.name, mode="rb") as c_stF: - d_err_v = c_stF.read().decode("utf-8", errors="continue") - - self.logger.debug(f"podman inspect stdout: {d_out_v}") - - self.logger.debug(f"podman inspect stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _pull( - self, dockerTag: "str", matEnv: "Mapping[str, str]" - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"pulling podman container {dockerTag}") - d_retval = subprocess.Popen( - [self.runtime_cmd, "pull", dockerTag], - env=matEnv, - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"podman pull {dockerTag} retval: {d_retval}") - - with open(d_out.name, mode="r") as c_stF: - d_out_v = c_stF.read() - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"podman pull stdout: {d_out_v}") - - self.logger.debug(f"podman pull stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _rmi( - self, dockerTag: "str", matEnv: "Mapping[str, str]" - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"removing podman container {dockerTag}") - d_retval = subprocess.Popen( - [self.runtime_cmd, "rmi", dockerTag], - env=matEnv, - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"podman rmi {dockerTag} retval: {d_retval}") - - with open(d_out.name, mode="r") as c_stF: - d_out_v = c_stF.read() - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"podman rmi stdout: {d_out_v}") - - self.logger.debug(f"podman rmi stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _load( - self, - archivefile: "AbsPath", - dockerTag: "str", - matEnv: "Mapping[str, str]", - ) -> "Tuple[ExitVal, str, str]": - with lzma.open( - archivefile, mode="rb" - ) as d_in, tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"loading podman container {dockerTag}") - with subprocess.Popen( - [self.runtime_cmd, "load"], - env=matEnv, - stdin=d_in, - stdout=d_out, - stderr=d_err, - ) as sp: - d_retval = sp.wait() - - self.logger.debug(f"podman load {dockerTag} retval: {d_retval}") - - with open(d_out.name, "r") as c_stF: - d_out_v = c_stF.read() - - self.logger.debug(f"podman load stdout: {d_out_v}") - - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"podman load stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v - - def _save( - self, - dockerTag: "str", - destfile: "AbsPath", - matEnv: "Mapping[str, str]", - ) -> "Tuple[ExitVal, str]": - with lzma.open( - destfile, mode="wb" - ) as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"saving podman container {dockerTag}") - with subprocess.Popen( - [self.runtime_cmd, "save", dockerTag], - env=matEnv, - stdout=subprocess.PIPE, - stderr=d_err, - ) as sp: - if sp.stdout is not None: - shutil.copyfileobj(sp.stdout, d_out, length=1024 * 1024) - d_retval = sp.wait() - - self.logger.debug(f"podman save {dockerTag} retval: {d_retval}") - - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"podman save stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_err_v - - def _version( - self, - ) -> "Tuple[ExitVal, str, str]": - with tempfile.NamedTemporaryFile() as d_out, tempfile.NamedTemporaryFile() as d_err: - self.logger.debug(f"querying podman version and details") - d_retval = subprocess.Popen( - [self.runtime_cmd, "version", "--format", "{{json .}}"], - stdout=d_out, - stderr=d_err, - ).wait() - - self.logger.debug(f"podman version retval: {d_retval}") - - with open(d_out.name, mode="r") as c_stF: - d_out_v = c_stF.read() - with open(d_err.name, "r") as c_stF: - d_err_v = c_stF.read() - - self.logger.debug(f"podman version stdout: {d_out_v}") - - self.logger.debug(f"podman version stderr: {d_err_v}") - - return cast("ExitVal", d_retval), d_out_v, d_err_v + @classmethod + def variant_name(self) -> "str": + return "podman" @property def architecture(self) -> "Tuple[ContainerOperatingSystem, ProcessorArchitecture]":