diff options
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/buildcache.py | 25 | ||||
| -rw-r--r-- | lib/spack/spack/oci/oci.py | 8 | ||||
| -rw-r--r-- | lib/spack/spack/test/oci/integration_test.py | 97 | ||||
| -rw-r--r-- | lib/spack/spack/test/oci/mock_registry.py | 10 |
5 files changed, 125 insertions, 17 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 6f401e4a97..e3075c5fbd 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -1541,7 +1541,7 @@ def download_tarball(spec, unsigned: Optional[bool] = False, mirrors_for_spec=No response = spack.oci.opener.urlopen( urllib.request.Request( url=ref.manifest_url(), - headers={"Accept": "application/vnd.oci.image.manifest.v1+json"}, + headers={"Accept": ", ".join(spack.oci.oci.manifest_content_type)}, ) ) except Exception: diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 46279cd06a..8a2398b705 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -594,6 +594,15 @@ def _put_manifest( base_manifest, base_config = base_images[architecture] env = _retrieve_env_dict_from_config(base_config) + # If the base image uses `vnd.docker.distribution.manifest.v2+json`, then we use that too. + # This is because Singularity / Apptainer is very strict about not mixing them. + base_manifest_mediaType = base_manifest.get( + "mediaType", "application/vnd.oci.image.manifest.v1+json" + ) + use_docker_format = ( + base_manifest_mediaType == "application/vnd.docker.distribution.manifest.v2+json" + ) + spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env) # Create an oci.image.config file @@ -625,8 +634,8 @@ def _put_manifest( # Upload the config file upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum) - oci_manifest = { - "mediaType": "application/vnd.oci.image.manifest.v1+json", + manifest = { + "mediaType": base_manifest_mediaType, "schemaVersion": 2, "config": { "mediaType": base_manifest["config"]["mediaType"], @@ -637,7 +646,11 @@ def _put_manifest( *(layer for layer in base_manifest["layers"]), *( { - "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "mediaType": ( + "application/vnd.docker.image.rootfs.diff.tar.gzip" + if use_docker_format + else "application/vnd.oci.image.layer.v1.tar+gzip" + ), "digest": str(checksums[s.dag_hash()].compressed_digest), "size": checksums[s.dag_hash()].size, } @@ -646,11 +659,11 @@ def _put_manifest( ], } - if annotations: - oci_manifest["annotations"] = annotations + if not use_docker_format and annotations: + manifest["annotations"] = annotations # Finally upload the manifest - upload_manifest_with_retry(image_ref, oci_manifest=oci_manifest) + upload_manifest_with_retry(image_ref, manifest=manifest) # delete the config file os.unlink(config_file) diff --git a/lib/spack/spack/oci/oci.py b/lib/spack/spack/oci/oci.py index ce9e3b9ded..fefc66674e 100644 --- a/lib/spack/spack/oci/oci.py +++ b/lib/spack/spack/oci/oci.py @@ -161,7 +161,7 @@ def upload_blob( def upload_manifest( ref: ImageReference, - oci_manifest: dict, + manifest: dict, tag: bool = True, _urlopen: spack.oci.opener.MaybeOpen = None, ): @@ -169,7 +169,7 @@ def upload_manifest( Args: ref: The image reference. - oci_manifest: The OCI manifest or index. + manifest: The manifest or index. tag: When true, use the tag, otherwise use the digest, this is relevant for multi-arch images, where the tag is an index, referencing the manifests by digest. @@ -179,7 +179,7 @@ def upload_manifest( """ _urlopen = _urlopen or spack.oci.opener.urlopen - data = json.dumps(oci_manifest, separators=(",", ":")).encode() + data = json.dumps(manifest, separators=(",", ":")).encode() digest = Digest.from_sha256(hashlib.sha256(data).hexdigest()) size = len(data) @@ -190,7 +190,7 @@ def upload_manifest( url=ref.manifest_url(), method="PUT", data=data, - headers={"Content-Type": oci_manifest["mediaType"]}, + headers={"Content-Type": manifest["mediaType"]}, ) response = _urlopen(request) diff --git a/lib/spack/spack/test/oci/integration_test.py b/lib/spack/spack/test/oci/integration_test.py index 8129dd22cf..c4e2636619 100644 --- a/lib/spack/spack/test/oci/integration_test.py +++ b/lib/spack/spack/test/oci/integration_test.py @@ -9,6 +9,7 @@ import hashlib import json import os +import pathlib from contextlib import contextmanager import spack.environment as ev @@ -172,6 +173,12 @@ def test_buildcache_push_with_base_image_command( dst_image = ImageReference.from_string(f"dst.example.com/image:{tag}") retrieved_manifest, retrieved_config = get_manifest_and_config(dst_image) + # Check that the media type is OCI + assert retrieved_manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json" + assert ( + retrieved_manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json" + ) + # Check that the base image layer is first. assert retrieved_manifest["layers"][0]["digest"] == str(tar_gz_digest) assert retrieved_config["rootfs"]["diff_ids"][0] == str(tar_digest) @@ -189,3 +196,93 @@ def test_buildcache_push_with_base_image_command( # And verify that all layers including the base layer are present for layer in retrieved_manifest["layers"]: assert blob_exists(dst_image, digest=Digest.from_string(layer["digest"])) + assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip" + + +def test_uploading_with_base_image_in_docker_image_manifest_v2_format( + tmp_path: pathlib.Path, mutable_database, disable_parallel_buildcache_push +): + """If the base image uses an old manifest schema, Spack should also use that. + That is necessary for container images to work with Apptainer, which is rather strict about + mismatching manifest/layer types.""" + + registry_src = InMemoryOCIRegistry("src.example.com") + registry_dst = InMemoryOCIRegistry("dst.example.com") + + base_image = ImageReference.from_string("src.example.com/my-base-image:latest") + + with oci_servers(registry_src, registry_dst): + mirror("add", "oci-test", "oci://dst.example.com/image") + + # Create a dummy base image (blob, config, manifest) in registry A in the Docker Image + # Manifest V2 format. + rootfs = tmp_path / "rootfs" + (rootfs / "bin").mkdir(parents=True) + (rootfs / "bin" / "sh").write_text("hello world") + tarball = tmp_path / "base.tar.gz" + with gzip_compressed_tarfile(tarball) as (tar, tar_gz_checksum, tar_checksum): + tar.add(rootfs, arcname=".") + tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest()) + tar_digest = Digest.from_sha256(tar_checksum.hexdigest()) + upload_blob(base_image, str(tarball), tar_gz_digest) + config = { + "created": "2015-10-31T22:22:56.015925234Z", + "author": "Foo <example@example.com>", + "architecture": "amd64", + "os": "linux", + "config": { + "User": "foo", + "Memory": 2048, + "MemorySwap": 4096, + "CpuShares": 8, + "ExposedPorts": {"8080/tcp": {}}, + "Env": ["PATH=/usr/bin:/bin"], + "Entrypoint": ["/bin/sh"], + "Cmd": ["-c", "'echo hello world'"], + "Volumes": {"/x": {}}, + "WorkingDir": "/", + }, + "rootfs": {"diff_ids": [str(tar_digest)], "type": "layers"}, + "history": [ + { + "created": "2015-10-31T22:22:54.690851953Z", + "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /", + } + ], + } + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps(config)) + config_digest = Digest.from_sha256(hashlib.sha256(config_file.read_bytes()).hexdigest()) + upload_blob(base_image, str(config_file), config_digest) + manifest = { + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": config_file.stat().st_size, + "digest": str(config_digest), + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": tarball.stat().st_size, + "digest": str(tar_gz_digest), + } + ], + } + upload_manifest(base_image, manifest) + + # Finally upload some package to registry B with registry A's image as base + buildcache("push", "--base-image", str(base_image), "oci-test", "mpileaks^mpich") + + # Should have some manifests uploaded to registry B now. + assert registry_dst.manifests + + # Verify that all manifest are in the Docker Image Manifest V2 format, not OCI. + # And also check that we're not using annotations, which is an OCI-only "feature". + for m in registry_dst.manifests.values(): + assert m["mediaType"] == "application/vnd.docker.distribution.manifest.v2+json" + assert m["config"]["mediaType"] == "application/vnd.docker.container.image.v1+json" + for layer in m["layers"]: + assert layer["mediaType"] == "application/vnd.docker.image.rootfs.diff.tar.gzip" + assert "annotations" not in m diff --git a/lib/spack/spack/test/oci/mock_registry.py b/lib/spack/spack/test/oci/mock_registry.py index 1bdf33a5d8..cc39904f3c 100644 --- a/lib/spack/spack/test/oci/mock_registry.py +++ b/lib/spack/spack/test/oci/mock_registry.py @@ -17,6 +17,7 @@ import uuid from typing import Callable, Dict, List, Optional, Pattern, Tuple from urllib.request import Request +import spack.oci.oci from spack.oci.image import Digest from spack.oci.opener import OCIAuthHandler @@ -171,7 +172,7 @@ class InMemoryOCIRegistry(DummyServer): self.blobs: Dict[str, bytes] = {} # Map from (name, tag) to manifest - self.manifests: Dict[Tuple[str, str], Dict] = {} + self.manifests: Dict[Tuple[str, str], dict] = {} def index(self, req: Request): return MockHTTPResponse.with_json(200, "OK", body={}) @@ -225,15 +226,12 @@ class InMemoryOCIRegistry(DummyServer): def put_manifest(self, req: Request, name: str, ref: str): # In requests, Python runs header.capitalize(). content_type = req.get_header("Content-type") - assert content_type in ( - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.oci.image.index.v1+json", - ) + assert content_type in spack.oci.oci.all_content_type index_or_manifest = json.loads(self._require_data(req)) # Verify that we have all blobs (layers for manifest, manifests for index) - if content_type == "application/vnd.oci.image.manifest.v1+json": + if content_type in spack.oci.oci.manifest_content_type: for layer in index_or_manifest["layers"]: assert layer["digest"] in self.blobs, "Missing blob while uploading manifest" |