diff --git a/CHANGES/+migrate-push-repository.feature b/CHANGES/+migrate-push-repository.feature new file mode 100644 index 000000000..047dbc79b --- /dev/null +++ b/CHANGES/+migrate-push-repository.feature @@ -0,0 +1,3 @@ +Added a `migrate` endpoint on push repositories that converts a legacy `ContainerPushRepository` +into a `ContainerRepository` in place. The repository primary key and version history are +preserved; only the repository type changes. diff --git a/docs/user/guides/push-image.md b/docs/user/guides/push-image.md index 2c4a25b0b..441c144ad 100644 --- a/docs/user/guides/push-image.md +++ b/docs/user/guides/push-image.md @@ -46,7 +46,8 @@ X-Frame-Options: SAMEORIGIN !!! note - Content is pushed to a push repository type. A push repository does not support mirroring of the + New registry pushes create a regular `ContainerRepository`. Older deployments may still have + legacy `ContainerPushRepository` instances. A push repository does not support mirroring of remote content via the Pulp API. Trying to push content with the same name as an existing "regular" repository will fail. @@ -54,6 +55,27 @@ X-Frame-Options: SAMEORIGIN Rollback to the previous repository versions is not possible with a push repository. Its latest version will always be served. +## Migrating legacy push repositories + +Legacy push repositories can be converted to regular container repositories with the `migrate` +action. Migration changes the repository type in place: the repository primary key, name, +metadata, content, version history, and distribution association are preserved. Registry paths +and tags stay the same. The repository href path changes from `container-push` to `container`; +the converted repository is returned in the task result. + +```bash +http POST $BASE_ADDR/pulp/api/v3/repositories/container/container-push/$PUSH_REPO_PK/migrate/ +``` + +After migration, the repository is listed under container repositories instead of push +repositories. You can continue to push and pull through the same registry path, and you gain +standard repository operations such as sync and version management. + +!!! warning + + Do not push or upload content to a repository while it is being migrated. In-flight blob + uploads tied to the push repository can fail when the repository type changes. + !!! warning Image that has been pulled from a registry and then subsequently pushed to another registy can lead to the blobs digest change. diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 657f1d650..0fc49f985 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -290,6 +290,12 @@ class Meta: model = models.ContainerPushRepository +class MigratePushRepositorySerializer(ValidateFieldsMixin, serializers.Serializer): + """ + Serializer for migrating a push repository to a container repository in place. + """ + + class ContainerRemoteSerializer(RemoteSerializer): """ A Serializer for ContainerRemote. diff --git a/pulp_container/app/tasks/__init__.py b/pulp_container/app/tasks/__init__.py index 794c190c3..f23c1b9ec 100644 --- a/pulp_container/app/tasks/__init__.py +++ b/pulp_container/app/tasks/__init__.py @@ -1,5 +1,6 @@ from .download_image_data import aadd_and_remove, download_image_data # noqa from .builder import build_image_from_containerfile, build_image # noqa +from .migrate_push_repository import migrate_push_repository # noqa from .recursive_add import recursive_add_content # noqa from .recursive_remove import recursive_remove_content # noqa from .sign import sign # noqa diff --git a/pulp_container/app/tasks/migrate_push_repository.py b/pulp_container/app/tasks/migrate_push_repository.py new file mode 100644 index 000000000..230ac9846 --- /dev/null +++ b/pulp_container/app/tasks/migrate_push_repository.py @@ -0,0 +1,133 @@ +from django.contrib.contenttypes.models import ContentType +from django.db import connection, transaction + +from pulpcore.plugin.models import CreatedResource +from pulpcore.plugin.models.role import GroupRole, UserRole + +from pulp_container.app.models import ContainerPushRepository, ContainerRepository +from pulp_container.app.serializers import ContainerRepositorySerializer + + +def migrate_push_repository(push_repository_pk): + """ + Convert a ContainerPushRepository into a ContainerRepository in place. + + Swaps the multi-table-inheritance child row and updates `pulp_type` on the + parent repository so the primary key is preserved. Repository versions, + distributions, and other FKs to `core.Repository` remain valid. + + Args: + push_repository_pk (str): The primary key for the push repository to migrate. + """ + with transaction.atomic(): + with connection.cursor() as cursor: + # First statement in the transaction: protect related reads/writes + # (pending M2Ms, role GFKs) that are not covered by row locks alone. + cursor.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") + + # FOR UPDATE on the MTI join locks both the push child row and the parent + # core_repository row for the duration of this transaction. + push_repository = ContainerPushRepository.objects.select_for_update().get( + pk=push_repository_pk + ) + # Sanity check only: pulp_type should still be container-push + if push_repository.pulp_type != ContainerPushRepository.get_pulp_type(): + raise RuntimeError( + f"Repository {push_repository_pk} has pulp_type " + f"{push_repository.pulp_type!r}, expected a push repository." + ) + if ContainerRepository.objects.filter(pk=push_repository_pk).exists(): + raise RuntimeError( + f"Container repository child row already exists for {push_repository_pk}." + ) + + signing_service_id = push_repository.manifest_signing_service_id + push_ct = ContentType.objects.get_for_model(ContainerPushRepository) + container_ct = ContentType.objects.get_for_model(ContainerRepository) + + with connection.cursor() as cursor: + cursor.execute( + """ + UPDATE core_repository + SET pulp_type = %s + WHERE pulp_id = %s AND pulp_type = %s + """, + [ + ContainerRepository.get_pulp_type(), + push_repository_pk, + ContainerPushRepository.get_pulp_type(), + ], + ) + if cursor.rowcount != 1: + raise RuntimeError( + f"Failed to update pulp_type for repository {push_repository_pk}." + ) + + cursor.execute( + """ + INSERT INTO container_containerrepository + (repository_ptr_id, manifest_signing_service_id) + VALUES (%s, %s) + """, + [push_repository_pk, signing_service_id], + ) + + cursor.execute( + """ + INSERT INTO container_containerrepository_pending_blobs + (containerrepository_id, blob_id) + SELECT containerpushrepository_id, blob_id + FROM container_containerpushrepository_pending_blobs + WHERE containerpushrepository_id = %s + """, + [push_repository_pk], + ) + cursor.execute( + """ + INSERT INTO container_containerrepository_pending_manifests + (containerrepository_id, manifest_id) + SELECT containerpushrepository_id, manifest_id + FROM container_containerpushrepository_pending_manifests + WHERE containerpushrepository_id = %s + """, + [push_repository_pk], + ) + + # Pending M2M FKs are ON DELETE NO ACTION at the DB level (not CASCADE), + # so remove them explicitly before deleting the push child. + cursor.execute( + """ + DELETE FROM container_containerpushrepository_pending_blobs + WHERE containerpushrepository_id = %s + """, + [push_repository_pk], + ) + cursor.execute( + """ + DELETE FROM container_containerpushrepository_pending_manifests + WHERE containerpushrepository_id = %s + """, + [push_repository_pk], + ) + cursor.execute( + """ + DELETE FROM container_containerpushrepository + WHERE repository_ptr_id = %s + """, + [push_repository_pk], + ) + + UserRole.objects.filter(content_type=push_ct, object_id=str(push_repository_pk)).update( + content_type=container_ct + ) + GroupRole.objects.filter(content_type=push_ct, object_id=str(push_repository_pk)).update( + content_type=container_ct + ) + + container_repository = ContainerRepository.objects.get(pk=push_repository_pk) + + CreatedResource(content_object=container_repository).save() + + return ContainerRepositorySerializer( + instance=container_repository, context={"request": None} + ).data diff --git a/pulp_container/app/viewsets.py b/pulp_container/app/viewsets.py index c5df96646..1e9e3ebde 100644 --- a/pulp_container/app/viewsets.py +++ b/pulp_container/app/viewsets.py @@ -1135,7 +1135,7 @@ class ContainerPushRepositoryViewSet( ], }, { - "action": ["update", "partial_update", "set_label", "unset_label"], + "action": ["update", "partial_update", "set_label", "unset_label", "migrate"], "principal": "authenticated", "effect": "allow", "condition_expression": [ @@ -1154,6 +1154,40 @@ class ContainerPushRepositoryViewSet( } LOCKED_ROLES = {} + @extend_schema( + description=( + "Trigger an asynchronous task to convert this push repository into a " + "container repository in place. The repository primary key and version " + "history are preserved; only the repository type changes." + ), + summary="Migrate push repository to container repository", + request=serializers.MigratePushRepositorySerializer, + responses={202: AsyncOperationResponseSerializer}, + ) + @action( + detail=True, methods=["post"], serializer_class=serializers.MigratePushRepositorySerializer + ) + def migrate(self, request, pk): + """ + Create a task which converts a push repository into a container repository in place. + """ + repository = self.get_object() + + serializer = serializers.MigratePushRepositorySerializer( + data=request.data, context={"request": request} + ) + serializer.is_valid(raise_exception=True) + + distributions = models.ContainerDistribution.objects.filter(repository=repository) + exclusive_resources = [repository, *distributions] + + result = dispatch( + tasks.migrate_push_repository, + exclusive_resources=exclusive_resources, + kwargs={"push_repository_pk": str(repository.pk)}, + ) + return OperationPostponedResponse(result, request) + @extend_schema( description=( "Trigger an asynchronous task to remove a manifest and all its associated " diff --git a/pulp_container/tests/functional/api/test_migrate_push_repository.py b/pulp_container/tests/functional/api/test_migrate_push_repository.py new file mode 100644 index 000000000..1ce2b5778 --- /dev/null +++ b/pulp_container/tests/functional/api/test_migrate_push_repository.py @@ -0,0 +1,181 @@ +"""Tests for migrating push repositories to container repositories.""" + +import uuid + +from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP + + +def _pk_from_href(href): + return href.rstrip("/").split("/")[-1] + + +def _setup_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + repo_name, + tag="1.0", +): + """Create a legacy push repository, push an image, and register cleanup.""" + local_url = full_path(f"{repo_name}:{tag}") + + container_push_repository_factory(name=repo_name) + image_path = f"{REGISTRY_V2_REPO_PULP}:manifest_a" + registry_client.pull(image_path) + local_registry.tag_and_push(image_path, local_url) + + distribution = container_bindings.DistributionsContainerApi.list(name=repo_name).results[0] + namespace = container_bindings.PulpContainerNamespacesApi.read(distribution.namespace) + add_to_cleanup(container_bindings.PulpContainerNamespacesApi, namespace.pulp_href) + + push_repository = container_bindings.RepositoriesContainerPushApi.list(name=repo_name).results[ + 0 + ] + return push_repository, local_url + + +def test_migrate_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + monitor_task, +): + """A push repository can be migrated in place and still accept pushes.""" + namespace_name = str(uuid.uuid4()) + repo_name = f"{namespace_name}/migrate" + + push_repository, local_url = _setup_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + repo_name, + ) + push_pk = _pk_from_href(push_repository.pulp_href) + latest_version_before = push_repository.latest_version_href + tags_before = container_bindings.ContentTagsApi.list(repository_version=latest_version_before) + assert tags_before.count == 1 + + migrate_response = container_bindings.RepositoriesContainerPushApi.migrate( + push_repository.pulp_href + ) + task = monitor_task(migrate_response.task) + container_repository = container_bindings.RepositoriesContainerApi.read( + task.result["pulp_href"] + ) + assert container_repository.name == repo_name + assert _pk_from_href(container_repository.pulp_href) == push_pk + assert container_repository.pulp_href in task.created_resources + assert container_repository.prn == task.result["prn"] + assert "container-push" not in container_repository.pulp_href + assert "/container/" in container_repository.pulp_href + + # Same version object/number; href path updates with the repository type. + assert _pk_from_href(container_repository.latest_version_href) == _pk_from_href( + latest_version_before + ) + tags_after = container_bindings.ContentTagsApi.list( + repository_version=container_repository.latest_version_href + ) + assert tags_after.count == 1 + assert tags_after.results[0].name == tags_before.results[0].name + assert tags_after.results[0].prn == tags_before.results[0].prn + + assert container_bindings.RepositoriesContainerPushApi.list(name=repo_name).count == 0 + + distribution = container_bindings.DistributionsContainerApi.list(name=repo_name).results[0] + assert distribution.repository == container_repository.pulp_href + + image_path_b = f"{REGISTRY_V2_REPO_PULP}:manifest_b" + registry_client.pull(image_path_b) + local_registry.tag_and_push(image_path_b, full_path(f"{repo_name}:2.0")) + + container_repository = container_bindings.RepositoriesContainerApi.read( + container_repository.pulp_href + ) + tags_after_push = container_bindings.ContentTagsApi.list( + repository_version=container_repository.latest_version_href + ) + assert tags_after_push.count == 2 + assert {tag.name for tag in tags_after_push.results} == {"1.0", "2.0"} + local_registry.pull(local_url) + + +def test_migrate_push_repository_preserves_versions( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + monitor_task, +): + """Migrating in place preserves repository version history identity and content.""" + namespace_name = str(uuid.uuid4()) + repo_name = f"{namespace_name}/migrate-versions" + + push_repository, _ = _setup_push_repository( + add_to_cleanup, + container_push_repository_factory, + registry_client, + local_registry, + container_bindings, + full_path, + repo_name, + tag="1.0", + ) + + image_path_b = f"{REGISTRY_V2_REPO_PULP}:manifest_b" + registry_client.pull(image_path_b) + local_registry.tag_and_push(image_path_b, full_path(f"{repo_name}:2.0")) + + push_repository = container_bindings.RepositoriesContainerPushApi.read( + push_repository.pulp_href + ) + push_pk = _pk_from_href(push_repository.pulp_href) + push_versions = container_bindings.RepositoriesContainerPushVersionsApi.list( + push_repository.pulp_href + ) + # version 0 (empty) plus one version per push + assert push_versions.count >= 3 + + version_tag_counts = [] + for version in sorted(push_versions.results, key=lambda v: v.number): + if version.number == 0: + continue + tags = container_bindings.ContentTagsApi.list(repository_version=version.pulp_href) + version_tag_counts.append((version.number, tags.count, {t.name for t in tags.results})) + + migrate_response = container_bindings.RepositoriesContainerPushApi.migrate( + push_repository.pulp_href + ) + task = monitor_task(migrate_response.task) + container_repository = container_bindings.RepositoriesContainerApi.read( + task.result["pulp_href"] + ) + assert _pk_from_href(container_repository.pulp_href) == push_pk + + container_versions = container_bindings.RepositoriesContainerVersionsApi.list( + container_repository.pulp_href + ) + migrated_versions = sorted( + [v for v in container_versions.results if v.number != 0], + key=lambda v: v.number, + ) + assert len(migrated_versions) == len(version_tag_counts) + + for (expected_number, expected_count, expected_names), migrated in zip( + version_tag_counts, migrated_versions + ): + assert migrated.number == expected_number + tags = container_bindings.ContentTagsApi.list(repository_version=migrated.pulp_href) + assert tags.count == expected_count + assert {t.name for t in tags.results} == expected_names