diff --git a/.github/java-tests/impact-map.json b/.github/java-tests/impact-map.json index cb6c6296c546..63412c985112 100644 --- a/.github/java-tests/impact-map.json +++ b/.github/java-tests/impact-map.json @@ -712,7 +712,9 @@ "openmetadata-service/src/main/resources/json/data/ontology/**", "openmetadata-spec/src/main/resources/json/schema/api/data/*Ontology*", "openmetadata-spec/src/main/resources/json/schema/api/data/ontology*", - "openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/Ontology*IT.java" + "openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/Ontology*IT.java", + "openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IndependentOntologyReleaseValidationIT.java", + "openmetadata-service/src/test/java/org/openmetadata/service/ontology/OntologyReleaseValidationTest.java" ], "tests": [ "*Ontology*IT", @@ -732,7 +734,11 @@ "openmetadata-spec/src/main/resources/json/schema/api/rdf/**", "openmetadata-spec/src/main/resources/json/schema/api/data/rdf*", "openmetadata-spec/src/main/resources/rdf/**", - "openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/Rdf*IT.java" + "openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/Rdf*IT.java", + "openmetadata-service/src/test/java/org/openmetadata/service/rdf/RdfShaclReportMapperTest.java", + "openmetadata-service/src/test/java/org/openmetadata/service/rdf/NativeOntologyShapesTest.java", + "openmetadata-service/src/test/java/org/openmetadata/service/rdf/OntologyReleaseProjectionTest.java", + "openmetadata-service/src/test/resources/rdf/shapes/malformed-native-test.ttl" ], "tests": [ "*Rdf*IT", diff --git a/docs/decisions/2026-10-23-native-ontology-release-validation-binds-captured-targets.md b/docs/decisions/2026-10-23-native-ontology-release-validation-binds-captured-targets.md new file mode 100644 index 000000000000..3f38c1a7c5e3 --- /dev/null +++ b/docs/decisions/2026-10-23-native-ontology-release-validation-binds-captured-targets.md @@ -0,0 +1,23 @@ +# Native release validation binds SHACL targets to captured definitions + +- **Status:** Accepted +- **Revisions:** v1 2026-10-23 (initial) +- **Deciders:** Harsha +- **Guard:** `NativeOntologyShapesTest`, `OntologyReleaseProjectionTest`, `OntologyReleaseValidationTest`, `IndependentOntologyReleaseValidationIT` +- **Related:** ADR:2026-10-14-ontology-definitions-have-independent-entity-storage + +## Context + +The release RDF projection declares referenced classes and properties, including IRIs from other ontologies. Raw SHACL shapes target every `owl:Class` or `owl:DatatypeProperty` declaration. Applying those targets unchanged to a captured release would report missing labels on referenced classes that the release does not own. + +## Decision + +The raw native shapes keep standard `sh:targetClass` declarations. A release validation request copies the shapes graph and replaces those targets with the captured OWL class and DATA property IRIs. The request does not change shared shapes or the export projection. + +`GET /api/v1/ontologyReleases/{id}/validation` combines structure findings, native SHACL findings, and current authorized impact for definitions newly deprecated against the literal captured base. It reports `logicalCheck: NOT_RUN`. Warnings do not make the report invalid. + +The server verifies and authorizes both complete release manifests on every call. It caches only serialized structure and native findings by verified content digest, with at most 256 entries and 131,072 UTF-16 code units per entry. Release IDs are restored on detached structure reports. Base comparison, captured direct dependencies, and current visible bindings are resolved on every call. Missing current concepts or ontologies retain the binding resolver's 404; authorization failures retain 403. + +## Consequences + +A historical validation read can return 404 after a newly deprecated current concept is deleted. The report does not turn that missing impact into an empty binding list. Cache hits still read and authorize the manifests, so the cache does not bypass access changes. The impact reads every page of visible bindings, following each `after` cursor until it is null, before it sets `boundAssetCount`. diff --git a/docs/generated/api-reference.md b/docs/generated/api-reference.md index 0a2b6bda2f91..295fc21cb9dc 100644 --- a/docs/generated/api-reference.md +++ b/docs/generated/api-reference.md @@ -13,7 +13,7 @@ hand-edit; run `make generate-api-reference` (or `make generate-reference-docs`) - Source is the annotations, **not** `openapi.yml` (a config stub with no endpoints; the full spec is assembled at runtime by Dropwizard). -**1989 endpoints** across 75 resource packages · 1979 carry a summary. +**1990 endpoints** across 75 resource packages · 1980 carry a summary. ## (root) @@ -1452,6 +1452,7 @@ hand-edit; run `make generate-api-reference` (or `make generate-reference-docs`) | `GET` | `/v1/ontologyReleases/{id}/members/{kind}/{memberId}` | Read an immutable release member | | `GET` | `/v1/ontologyReleases/{id}/review` | Read this release's review as the calling user sees it | | `GET` | `/v1/ontologyReleases/{id}/structure` | Check property declarations and axiom structure in a sealed release | +| `GET` | `/v1/ontologyReleases/{id}/validation` | Validate a sealed ontology release | | `GET` | `/v1/ontologyReleases/{id}/versions/{version}` | Read the release header version submitted to an approval workflow | | `GET` | `/v1/relationshipTypes` | List relationship types | | `POST` | `/v1/relationshipTypes` | Create a relationship type | diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IndependentOntologyReleaseValidationIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IndependentOntologyReleaseValidationIT.java new file mode 100644 index 000000000000..6f01231341ce --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IndependentOntologyReleaseValidationIT.java @@ -0,0 +1,464 @@ +package org.openmetadata.it.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.openmetadata.it.factories.DatabaseSchemaTestFactory; +import org.openmetadata.it.util.RdfAccessFixtures; +import org.openmetadata.it.util.SdkClients; +import org.openmetadata.it.util.TestNamespace; +import org.openmetadata.it.util.TestNamespaceExtension; +import org.openmetadata.schema.api.data.CreateOntologyAxiom; +import org.openmetadata.schema.api.data.CreateOntologyProperty; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.data.OntologyReleaseValidation; +import org.openmetadata.schema.api.data.UpdateOntologyConceptBinding; +import org.openmetadata.schema.entity.data.Ontology; +import org.openmetadata.schema.entity.data.OntologyConcept; +import org.openmetadata.schema.entity.data.OntologyRelease; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.NativeOntologyShaclSeverity; +import org.openmetadata.schema.type.OntologyAssetBinding; +import org.openmetadata.schema.type.OntologyAxiomType; +import org.openmetadata.schema.type.OntologyBindingAssetType; +import org.openmetadata.schema.type.OntologyConceptBindingsResponse; +import org.openmetadata.schema.type.OntologyExpression; +import org.openmetadata.schema.type.OntologyExpressionKind; +import org.openmetadata.schema.type.OntologyLiteral; +import org.openmetadata.schema.type.OntologyPropertyKind; +import org.openmetadata.schema.type.OntologyReleaseLogicalCheck; +import org.openmetadata.schema.type.OntologyReleaseMember; +import org.openmetadata.schema.type.RelationProvenance; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.sdk.client.OpenMetadataClient; +import org.openmetadata.sdk.config.OpenMetadataConfig; +import org.openmetadata.sdk.exceptions.OpenMetadataException; +import org.openmetadata.sdk.network.HttpMethod; +import org.openmetadata.service.Entity; + +@Execution(ExecutionMode.CONCURRENT) +@ExtendWith(TestNamespaceExtension.class) +public class IndependentOntologyReleaseValidationIT extends BaseOntologyApprovalIT { + @Test + void cleanAndForeignCapturedClassesUseTheNameLabelFallback(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + declareClass( + ontology, + "ExternalClass", + URI.create("https://standard.example.org/" + ns.shortPrefix() + "/Class")); + final OntologyRelease release = activate(ns, ontology); + + final OntologyReleaseValidation report = validate(release); + assertTrue(report.getValid()); + assertEquals(OntologyReleaseLogicalCheck.NOT_RUN, report.getLogicalCheck()); + assertEquals(release.getId(), report.getReleaseId()); + assertEquals(0, report.getShacl().getViolationCount()); + assertEquals(0, report.getShacl().getWarningCount()); + assertTrue(report.getDeprecationImpacts().isEmpty()); + assertTrue(report.getStructure().getViolations().isEmpty()); + assertEquals(report, validate(release)); + final OntologyRelease sameContent = capture(ontology); + assertEquals(release.getContentDigest(), sameContent.getContentDigest()); + final OntologyReleaseValidation second = validate(sameContent); + assertEquals(sameContent.getId(), second.getReleaseId()); + assertEquals(release.getId(), second.getBaseReleaseId()); + assertEquals(sameContent.getId(), second.getStructure().getReleaseId()); + } + + @Test + void twoValueEnumerationPassesAndSingleValueIsRejectedBeforeCapture(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final var properties = SdkClients.adminClient().ontologyProperties(); + final var one = new OntologyLiteral().withValue("one"); + final var two = new OntologyLiteral().withValue("two"); + final CreateOntologyProperty request = + new CreateOntologyProperty() + .withOntology(ontology.getFullyQualifiedName()) + .withName("status") + .withKind(OntologyPropertyKind.DATA) + .withEnumValues(List.of(one)); + assertEquals( + 400, + assertThrows(OpenMetadataException.class, () -> properties.create(request)) + .getStatusCode()); + properties.create(request.withEnumValues(List.of(one, two))); + assertTrue(validate(capture(ontology)).getValid()); + } + + @Test + void invalidPropertyStructureIsRejectedBeforeCapture(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final var property = + SdkClients.adminClient() + .ontologyProperties() + .create( + new CreateOntologyProperty() + .withName("related") + .withOntology(ontology.getFullyQualifiedName()) + .withKind(OntologyPropertyKind.DATA) + .withEnumValues( + List.of( + new OntologyLiteral().withValue("one"), + new OntologyLiteral().withValue("two")))); + final OntologyRelease release = capture(ontology); + assertEquals( + 400, + assertThrows( + OpenMetadataException.class, + () -> + SdkClients.adminClient() + .ontologyProperties() + .patch( + property.getId(), + JsonUtils.readTree( + """ + [{"op":"replace","path":"/kind","value":"OBJECT"}] + """))) + .getStatusCode()); + assertTrue(validate(release).getValid()); + } + + @Test + void deprecationReportsWarningCapturedDependenciesAndFreshBoundAssets(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final OntologyConcept parent = + declareClass(ontology, "Parent", URI.create("urn:" + ns.shortPrefix() + ":Parent")); + final OntologyConcept child = + declareClass(ontology, "Child", URI.create("urn:" + ns.shortPrefix() + ":Child")); + final OntologyConcept sibling = + declareClass(ontology, "Sibling", URI.create("urn:" + ns.shortPrefix() + ":Sibling")); + for (final OntologyConcept descendant : List.of(child, sibling)) { + SdkClients.adminClient() + .ontologyAxioms() + .create( + new CreateOntologyAxiom() + .withName(descendant.getName() + "OfParent") + .withDisplayName("Direct subclass") + .withDescription("A direct named parent") + .withOntology(ontology.getFullyQualifiedName()) + .withAxiomType(OntologyAxiomType.SUBCLASS_OF) + .withProvenance(RelationProvenance.MANUAL) + .withSubjectIri(descendant.getIri()) + .withExpressions( + List.of( + new OntologyExpression() + .withKind(OntologyExpressionKind.NAMED_CLASS) + .withClassIri(parent.getIri())))); + } + final var property = + SdkClients.adminClient() + .ontologyProperties() + .create( + new CreateOntologyProperty() + .withName("related") + .withOntology(ontology.getFullyQualifiedName()) + .withKind(OntologyPropertyKind.OBJECT) + .withDomain(List.of(parent.getIri())) + .withRange(List.of(parent.getIri()))); + final OntologyRelease base = activate(ns, ontology); + SdkClients.adminClient().ontologyConcepts().update(parent.getId(), parent.withDeprecated(true)); + final OntologyRelease release = capture(ontology); + + final OntologyReleaseValidation first = validate(release); + assertTrue(first.getValid()); + assertEquals(base.getId(), first.getBaseReleaseId()); + assertEquals(2, first.getShacl().getWarningCount()); + assertEquals( + NativeOntologyShaclSeverity.WARNING, + first.getShacl().getFindings().getFirst().getSeverity()); + assertEquals( + Set.of(child.getIri(), sibling.getIri()), + first.getShacl().getFindings().stream() + .map(finding -> finding.getFocusIri()) + .collect(Collectors.toSet())); + final var impact = first.getDeprecationImpacts().getFirst(); + assertEquals(parent.getId(), impact.getDefinition().getId()); + assertEquals( + Set.of(child.getId(), sibling.getId()), + impact.getSubclasses().stream() + .map(reference -> reference.getId()) + .collect(Collectors.toSet())); + assertEquals( + List.of(property.getId()), + impact.getDomainProperties().stream().map(reference -> reference.getId()).toList()); + assertEquals( + List.of(property.getId()), + impact.getRangeProperties().stream().map(reference -> reference.getId()).toList()); + assertEquals(0, impact.getBoundAssetCount()); + + final var schema = DatabaseSchemaTestFactory.createSimple(ns); + final var firstTable = + SdkClients.adminClient() + .tables() + .create( + new CreateTable() + .withName(ns.prefix("boundTableOne")) + .withDatabaseSchema(schema.getFullyQualifiedName()) + .withColumns( + List.of(new Column().withName("id").withDataType(ColumnDataType.STRING)))); + final var secondTable = + SdkClients.adminClient() + .tables() + .create( + new CreateTable() + .withName(ns.prefix("boundTableTwo")) + .withDatabaseSchema(schema.getFullyQualifiedName()) + .withColumns( + List.of(new Column().withName("id").withDataType(ColumnDataType.STRING)))); + bind(parent, firstTable.getId(), 0L); + bind(parent, secondTable.getId(), 1L); + final OntologyReleaseValidation refreshed = validate(release); + assertEquals(first.getShacl(), refreshed.getShacl()); + assertEquals(2, refreshed.getDeprecationImpacts().getFirst().getBoundAssetCount()); + assertEquals( + Set.of(firstTable.getId(), secondTable.getId()), + refreshed.getDeprecationImpacts().getFirst().getBoundAssets().stream() + .map(reference -> reference.getId()) + .collect(Collectors.toSet())); + assertFalse( + refreshed + .getDeprecationImpacts() + .getFirst() + .getBoundAssets() + .getFirst() + .getFullyQualifiedName() + .isBlank()); + try (var fixtures = new RdfAccessFixtures("validationAssets" + ns.shortPrefix())) { + final var denyTables = fixtures.denyRole(MetadataOperation.VIEW_ALL, Entity.TABLE); + final OpenMetadataClient restricted = + new OpenMetadataClient( + OpenMetadataConfig.builder() + .serverUrl(SdkClients.baseUrl()) + .accessToken(fixtures.userToken("reader", List.of(denyTables.getId()))) + .build()); + final var visible = validate(restricted, release).getDeprecationImpacts().getFirst(); + assertEquals(0, visible.getBoundAssetCount()); + assertTrue(visible.getBoundAssets().isEmpty()); + } + } + + @Test + void cachedFindingsStillRequireCapturedMemberView(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + declareClass(ontology, "Restricted", URI.create("urn:" + ns.shortPrefix() + ":Restricted")); + final OntologyRelease release = capture(ontology); + assertTrue(validate(release).getValid()); + + try (var fixtures = new RdfAccessFixtures("validation" + ns.shortPrefix())) { + final var denyConcepts = + fixtures.denyRole(MetadataOperation.VIEW_ALL, Entity.ONTOLOGY_CONCEPT); + final OpenMetadataClient denied = + new OpenMetadataClient( + OpenMetadataConfig.builder() + .serverUrl(SdkClients.baseUrl()) + .accessToken(fixtures.userToken("reader", List.of(denyConcepts.getId()))) + .build()); + assertEquals(release.getId(), denied.ontologyReleases().get(release.getId()).getId()); + assertEquals( + 403, + assertThrows( + OpenMetadataException.class, + () -> + denied + .getHttpClient() + .execute( + HttpMethod.GET, + "/v1/ontologyReleases/" + release.getId() + "/validation", + null, + OntologyReleaseValidation.class)) + .getStatusCode()); + } + } + + @Test + void cachedFindingsStillRequireCapturedBaseMemberView(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final OntologyConcept concept = + declareClass(ontology, "BaseOnly", URI.create("urn:" + ns.shortPrefix() + ":BaseOnly")); + final OntologyRelease base = activate(ns, ontology); + SdkClients.adminClient() + .ontologyConcepts() + .delete(concept.getId().toString(), Map.of("hardDelete", "true")); + final OntologyRelease release = capture(ontology); + assertEquals(base.getId(), validate(release).getBaseReleaseId()); + + try (var fixtures = new RdfAccessFixtures("validationBase" + ns.shortPrefix())) { + final var denyConcepts = + fixtures.denyRole(MetadataOperation.VIEW_ALL, Entity.ONTOLOGY_CONCEPT); + final OpenMetadataClient denied = + new OpenMetadataClient( + OpenMetadataConfig.builder() + .serverUrl(SdkClients.baseUrl()) + .accessToken(fixtures.userToken("reader", List.of(denyConcepts.getId()))) + .build()); + assertEquals(release.getId(), denied.ontologyReleases().get(release.getId()).getId()); + assertEquals( + 403, + assertThrows( + OpenMetadataException.class, + () -> + denied + .getHttpClient() + .execute( + HttpMethod.GET, + "/v1/ontologyReleases/" + release.getId() + "/validation", + null, + OntologyReleaseValidation.class)) + .getStatusCode()); + } + } + + @Test + void missingCurrentConceptDoesNotBecomeEmptyImpact(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final OntologyConcept concept = + declareClass(ontology, "Retired", URI.create("urn:" + ns.shortPrefix() + ":Retired")); + activate(ns, ontology); + SdkClients.adminClient() + .ontologyConcepts() + .update(concept.getId(), concept.withDeprecated(true)); + final OntologyRelease release = capture(ontology); + assertEquals(1, validate(release).getDeprecationImpacts().size()); + + SdkClients.adminClient() + .ontologyConcepts() + .delete(concept.getId().toString(), Map.of("hardDelete", "true")); + assertEquals( + 404, assertThrows(OpenMetadataException.class, () -> validate(release)).getStatusCode()); + } + + @Test + void newlyDeprecatedPropertyHasNoInventedClassOrAssetImpact(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final var property = + SdkClients.adminClient() + .ontologyProperties() + .create( + new CreateOntologyProperty() + .withName("legacyCode") + .withOntology(ontology.getFullyQualifiedName()) + .withKind(OntologyPropertyKind.DATA) + .withRange(List.of(URI.create("http://www.w3.org/2001/XMLSchema#string")))); + activate(ns, ontology); + SdkClients.adminClient() + .ontologyProperties() + .update(property.getId(), property.withDeprecated(true)); + + final var impact = validate(capture(ontology)).getDeprecationImpacts().getFirst(); + assertEquals(property.getId(), impact.getDefinition().getId()); + assertEquals(0, impact.getBoundAssetCount()); + assertTrue(impact.getBoundAssets().isEmpty()); + assertTrue(impact.getSubclasses().isEmpty()); + assertTrue(impact.getDomainProperties().isEmpty()); + assertTrue(impact.getRangeProperties().isEmpty()); + } + + @Test + void warmFindingsDoNotBypassReleaseOrOntologyView(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final OntologyRelease release = capture(ontology); + assertTrue(validate(release).getValid()); + + for (int index = 0; index < 2; index++) { + try (var fixtures = new RdfAccessFixtures("validationHeader" + ns.shortPrefix() + index)) { + final var role = + fixtures.denyRole( + MetadataOperation.VIEW_ALL, index == 0 ? Entity.ONTOLOGY_RELEASE : Entity.ONTOLOGY); + final OpenMetadataClient denied = + new OpenMetadataClient( + OpenMetadataConfig.builder() + .serverUrl(SdkClients.baseUrl()) + .accessToken(fixtures.userToken("reader", List.of(role.getId()))) + .build()); + assertEquals( + 403, + assertThrows(OpenMetadataException.class, () -> validate(denied, release)) + .getStatusCode()); + } + } + } + + @Test + void currentConceptBasicViewIsCheckedAfterCapturedMemberAllView(TestNamespace ns) { + final Ontology ontology = createOntology(ns); + final OntologyConcept concept = + declareClass(ontology, "Deprecated", URI.create("urn:" + ns.shortPrefix() + ":Deprecated")); + activate(ns, ontology); + SdkClients.adminClient() + .ontologyConcepts() + .update(concept.getId(), concept.withDeprecated(true)); + final OntologyRelease release = capture(ontology); + assertEquals(1, validate(release).getDeprecationImpacts().size()); + + try (var fixtures = new RdfAccessFixtures("validationCurrent" + ns.shortPrefix())) { + final var denyBasic = + fixtures.denyRole(MetadataOperation.VIEW_BASIC, Entity.ONTOLOGY_CONCEPT); + final OpenMetadataClient denied = + new OpenMetadataClient( + OpenMetadataConfig.builder() + .serverUrl(SdkClients.baseUrl()) + .accessToken(fixtures.userToken("reader", List.of(denyBasic.getId()))) + .build()); + assertNotNull( + denied + .getHttpClient() + .execute( + HttpMethod.GET, + "/v1/ontologyReleases/" + release.getId() + "/members/CONCEPT/" + concept.getId(), + null, + OntologyReleaseMember.class)); + assertEquals( + 403, + assertThrows(OpenMetadataException.class, () -> validate(denied, release)) + .getStatusCode()); + } + } + + private static OntologyReleaseValidation validate(OntologyRelease release) { + return validate(SdkClients.adminClient(), release); + } + + private static void bind(OntologyConcept concept, UUID tableId, long revision) { + SdkClients.adminClient() + .getHttpClient() + .execute( + HttpMethod.PUT, + "/v1/ontologyConcepts/" + concept.getId() + "/bindings", + new UpdateOntologyConceptBinding() + .withExpectedRevision(revision) + .withAsset( + new OntologyAssetBinding() + .withAssetType(OntologyBindingAssetType.TABLE) + .withAssetId(tableId)), + OntologyConceptBindingsResponse.class); + } + + private static OntologyReleaseValidation validate( + OpenMetadataClient client, OntologyRelease release) { + final OntologyReleaseValidation report = + client + .getHttpClient() + .execute( + HttpMethod.GET, + "/v1/ontologyReleases/" + release.getId() + "/validation", + null, + OntologyReleaseValidation.class); + assertNotNull(report); + return report; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/ontology/OntologyReleaseValidationService.java b/openmetadata-service/src/main/java/org/openmetadata/service/ontology/OntologyReleaseValidationService.java new file mode 100644 index 000000000000..06f09fffa7fa --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/ontology/OntologyReleaseValidationService.java @@ -0,0 +1,285 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.ontology; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.function.Consumer; +import org.openmetadata.schema.OntologyDefinition; +import org.openmetadata.schema.api.data.OntologyReleaseStructureReport; +import org.openmetadata.schema.api.data.OntologyReleaseValidation; +import org.openmetadata.schema.entity.data.OntologyAxiom; +import org.openmetadata.schema.entity.data.OntologyConcept; +import org.openmetadata.schema.entity.data.OntologyProperty; +import org.openmetadata.schema.entity.data.OntologyRelease; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.NativeOntologyDeprecationImpact; +import org.openmetadata.schema.type.NativeOntologyShaclReport; +import org.openmetadata.schema.type.OntologyConceptBindingsResponse; +import org.openmetadata.schema.type.OntologyReleaseChangeType; +import org.openmetadata.schema.type.OntologyReleaseLogicalCheck; +import org.openmetadata.schema.type.OntologyReleaseMember; +import org.openmetadata.schema.type.OntologyReleaseMemberChange; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.exception.BadRequestException; +import org.openmetadata.service.jdbi3.OntologyReleaseRepository; +import org.openmetadata.service.rdf.NativeOntologyValidation; + +public final class OntologyReleaseValidationService { + private static final int MAX_CACHE_STRING_LENGTH = 131_072; + private final OntologyReleaseReader reader; + private final OntologyConceptBindingService bindings; + private final NativeOntologyValidation nativeValidation = new NativeOntologyValidation(); + private final Cache cache = + Caffeine.newBuilder().maximumSize(256).build(); + + public OntologyReleaseValidationService( + final OntologyReleaseRepository releases, final OntologyConceptBindingService bindings) { + reader = new OntologyReleaseReader(releases.getDaoCollection()); + this.bindings = bindings; + } + + public OntologyReleaseValidation report( + final OntologyRelease release, + final OntologyRelease base, + final Consumer> authorize, + final OntologyConceptBindingService.Access access) { + requireCapturedBase(release, base); + final List members = reader.read(release, authorize); + final List previous = + base == null ? List.of() : reader.read(base, authorize); + FrozenFindings frozen = cache.getIfPresent(release.getContentDigest()); + if (frozen == null) { + frozen = freeze(release, members); + if ((long) frozen.structure().length() + frozen.shacl().length() <= MAX_CACHE_STRING_LENGTH) { + cache.put(release.getContentDigest(), frozen); + } + } + final OntologyReleaseStructureReport structure = structure(release.getId(), frozen); + final NativeOntologyShaclReport shacl = + JsonUtils.readValue(frozen.shacl(), NativeOntologyShaclReport.class); + final List impacts = + impacts(release, previous, members, access); + return new OntologyReleaseValidation() + .withReleaseId(release.getId()) + .withContentDigest(release.getContentDigest()) + .withBaseReleaseId(base == null ? null : base.getId()) + .withValid(Boolean.TRUE.equals(structure.getValid()) && shacl.getViolationCount() == 0) + .withLogicalCheck(OntologyReleaseLogicalCheck.NOT_RUN) + .withStructure(structure) + .withShacl(shacl) + .withDeprecationImpacts(impacts); + } + + private static void requireCapturedBase( + final OntologyRelease release, final OntologyRelease base) { + if (!Objects.equals(release.getBaseReleaseId(), base == null ? null : base.getId()) + || (base != null + && !Objects.equals(release.getOntology().getId(), base.getOntology().getId()))) { + throw new BadRequestException( + "Validation requires the release's captured same-ontology base"); + } + } + + private FrozenFindings freeze( + final OntologyRelease release, final List members) { + final OntologyReleaseStructureReport structure = + OntologyStructureValidator.validate(release, members); + final NativeOntologyShaclReport shacl = nativeValidation.validate(release, members); + final ObjectNode detached = JsonUtils.getObjectMapper().valueToTree(structure); + detached.remove("releaseId"); + detached.withArray("violations").forEach(issue -> ((ObjectNode) issue).remove("releaseId")); + return new FrozenFindings(detached.toString(), JsonUtils.pojoToJson(shacl)); + } + + private static OntologyReleaseStructureReport structure( + final UUID releaseId, final FrozenFindings frozen) { + final OntologyReleaseStructureReport structure = + JsonUtils.readValue(frozen.structure(), OntologyReleaseStructureReport.class); + structure.withReleaseId(releaseId); + structure.getViolations().forEach(issue -> issue.withReleaseId(releaseId)); + return structure; + } + + private List impacts( + final OntologyRelease release, + final List previous, + final List members, + final OntologyConceptBindingService.Access access) { + final List deprecated = + OntologyReleaseComparison.compareMembers(previous, members).stream() + .filter(change -> change.getChangeType() == OntologyReleaseChangeType.DEPRECATED) + .toList(); + if (deprecated.isEmpty()) { + return List.of(); + } + final CapturedDependencies dependencies = CapturedDependencies.from(release, members); + return deprecated.stream().map(change -> impact(change, dependencies, access)).toList(); + } + + private NativeOntologyDeprecationImpact impact( + final OntologyReleaseMemberChange change, + final CapturedDependencies dependencies, + final OntologyConceptBindingService.Access access) { + final EntityReference definition = change.getEntity(); + final List assets = + change.getMemberKind() == OntologyReleaseMember.MemberKind.CONCEPT + ? boundAssets(definition.getId(), access) + : List.of(); + final String iri = dependencies.iri(change); + final boolean concept = change.getMemberKind() == OntologyReleaseMember.MemberKind.CONCEPT; + return new NativeOntologyDeprecationImpact() + .withDefinition(definition) + .withBoundAssetCount(assets.size()) + .withBoundAssets(assets) + .withSubclasses(concept ? dependencies.subclasses(iri) : List.of()) + .withDomainProperties(concept ? dependencies.domainProperties(iri) : List.of()) + .withRangeProperties(concept ? dependencies.rangeProperties(iri) : List.of()); + } + + /** A page can end short with a cursor when hidden rows use up its scan budget. */ + private List boundAssets( + final UUID conceptId, final OntologyConceptBindingService.Access access) { + final List assets = new ArrayList<>(); + String after = null; + do { + final OntologyConceptBindingsResponse page = + bindings.get(access, conceptId, OntologyConceptBindingService.MAX_PAGE_SIZE, after); + page.getAssets().forEach(binding -> assets.add(identity(binding.getAsset()))); + after = page.getAfter(); + } while (after != null); + assets.sort(CapturedDependencies.REFERENCE_ORDER); + return assets; + } + + private static EntityReference identity(final EntityReference source) { + return new EntityReference() + .withId(source.getId()) + .withType(source.getType()) + .withName(source.getName()) + .withFullyQualifiedName(source.getFullyQualifiedName()) + .withDisplayName(source.getDisplayName()); + } + + private record FrozenFindings(String structure, String shacl) {} + + private record CapturedDependencies( + Map irisByMember, + Map> subclassesByParent, + Map> domainByClass, + Map> rangeByClass) { + private static final Comparator REFERENCE_ORDER = + Comparator.comparing(EntityReference::getType) + .thenComparing(reference -> reference.getId().toString()); + + static CapturedDependencies from( + final OntologyRelease release, final List members) { + final Map irisByMember = new HashMap<>(); + final Map concepts = new HashMap<>(); + final List definitions = + members.stream() + .map(member -> OntologyReleaseReader.definition(release, member)) + .toList(); + for (final OntologyDefinition definition : definitions) { + switch (definition) { + case OntologyConcept concept -> { + irisByMember.put( + key(OntologyReleaseMember.MemberKind.CONCEPT, concept.getId()), + concept.getIri().toString()); + concepts.put(concept.getIri().toString(), concept); + } + case OntologyProperty property -> irisByMember.put( + key(OntologyReleaseMember.MemberKind.PROPERTY, property.getId()), + property.getIri().toString()); + default -> {} + } + } + final Map> subclasses = new HashMap<>(); + final Map> domains = new HashMap<>(); + final Map> ranges = new HashMap<>(); + for (final OntologyDefinition definition : definitions) { + switch (definition) { + case OntologyAxiom axiom -> { + final OntologyConcept child = concepts.get(String.valueOf(axiom.getSubjectIri())); + if (child != null) { + OntologyClassHierarchy.namedParents(axiom) + .forEach(parent -> add(subclasses, parent.toString(), captured(child))); + } + } + case OntologyProperty property -> { + listOrEmpty(property.getDomain()) + .forEach(domain -> add(domains, domain.toString(), captured(property))); + listOrEmpty(property.getRange()) + .forEach(range -> add(ranges, range.toString(), captured(property))); + } + default -> {} + } + } + return new CapturedDependencies(irisByMember, subclasses, domains, ranges); + } + + String iri(final OntologyReleaseMemberChange change) { + final String iri = irisByMember.get(key(change.getMemberKind(), change.getEntity().getId())); + if (iri == null) { + throw new BadRequestException( + "A deprecated definition is missing from its captured release"); + } + return iri; + } + + List subclasses(final String iri) { + return values(subclassesByParent, iri); + } + + List domainProperties(final String iri) { + return values(domainByClass, iri); + } + + List rangeProperties(final String iri) { + return values(rangeByClass, iri); + } + + private static EntityReference captured(final OntologyDefinition definition) { + return definition.getEntityReference().withDescription(null).withHref(null); + } + + private static void add( + final Map> index, + final String iri, + final EntityReference reference) { + index + .computeIfAbsent(iri, ignored -> new HashMap<>()) + .put(reference.getType() + ":" + reference.getId(), reference); + } + + private static List values( + final Map> index, final String iri) { + return index.getOrDefault(iri, Map.of()).values().stream().sorted(REFERENCE_ORDER).toList(); + } + + private static String key(final OntologyReleaseMember.MemberKind kind, final UUID id) { + return kind.value() + ":" + id; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/rdf/NativeOntologyValidation.java b/openmetadata-service/src/main/java/org/openmetadata/service/rdf/NativeOntologyValidation.java new file mode 100644 index 000000000000..43fd98262708 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/rdf/NativeOntologyValidation.java @@ -0,0 +1,108 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.rdf; + +import java.util.List; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.shacl.ShaclValidator; +import org.apache.jena.shacl.Shapes; +import org.apache.jena.shacl.vocabulary.SHACL; +import org.apache.jena.vocabulary.OWL2; +import org.openmetadata.schema.OntologyDefinition; +import org.openmetadata.schema.entity.data.OntologyConcept; +import org.openmetadata.schema.entity.data.OntologyProperty; +import org.openmetadata.schema.entity.data.OntologyRelease; +import org.openmetadata.schema.type.NativeOntologyShaclReport; +import org.openmetadata.schema.type.OntologyConceptType; +import org.openmetadata.schema.type.OntologyPropertyKind; +import org.openmetadata.schema.type.OntologyReleaseMember; +import org.openmetadata.service.ontology.OntologyReleaseReader; +import org.openmetadata.service.rdf.RdfShaclValidator.ShapeSet; + +public final class NativeOntologyValidation { + private static final Property TARGET_CLASS = + ResourceFactory.createProperty(SHACL.NS, "targetClass"); + private static final Property TARGET_NODE = + ResourceFactory.createProperty(SHACL.NS, "targetNode"); + private final OntologyReleaseProjection projection = new OntologyReleaseProjection(); + private final RdfShaclReportMapper mapper = new RdfShaclReportMapper(); + + public NativeOntologyShaclReport validate( + final OntologyRelease release, final List members) { + final Model localShapes = boundShapes(release, members); + try { + final Model data = projection.build(release, members); + try { + return mapper.mapNative( + ShaclValidator.get().validate(Shapes.parse(localShapes.getGraph()), data.getGraph())); + } finally { + data.close(); + } + } finally { + localShapes.close(); + } + } + + private static Model boundShapes( + final OntologyRelease release, final List members) { + final Model copy = ModelFactory.createDefaultModel(); + try { + copy.add( + ModelFactory.createModelForGraph( + RdfShaclValidator.shapes(ShapeSet.NATIVE_ONTOLOGY).getGraph())); + final List definitions = + members.stream() + .map(member -> OntologyReleaseReader.definition(release, member)) + .toList(); + final var targets = copy.listStatements(null, TARGET_CLASS, (RDFNode) null).toList(); + for (final var target : targets) { + copy.remove(target); + targetIris(definitions, target.getResource()) + .forEach(iri -> copy.add(target.getSubject(), TARGET_NODE, copy.createResource(iri))); + } + return copy; + } catch (RuntimeException failure) { + copy.close(); + throw failure; + } + } + + private static List targetIris( + final List definitions, final Resource targetClass) { + if (OWL2.Class.equals(targetClass)) { + return definitions.stream() + .filter(OntologyConcept.class::isInstance) + .map(OntologyConcept.class::cast) + .filter(concept -> concept.getConceptType() != OntologyConceptType.SKOS_CONCEPT) + .map(concept -> concept.getIri().toString()) + .distinct() + .toList(); + } + if (OWL2.DatatypeProperty.equals(targetClass)) { + return definitions.stream() + .filter(OntologyProperty.class::isInstance) + .map(OntologyProperty.class::cast) + .filter(property -> property.getKind() == OntologyPropertyKind.DATA) + .map(property -> property.getIri().toString()) + .distinct() + .toList(); + } + throw new IllegalStateException("Unexpected native ontology shape target: " + targetClass); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclReportMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclReportMapper.java index 8121620894d0..07e3d125cc93 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclReportMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclReportMapper.java @@ -14,12 +14,17 @@ package org.openmetadata.service.rdf; import java.io.StringWriter; +import java.net.URI; +import java.util.Comparator; import java.util.List; import org.apache.jena.riot.RDFDataMgr; import org.apache.jena.riot.RDFFormat; import org.apache.jena.shacl.ValidationReport; import org.apache.jena.shacl.validation.ReportEntry; import org.apache.jena.shacl.validation.Severity; +import org.openmetadata.schema.type.NativeOntologyShaclFinding; +import org.openmetadata.schema.type.NativeOntologyShaclReport; +import org.openmetadata.schema.type.NativeOntologyShaclSeverity; import org.openmetadata.schema.type.RdfValidationReport; import org.openmetadata.schema.type.RdfValidationViolation; @@ -49,6 +54,56 @@ public RdfValidationReport skipped() { .withViolations(List.of()); } + public NativeOntologyShaclReport mapNative(final ValidationReport report) { + final List entries = List.copyOf(report.getEntries()); + final List findings = + entries.stream() + .map(this::mapNative) + .sorted( + Comparator.comparing( + (NativeOntologyShaclFinding finding) -> finding.getFocusIri().toString()) + .thenComparing(finding -> finding.getShapeIri().toString()) + .thenComparing(finding -> finding.getSeverity().value()) + .thenComparing(NativeOntologyShaclFinding::getMessage)) + .limit(MAX_TYPED_VIOLATIONS) + .toList(); + final long warnings = + entries.stream() + .filter(entry -> severity(entry) == NativeOntologyShaclSeverity.WARNING) + .count(); + return new NativeOntologyShaclReport() + .withViolationCount(Math.toIntExact(entries.size() - warnings)) + .withWarningCount(Math.toIntExact(warnings)) + .withTruncated(entries.size() > MAX_TYPED_VIOLATIONS) + .withFindings(findings); + } + + private NativeOntologyShaclFinding mapNative(final ReportEntry entry) { + if (entry.focusNode() == null + || !entry.focusNode().isURI() + || entry.source() == null + || !entry.source().isURI()) { + throw new IllegalStateException("Native SHACL finding requires named focus and shape IRIs"); + } + return new NativeOntologyShaclFinding() + .withFocusIri(URI.create(entry.focusNode().getURI())) + .withShapeIri(URI.create(entry.source().getURI())) + .withSeverity(severity(entry)) + .withMessage( + entry.message() == null ? "Native ontology constraint violation" : entry.message()); + } + + private static NativeOntologyShaclSeverity severity(final ReportEntry entry) { + final String level = entry.severity().level().getURI(); + if ("http://www.w3.org/ns/shacl#Warning".equals(level)) { + return NativeOntologyShaclSeverity.WARNING; + } + if ("http://www.w3.org/ns/shacl#Violation".equals(level)) { + return NativeOntologyShaclSeverity.VIOLATION; + } + throw new IllegalStateException("Unexpected native SHACL severity: " + level); + } + private RdfValidationViolation map(final ReportEntry entry) { return new RdfValidationViolation() .withFocusNode(stringValue(entry.focusNode())) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclValidator.java b/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclValidator.java index a959dccd79e9..35465bb297d5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclValidator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/rdf/RdfShaclValidator.java @@ -31,23 +31,45 @@ public final class RdfShaclValidator { private static final String SHAPES_RESOURCE = "/rdf/shapes/openmetadata-shapes.ttl"; + private static final String NATIVE_SHAPES_RESOURCE = "/rdf/shapes/native-ontology-shapes.ttl"; + + public enum ShapeSet { + CATALOG, + NATIVE_ONTOLOGY + } private RdfShaclValidator() {} - private static final class Holder { + private static final class CatalogHolder { private static final Shapes SHAPES = loadShapes(SHAPES_RESOURCE); } + private static final class NativeHolder { + private static final Shapes NATIVE_SHAPES = loadShapes(NATIVE_SHAPES_RESOURCE); + } + public static ValidationReport validate(Graph data) { - return org.apache.jena.shacl.ShaclValidator.get().validate(Holder.SHAPES, data); + return validate(data, ShapeSet.CATALOG); + } + + public static ValidationReport validate(Graph data, ShapeSet shapeSet) { + return org.apache.jena.shacl.ShaclValidator.get().validate(shapes(shapeSet), data); } public static ValidationReport validate(Model data) { return validate(data.getGraph()); } + public static ValidationReport validate(Model data, ShapeSet shapeSet) { + return validate(data.getGraph(), shapeSet); + } + public static Shapes shapes() { - return Holder.SHAPES; + return CatalogHolder.SHAPES; + } + + public static Shapes shapes(ShapeSet shapeSet) { + return shapeSet == ShapeSet.CATALOG ? CatalogHolder.SHAPES : NativeHolder.NATIVE_SHAPES; } static Shapes loadShapes(String resourcePath) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ontology/OntologyReleaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ontology/OntologyReleaseResource.java index 21b0a6dcff83..09a310af3066 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ontology/OntologyReleaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ontology/OntologyReleaseResource.java @@ -20,6 +20,7 @@ import org.openmetadata.schema.api.data.OntologyReleaseDiff; import org.openmetadata.schema.api.data.OntologyReleaseReview; import org.openmetadata.schema.api.data.OntologyReleaseStructureReport; +import org.openmetadata.schema.api.data.OntologyReleaseValidation; import org.openmetadata.schema.entity.data.OntologyRelease; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.OntologyReleaseEvent; @@ -28,11 +29,14 @@ import org.openmetadata.schema.utils.ResultList; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.OntologyConceptBindingStore; import org.openmetadata.service.jdbi3.OntologyReleaseRepository; import org.openmetadata.service.limits.Limits; +import org.openmetadata.service.ontology.OntologyConceptBindingService; import org.openmetadata.service.ontology.OntologyReleaseComparison; import org.openmetadata.service.ontology.OntologyReleaseReader; import org.openmetadata.service.ontology.OntologyReleaseStructure; +import org.openmetadata.service.ontology.OntologyReleaseValidationService; import org.openmetadata.service.rdf.OntologyReleaseExporter; import org.openmetadata.service.rdf.RdfSerializationFormat; import org.openmetadata.service.resources.Collection; @@ -50,12 +54,21 @@ public final class OntologyReleaseResource private final OntologyReleaseExporter exporter; private final OntologyReleaseStructure structure; private final OntologyReleaseComparison comparison; + private final OntologyReleaseValidationService validation; public OntologyReleaseResource(final Authorizer authorizer, final Limits limits) { super(Entity.ONTOLOGY_RELEASE, authorizer, limits); exporter = new OntologyReleaseExporter(repository); structure = new OntologyReleaseStructure(repository); comparison = new OntologyReleaseComparison(repository); + validation = + new OntologyReleaseValidationService( + repository, + new OntologyConceptBindingService( + new OntologyConceptBindingStore( + Entity.getCollectionDAO(), OntologyConceptBindingStore.registeredAssetTables()), + authorizer, + Entity::getEntityRepository)); } public static class OntologyReleaseList extends ResultList {} @@ -140,6 +153,29 @@ public OntologyReleaseStructureReport structure( definition -> OntologyResourceAccess.requireView(authorizer, securityContext, definition)); } + @GET + @Path("/{id}/validation") + @Operation( + operationId = "getOntologyReleaseValidation", + summary = "Validate a sealed ontology release", + description = + "Returns structure and native SHACL findings with current authorized deprecation impact. Logical consistency is not checked and release state does not change.") + public OntologyReleaseValidation validation( + @Context final UriInfo uriInfo, + @Context final SecurityContext securityContext, + @PathParam("id") final UUID id) { + final OntologyRelease release = get(uriInfo, securityContext, id); + final OntologyRelease base = + release.getBaseReleaseId() == null + ? null + : get(uriInfo, securityContext, release.getBaseReleaseId()); + return validation.report( + release, + base, + definitions -> OntologyResourceAccess.requireView(authorizer, securityContext, definitions), + new OntologyConceptBindingService.Access(uriInfo, securityContext)); + } + @GET @Path("/{id}/export") @Produces({"text/turtle", "application/rdf+xml", "application/n-triples", "application/ld+json"}) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/ontology/OntologyReleaseValidationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/ontology/OntologyReleaseValidationTest.java new file mode 100644 index 000000000000..a04d920e0616 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/ontology/OntologyReleaseValidationTest.java @@ -0,0 +1,338 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.ontology; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.github.benmanes.caffeine.cache.Cache; +import java.lang.reflect.Field; +import java.net.URI; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.OntologyDefinition; +import org.openmetadata.schema.api.data.OntologyReleaseValidation; +import org.openmetadata.schema.entity.data.Ontology; +import org.openmetadata.schema.entity.data.OntologyConcept; +import org.openmetadata.schema.entity.data.OntologyProperty; +import org.openmetadata.schema.entity.data.OntologyRelease; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.NativeOntologyDeprecationImpact; +import org.openmetadata.schema.type.OntologyConceptBindingsResponse; +import org.openmetadata.schema.type.OntologyConceptType; +import org.openmetadata.schema.type.OntologyHydratedAssetBinding; +import org.openmetadata.schema.type.OntologyLiteral; +import org.openmetadata.schema.type.OntologyPropertyKind; +import org.openmetadata.schema.type.OntologyReleaseMember; +import org.openmetadata.schema.type.OntologyReleaseMember.MemberKind; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.OntologyReleaseDAO; +import org.openmetadata.service.jdbi3.OntologyReleaseRepository; + +class OntologyReleaseValidationTest { + @Test + void equalDigestsRestoreEachReleaseIdAndReauthorizeCacheHits() { + final Ontology ontology = + new Ontology() + .withId(UUID.randomUUID()) + .withName("model") + .withFullyQualifiedName("model") + .withIri(URI.create("https://example.org/model/")) + .withNamespace(URI.create("https://example.org/model/")); + final OntologyProperty property = + new OntologyProperty() + .withId(UUID.randomUUID()) + .withName("code") + .withIri(URI.create("https://example.org/model/code")) + .withOntology(ontology.getEntityReference()) + .withKind(OntologyPropertyKind.DATA) + .withRange(List.of(URI.create("urn:UnknownDatatype"))); + final Bundle first = bundle(ontology, property); + final Bundle second = bundle(ontology, property); + assertEquals(first.release().getContentDigest(), second.release().getContentDigest()); + + final Map> stored = + Map.of( + first.release().getId(), first.members(), second.release().getId(), second.members()); + final OntologyReleaseValidationService validation = validation(stored); + final OntologyConceptBindingService.Access access = + new OntologyConceptBindingService.Access(null, null); + + final OntologyReleaseValidation firstReport = + validation.report(first.release(), null, ignored -> {}, access); + final OntologyReleaseValidation secondReport = + validation.report(second.release(), null, ignored -> {}, access); + assertFalse(firstReport.getValid()); + assertEquals(first.release().getId(), firstReport.getStructure().getReleaseId()); + assertEquals(second.release().getId(), secondReport.getStructure().getReleaseId()); + assertEquals( + second.release().getId(), + secondReport.getStructure().getViolations().getFirst().getReleaseId()); + firstReport.getStructure().getViolations().clear(); + assertEquals( + 1, + validation + .report(first.release(), null, ignored -> {}, access) + .getStructure() + .getViolations() + .size()); + assertThrows( + IllegalStateException.class, + () -> + validation.report( + first.release(), + null, + ignored -> { + throw new IllegalStateException("view denied"); + }, + access)); + } + + @Test + void equalDigestsKeepDeprecationSelectionTiedToTheLiteralBase() { + final Ontology ontology = ontology("baseSelection"); + final OntologyProperty property = + new OntologyProperty() + .withId(UUID.randomUUID()) + .withName("code") + .withIri(URI.create("https://example.org/baseSelection/code")) + .withOntology(ontology.getEntityReference()) + .withKind(OntologyPropertyKind.DATA); + final Bundle base = bundle(ontology, property); + property.withDeprecated(true); + final Bundle changed = bundle(ontology, property); + final Bundle unrelated = bundle(ontology, property); + changed.release().withBaseReleaseId(base.release().getId()); + assertEquals(changed.release().getContentDigest(), unrelated.release().getContentDigest()); + final Map> stored = + Map.of( + base.release().getId(), + base.members(), + changed.release().getId(), + changed.members(), + unrelated.release().getId(), + unrelated.members()); + final var validation = validation(stored); + final var access = new OntologyConceptBindingService.Access(null, null); + + final var changedReport = + validation.report(changed.release(), base.release(), ignored -> {}, access); + final var unrelatedReport = validation.report(unrelated.release(), null, ignored -> {}, access); + assertEquals(base.release().getId(), changedReport.getBaseReleaseId()); + assertEquals(1, changedReport.getDeprecationImpacts().size()); + assertEquals( + property.getId(), changedReport.getDeprecationImpacts().getFirst().getDefinition().getId()); + assertTrue(unrelatedReport.getDeprecationImpacts().isEmpty()); + } + + @Test + void cacheEvictsAfterItsMaximumAndDoesNotAdmitOversizedFindings() throws Exception { + final Map> stored = new HashMap<>(); + final List small = new ArrayList<>(); + for (int index = 0; index < 257; index++) { + final Ontology ontology = ontology("model" + index); + final Bundle bundle = bundle(ontology); + stored.put(bundle.release().getId(), bundle.members()); + small.add(bundle); + } + final Ontology ontology = ontology("large"); + final List properties = new ArrayList<>(); + for (int index = 0; index < 1_000; index++) { + properties.add( + new OntologyProperty() + .withId(UUID.randomUUID()) + .withName("status" + index) + .withIri(URI.create("https://example.org/" + "long".repeat(35) + "/status" + index)) + .withOntology(ontology.getEntityReference()) + .withKind(OntologyPropertyKind.DATA) + .withEnumValues(List.of(new OntologyLiteral().withValue("one")))); + } + final Bundle large = bundle(ontology, properties.toArray(OntologyProperty[]::new)); + stored.put(large.release().getId(), large.members()); + final var validation = validation(stored); + final var access = new OntologyConceptBindingService.Access(null, null); + + for (final Bundle bundle : small) { + assertTrue(validation.report(bundle.release(), null, ignored -> {}, access).getValid()); + } + final Cache cache = cache(validation); + cache.cleanUp(); + assertEquals(256, cache.estimatedSize()); + + final var largeReport = validation.report(large.release(), null, ignored -> {}, access); + assertEquals(1_000, largeReport.getShacl().getViolationCount()); + assertEquals(1_000, largeReport.getShacl().getFindings().size()); + assertFalse(cache.asMap().containsKey(large.release().getContentDigest())); + assertEquals(256, cache.estimatedSize()); + } + + @Test + void deprecatedConceptCountsBindingsFromEveryPageIncludingShortPagesWithACursor() { + final Ontology ontology = ontology("pagedBindings"); + final OntologyConcept concept = + new OntologyConcept() + .withId(UUID.randomUUID()) + .withName("Customer") + .withFullyQualifiedName("pagedBindings.Customer") + .withIri(URI.create("https://example.org/pagedBindings/Customer")) + .withOntology(ontology.getEntityReference()) + .withConceptType(OntologyConceptType.OWL_CLASS); + final Bundle base = bundle(ontology, concept); + concept.withDeprecated(true); + final Bundle changed = bundle(ontology, concept); + changed.release().withBaseReleaseId(base.release().getId()); + final var access = new OntologyConceptBindingService.Access(null, null); + final OntologyConceptBindingService bindings = mock(OntologyConceptBindingService.class); + when(bindings.get(eq(access), eq(concept.getId()), eq(100), isNull())) + .thenReturn(page("page-2", IntStream.rangeClosed(2, 101).toArray())); + when(bindings.get(access, concept.getId(), 100, "page-2")).thenReturn(page("page-3")); + when(bindings.get(access, concept.getId(), 100, "page-3")).thenReturn(page(null, 102, 1)); + final var validation = + validation( + Map.of( + base.release().getId(), + base.members(), + changed.release().getId(), + changed.members()), + bindings); + + final NativeOntologyDeprecationImpact impact = + validation + .report(changed.release(), base.release(), ignored -> {}, access) + .getDeprecationImpacts() + .getFirst(); + + assertEquals(concept.getId(), impact.getDefinition().getId()); + assertEquals(102, impact.getBoundAssetCount()); + final List assets = impact.getBoundAssets(); + assertEquals(102, assets.size()); + assertEquals("00000000-0000-0000-0000-000000000001", assets.getFirst().getId().toString()); + assertEquals("table1", assets.getFirst().getName()); + assertEquals("00000000-0000-0000-0000-000000000002", assets.get(1).getId().toString()); + assertEquals("00000000-0000-0000-0000-000000000101", assets.get(100).getId().toString()); + assertEquals("00000000-0000-0000-0000-000000000102", assets.getLast().getId().toString()); + assertEquals("table102", assets.getLast().getName()); + } + + private static OntologyConceptBindingsResponse page(final String after, final int... tables) { + return new OntologyConceptBindingsResponse() + .withRevision(1L) + .withAfter(after) + .withAssets( + IntStream.of(tables) + .mapToObj( + index -> + new OntologyHydratedAssetBinding() + .withAsset( + new EntityReference() + .withId( + UUID.fromString( + String.format("00000000-0000-0000-0000-%012d", index))) + .withType("table") + .withName("table" + index) + .withFullyQualifiedName("service.db.schema.table" + index))) + .toList()); + } + + private static OntologyReleaseValidationService validation( + final Map> stored) { + return validation(stored, null); + } + + private static OntologyReleaseValidationService validation( + final Map> stored, + final OntologyConceptBindingService bindings) { + final OntologyReleaseDAO dao = mock(OntologyReleaseDAO.class); + when(dao.listMembers(any(UUID.class), anyString(), anyString(), eq(100))) + .thenAnswer( + invocation -> { + final String kind = invocation.getArgument(1); + final String after = invocation.getArgument(2); + return stored.get(invocation.getArgument(0, UUID.class)).stream() + .filter(member -> member.getMemberKind().value().equals(kind)) + .filter( + member -> + after.isEmpty() || member.getMemberId().toString().compareTo(after) > 0) + .sorted(Comparator.comparing(member -> member.getMemberId().toString())) + .limit(100) + .toList(); + }); + final CollectionDAO collection = mock(CollectionDAO.class); + when(collection.ontologyReleaseDAO()).thenReturn(dao); + final OntologyReleaseRepository releases = mock(OntologyReleaseRepository.class); + when(releases.getDaoCollection()).thenReturn(collection); + return new OntologyReleaseValidationService(releases, bindings); + } + + private static Cache cache(final OntologyReleaseValidationService validation) + throws ReflectiveOperationException { + final Field field = OntologyReleaseValidationService.class.getDeclaredField("cache"); + field.setAccessible(true); + return (Cache) field.get(validation); + } + + private static Ontology ontology(final String name) { + return new Ontology() + .withId(UUID.randomUUID()) + .withName(name) + .withFullyQualifiedName(name) + .withIri(URI.create("https://example.org/" + name + "/")) + .withNamespace(URI.create("https://example.org/" + name + "/")); + } + + private static Bundle bundle(final Ontology ontology, final OntologyDefinition... definitions) { + final UUID id = UUID.randomUUID(); + final List members = new ArrayList<>(); + members.add( + OntologyReleaseManifest.member( + id, MemberKind.ONTOLOGY, ontology.getId(), OntologyReleaseContent.project(ontology))); + for (final OntologyDefinition definition : definitions) { + final MemberKind kind = + definition instanceof OntologyConcept ? MemberKind.CONCEPT : MemberKind.PROPERTY; + members.add( + OntologyReleaseManifest.member( + id, kind, definition.getId(), OntologyReleaseContent.project(definition))); + } + final OntologyRelease release = + new OntologyRelease() + .withId(id) + .withOntology(ontology.getEntityReference()) + .withOntologyIri(ontology.getIri()) + .withVersionIri(URI.create("urn:openmetadata:ontology-release:" + id)) + .withSealedAt(1L) + .withEncodingVersion( + OntologyRelease.EncodingVersion.fromValue(OntologyReleaseManifest.ENCODING_VERSION)) + .withMemberCount(members.size()) + .withTotalBytes( + members.stream().mapToLong(OntologyReleaseMember::getPayloadBytes).sum()) + .withContentDigest(OntologyReleaseManifest.digest(members)); + return new Bundle(release, members); + } + + private record Bundle(OntologyRelease release, List members) {} +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/rdf/NativeOntologyShapesTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/rdf/NativeOntologyShapesTest.java new file mode 100644 index 000000000000..b869ac9d1118 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/rdf/NativeOntologyShapesTest.java @@ -0,0 +1,210 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.rdf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.shacl.ValidationReport; +import org.apache.jena.vocabulary.OWL2; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.vocabulary.RDFS; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.type.NativeOntologyShaclReport; +import org.openmetadata.schema.type.NativeOntologyShaclSeverity; +import org.openmetadata.service.rdf.RdfShaclValidator.ShapeSet; + +class NativeOntologyShapesTest { + private static final String EX = "https://example.org/ontology/"; + private static final String SHAPES = "https://open-metadata.org/shapes/native-ontology#"; + + private final RdfShaclReportMapper mapper = new RdfShaclReportMapper(); + + @Test + void namedCatalogSelectionRetainsTheDefaultShapes() { + assertSame(RdfShaclValidator.shapes(), RdfShaclValidator.shapes(ShapeSet.CATALOG)); + } + + @Test + void missingOrMalformedShapeResourcesFailClosed() { + assertThrows( + IllegalStateException.class, + () -> RdfShaclValidator.loadShapes("/rdf/shapes/missing-native-test.ttl")); + assertThrows( + IllegalStateException.class, + () -> RdfShaclValidator.loadShapes("/rdf/shapes/malformed-native-test.ttl")); + } + + @Test + void missingClassLabelIsNamedViolation() { + Model model = ModelFactory.createDefaultModel(); + try { + namedClass(model, "Unlabeled"); + NativeOntologyShaclReport report = validate(model); + assertEquals(1, report.getViolationCount()); + assertEquals(0, report.getWarningCount()); + assertEquals(EX + "Unlabeled", report.getFindings().getFirst().getFocusIri().toString()); + assertEquals( + SHAPES + "ClassLabelShape", report.getFindings().getFirst().getShapeIri().toString()); + assertEquals("Class needs an rdfs:label", report.getFindings().getFirst().getMessage()); + } finally { + model.close(); + } + } + + @Test + void labeledClassPasses() { + Model model = ModelFactory.createDefaultModel(); + try { + namedClass(model, "Labeled").addProperty(RDFS.label, "Labeled"); + assertEquals(0, validate(model).getViolationCount()); + } finally { + model.close(); + } + } + + @Test + void zeroAndOneCellEnumsFailAndTwoCellsPass() { + for (int size = 0; size <= 2; size++) { + Model model = ModelFactory.createDefaultModel(); + try { + Resource property = + model.createResource(EX + "status").addProperty(RDF.type, OWL2.DatatypeProperty); + Resource range = model.createResource(EX + "statusValues"); + property.addProperty(RDFS.range, range); + range.addProperty( + OWL2.equivalentClass, + model + .createResource() + .addProperty( + OWL2.oneOf, + model.createList( + List.of(model.createLiteral("same"), model.createLiteral("same")) + .subList(0, size) + .iterator()))); + NativeOntologyShaclReport report = validate(model); + assertEquals(size < 2 ? 1 : 0, report.getViolationCount(), "list size " + size); + if (size < 2) { + assertEquals(EX + "status", report.getFindings().getFirst().getFocusIri().toString()); + assertEquals( + SHAPES + "EnumerationShape", + report.getFindings().getFirst().getShapeIri().toString()); + } + } finally { + model.close(); + } + } + } + + @Test + void propertyWithoutEnumerationPasses() { + Model model = ModelFactory.createDefaultModel(); + try { + model.createResource(EX + "status").addProperty(RDF.type, OWL2.DatatypeProperty); + assertEquals(0, validate(model).getViolationCount()); + } finally { + model.close(); + } + } + + @Test + void findingsCapPreservesTotalViolationCount() { + Model model = ModelFactory.createDefaultModel(); + try { + for (int index = 0; index < 1_001; index++) { + namedClass(model, "Unlabeled" + index); + } + Resource deprecated = + namedClass(model, "Deprecated") + .addProperty(RDFS.label, "Deprecated") + .addLiteral(OWL2.deprecated, true); + namedClass(model, "Child") + .addProperty(RDFS.label, "Child") + .addProperty(RDFS.subClassOf, deprecated); + NativeOntologyShaclReport report = validate(model); + assertEquals(1_001, report.getViolationCount()); + assertEquals(1, report.getWarningCount()); + assertEquals(1_000, report.getFindings().size()); + assertTrue(report.getTruncated()); + assertEquals( + SHAPES + "DeprecatedParentShape", + report.getFindings().getFirst().getShapeIri().toString()); + } finally { + model.close(); + } + } + + @Test + void directDeprecatedParentWarnsOnlyForActiveChild() { + Model model = ModelFactory.createDefaultModel(); + try { + Resource parent = + namedClass(model, "Parent") + .addProperty(RDFS.label, "Parent") + .addLiteral(OWL2.deprecated, true); + Resource child = + namedClass(model, "Child") + .addProperty(RDFS.label, "Child") + .addProperty(RDFS.subClassOf, parent); + NativeOntologyShaclReport warning = validate(model); + assertEquals(0, warning.getViolationCount()); + assertEquals(1, warning.getWarningCount()); + assertEquals( + NativeOntologyShaclSeverity.WARNING, warning.getFindings().getFirst().getSeverity()); + assertEquals( + SHAPES + "DeprecatedParentShape", + warning.getFindings().getFirst().getShapeIri().toString()); + child.addLiteral(OWL2.deprecated, true); + assertEquals(0, validate(model).getWarningCount()); + } finally { + model.close(); + } + } + + @Test + void anonymousOrNonDeprecatedParentDoesNotWarn() { + Model model = ModelFactory.createDefaultModel(); + try { + Resource child = namedClass(model, "Child").addProperty(RDFS.label, "Child"); + child.addProperty(RDFS.subClassOf, model.createResource().addLiteral(OWL2.deprecated, true)); + assertEquals(0, validate(model).getWarningCount()); + assertFalse( + RdfShaclValidator.validate(model, ShapeSet.CATALOG).getEntries().stream() + .anyMatch( + entry -> + SHAPES + .concat("DeprecatedParentShape") + .equals(String.valueOf(entry.source())))); + } finally { + model.close(); + } + } + + private NativeOntologyShaclReport validate(Model model) { + ValidationReport report = RdfShaclValidator.validate(model, ShapeSet.NATIVE_ONTOLOGY); + return mapper.mapNative(report); + } + + private static Resource namedClass(Model model, String localName) { + return model.createResource(EX + localName).addProperty(RDF.type, OWL2.Class); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/rdf/OntologyReleaseProjectionTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/rdf/OntologyReleaseProjectionTest.java index 3a5d403dc9f5..b8087571204d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/rdf/OntologyReleaseProjectionTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/rdf/OntologyReleaseProjectionTest.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -38,6 +39,7 @@ import org.openmetadata.schema.entity.data.OntologyProperty; import org.openmetadata.schema.entity.data.OntologyRelease; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.NativeOntologyShaclReport; import org.openmetadata.schema.type.OntologyAxiomType; import org.openmetadata.schema.type.OntologyConceptType; import org.openmetadata.schema.type.OntologyExpression; @@ -150,6 +152,111 @@ void classModelFieldsProjectAsStandardAnnotations() { } } + @Test + void referencedOnlyForeignClassDoesNotFailCapturedValidation() { + final Bundle bundle = + bundle( + List.of( + property("related") + .withDomain(List.of(URI.create("https://foreign.example.org/Party"))))); + final Model model = new OntologyReleaseProjection().build(bundle.release(), bundle.members()); + try { + assertTrue( + model.contains( + model.createResource("https://foreign.example.org/Party"), RDF.type, OWL2.Class)); + final NativeOntologyShaclReport report = + new NativeOntologyValidation().validate(bundle.release(), bundle.members()); + assertEquals(0, report.getViolationCount()); + } finally { + model.close(); + } + } + + @Test + void capturedEnumerationUsesProjectedListCells() { + final OntologyProperty property = + property("status") + .withKind(OntologyPropertyKind.DATA) + .withEnumValues(List.of(new OntologyLiteral().withValue("one"))); + final Bundle singleton = bundle(List.of(property)); + assertEquals( + 1, + new NativeOntologyValidation() + .validate(singleton.release(), singleton.members()) + .getViolationCount()); + final Bundle pair = + bundle( + List.of( + property.withEnumValues( + List.of( + new OntologyLiteral().withValue("one"), + new OntologyLiteral().withValue("two"))))); + assertEquals( + 0, + new NativeOntologyValidation() + .validate(pair.release(), pair.members()) + .getViolationCount()); + } + + @Test + void capturedTargetsKeepRawShapesAndExportUnchanged() { + final Bundle bundle = + bundle( + List.of( + property("related") + .withDomain(List.of(URI.create("https://foreign.example.org/Party"))))); + final Model model = new OntologyReleaseProjection().build(bundle.release(), bundle.members()); + try { + final String before = RdfGraphSerializer.asString(model, RdfSerializationFormat.TURTLE); + assertEquals( + 1, + new RdfShaclReportMapper() + .mapNative( + RdfShaclValidator.validate(model, RdfShaclValidator.ShapeSet.NATIVE_ONTOLOGY)) + .getViolationCount()); + assertEquals( + 0, + new NativeOntologyValidation() + .validate(bundle.release(), bundle.members()) + .getViolationCount()); + assertEquals(before, RdfGraphSerializer.asString(model, RdfSerializationFormat.TURTLE)); + assertEquals( + 1, + new RdfShaclReportMapper() + .mapNative( + RdfShaclValidator.validate(model, RdfShaclValidator.ShapeSet.NATIVE_ONTOLOGY)) + .getViolationCount()); + } finally { + model.close(); + } + } + + @Test + void disjointConcurrentReleasesKeepTheirOwnTargets() { + final URI foreign = URI.create("https://foreign.example.org/Party"); + final Bundle first = + bundle( + List.of( + concept("Party").withIri(foreign).withDeprecated(true), + concept("Customer"), + axiom("subclass") + .withExpressions( + List.of( + new OntologyExpression() + .withKind(OntologyExpressionKind.NAMED_CLASS) + .withClassIri(foreign))))); + final Bundle second = bundle(List.of(property("related").withDomain(List.of(foreign)))); + final NativeOntologyValidation validator = new NativeOntologyValidation(); + final CompletableFuture firstResult = + CompletableFuture.supplyAsync(() -> validator.validate(first.release(), first.members())); + final CompletableFuture secondResult = + CompletableFuture.supplyAsync(() -> validator.validate(second.release(), second.members())); + assertEquals(1, firstResult.join().getWarningCount()); + assertEquals(0, firstResult.join().getViolationCount()); + assertEquals(0, secondResult.join().getWarningCount()); + assertEquals(0, secondResult.join().getViolationCount()); + } + @ParameterizedTest @EnumSource(OntologyAxiomType.class) void everyAxiomKindReimportsWithoutDuplicateOperations(OntologyAxiomType kind) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/rdf/RdfShaclReportMapperTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/rdf/RdfShaclReportMapperTest.java index 532068ca35b0..f5756f15cf4b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/rdf/RdfShaclReportMapperTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/rdf/RdfShaclReportMapperTest.java @@ -40,6 +40,17 @@ class RdfShaclReportMapperTest { private final RdfShaclReportMapper mapper = new RdfShaclReportMapper(); + @Test + void skippedCatalogReportKeepsItsExistingEmptyContract() { + RdfValidationReport report = mapper.skipped(); + assertTrue(report.getConforms()); + assertFalse(report.getPerformed()); + assertEquals(0, report.getViolationCount()); + assertEquals("", report.getReportTurtle()); + assertFalse(report.getTruncated()); + assertTrue(report.getViolations().isEmpty()); + } + @Test void severityIsShaclIriNotObjectHash() { ValidationReport report = diff --git a/openmetadata-service/src/test/resources/rdf/shapes/malformed-native-test.ttl b/openmetadata-service/src/test/resources/rdf/shapes/malformed-native-test.ttl new file mode 100644 index 000000000000..a333ba9609bd --- /dev/null +++ b/openmetadata-service/src/test/resources/rdf/shapes/malformed-native-test.ttl @@ -0,0 +1,3 @@ +@prefix sh: . + a sh:NodeShape ; +not-a-turtle-token! diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/ontologyReleaseValidation.json b/openmetadata-spec/src/main/resources/json/schema/api/data/ontologyReleaseValidation.json new file mode 100644 index 000000000000..e3f1fbe31091 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/ontologyReleaseValidation.json @@ -0,0 +1,70 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/ontologyReleaseValidation.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OntologyReleaseValidation", + "description": "Structure and native SHACL findings for a captured release, with current authorized bindings for newly deprecated definitions. Logical consistency is not checked.", + "type": "object", + "javaType": "org.openmetadata.schema.api.data.OntologyReleaseValidation", + "definitions": { + "logicalCheck": { + "type": "string", + "javaType": "org.openmetadata.schema.type.OntologyReleaseLogicalCheck", + "enum": ["NOT_RUN"] + }, + "severity": { + "type": "string", + "javaType": "org.openmetadata.schema.type.NativeOntologyShaclSeverity", + "enum": ["VIOLATION", "WARNING"] + }, + "finding": { + "type": "object", + "javaType": "org.openmetadata.schema.type.NativeOntologyShaclFinding", + "properties": { + "focusIri": {"$ref": "../../type/ontologyIri.json"}, + "shapeIri": {"$ref": "../../type/ontologyIri.json"}, + "severity": {"$ref": "#/definitions/severity"}, + "message": {"type": "string"} + }, + "required": ["focusIri", "shapeIri", "severity", "message"], + "additionalProperties": false + }, + "shacl": { + "type": "object", + "javaType": "org.openmetadata.schema.type.NativeOntologyShaclReport", + "properties": { + "violationCount": {"type": "integer", "minimum": 0}, + "warningCount": {"type": "integer", "minimum": 0}, + "truncated": {"type": "boolean"}, + "findings": {"type": "array", "items": {"$ref": "#/definitions/finding"}, "maxItems": 1000, "default": []} + }, + "required": ["violationCount", "warningCount", "truncated", "findings"], + "additionalProperties": false + }, + "impact": { + "type": "object", + "javaType": "org.openmetadata.schema.type.NativeOntologyDeprecationImpact", + "properties": { + "definition": {"$ref": "../../type/entityReference.json"}, + "boundAssetCount": {"type": "integer", "minimum": 0}, + "boundAssets": {"type": "array", "items": {"$ref": "../../type/entityReference.json"}, "default": []}, + "subclasses": {"type": "array", "items": {"$ref": "../../type/entityReference.json"}, "default": []}, + "domainProperties": {"type": "array", "items": {"$ref": "../../type/entityReference.json"}, "default": []}, + "rangeProperties": {"type": "array", "items": {"$ref": "../../type/entityReference.json"}, "default": []} + }, + "required": ["definition", "boundAssetCount", "boundAssets", "subclasses", "domainProperties", "rangeProperties"], + "additionalProperties": false + } + }, + "properties": { + "releaseId": {"$ref": "../../type/basic.json#/definitions/uuid"}, + "contentDigest": {"$ref": "../../entity/data/ontologyRelease.json#/properties/contentDigest"}, + "baseReleaseId": {"$ref": "../../type/basic.json#/definitions/uuid"}, + "valid": {"type": "boolean"}, + "logicalCheck": {"$ref": "#/definitions/logicalCheck"}, + "structure": {"$ref": "ontologyReleaseStructureReport.json"}, + "shacl": {"$ref": "#/definitions/shacl"}, + "deprecationImpacts": {"type": "array", "items": {"$ref": "#/definitions/impact"}, "default": []} + }, + "required": ["releaseId", "contentDigest", "valid", "logicalCheck", "structure", "shacl", "deprecationImpacts"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/rdf/shapes/native-ontology-shapes.ttl b/openmetadata-spec/src/main/resources/rdf/shapes/native-ontology-shapes.ttl new file mode 100644 index 000000000000..7234d85a995b --- /dev/null +++ b/openmetadata-spec/src/main/resources/rdf/shapes/native-ontology-shapes.ttl @@ -0,0 +1,45 @@ +@prefix native: . +@prefix sh: . +@prefix owl: . +@prefix rdfs: . + +native:ClassShape a sh:NodeShape ; + sh:targetClass owl:Class ; + sh:property native:ClassLabelShape . + +native:ClassLabelShape a sh:PropertyShape ; + sh:path rdfs:label ; + sh:minCount 1 ; + sh:message "Class needs an rdfs:label" . + +native:EnumerationShape a sh:NodeShape ; + sh:targetClass owl:DatatypeProperty ; + sh:sparql [ + sh:message "Enumeration needs at least two list entries" ; + sh:select """ + PREFIX rdfs: + PREFIX owl: + PREFIX rdf: + SELECT DISTINCT $this WHERE { + $this rdfs:range / owl:equivalentClass / owl:oneOf ?list . + FILTER (?list = rdf:nil || EXISTS { ?list rdf:rest rdf:nil }) + } + """ ; + ] . + +native:DeprecatedParentShape a sh:NodeShape ; + sh:targetClass owl:Class ; + sh:severity sh:Warning ; + sh:sparql [ + sh:message "Class has a direct deprecated parent" ; + sh:select """ + PREFIX rdfs: + PREFIX owl: + SELECT DISTINCT $this WHERE { + $this rdfs:subClassOf ?parent . + FILTER isIRI(?parent) + ?parent owl:deprecated true . + FILTER NOT EXISTS { $this owl:deprecated true } + } + """ ; + ] . diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/ontologyReleaseValidation.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/ontologyReleaseValidation.ts new file mode 100644 index 000000000000..b900141df379 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/ontologyReleaseValidation.ts @@ -0,0 +1,198 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Structure and native SHACL findings for a captured release, with current authorized + * bindings for newly deprecated definitions. Logical consistency is not checked. + */ +export interface OntologyReleaseValidation { + baseReleaseId?: string; + contentDigest: string; + deprecationImpacts: Impact[]; + logicalCheck: LogicalCheck; + releaseId: string; + shacl: Shacl; + structure: OntologyReleaseStructureReport; + valid: boolean; +} + +export interface Impact { + boundAssetCount: number; + boundAssets: EntityReference[]; + definition: EntityReference; + domainProperties: EntityReference[]; + rangeProperties: EntityReference[]; + subclasses: EntityReference[]; +} + +/** + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface EntityReference { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + +export enum LogicalCheck { + NotRun = "NOT_RUN", +} + +export interface Shacl { + findings: Finding[]; + truncated: boolean; + violationCount: number; + warningCount: number; +} + +export interface Finding { + focusIri: string; + message: string; + severity: FindingSeverity; + shapeIri: string; +} + +export enum FindingSeverity { + Violation = "VIOLATION", + Warning = "WARNING", +} + +/** + * Declaration, reference-kind and axiom-shape checks over a sealed release and its pinned + * imports. This is not a complete OWL 2 DL profile check, a logical consistency result, or + * publication approval. + */ +export interface OntologyReleaseStructureReport { + contentDigest: string; + releaseId: string; + /** + * True only when these structural checks find no violations. Other required publication + * checks are independent. + */ + valid: boolean; + /** + * Total findings, including those beyond the response limit. + */ + violationCount: number; + /** + * At most 1000 findings in release/member order. Compare the length with violationCount to + * detect truncation. + */ + violations: Issue[]; +} + +export interface Issue { + memberId: string; + memberKind: MemberKind; + /** + * Owning release when validating captured content; absent for a working definition. + */ + releaseId?: string; + violation: OntologyProfileViolation; +} + +/** + * Unique id used to identify an entity. + * + * Unique identifier that identifies an entity instance. + * + * Owning release when validating captured content; absent for a working definition. + * + * Optional description of entity. + * + * Text in Markdown format. + * + * Link to the entity resource. + * + * URI that points to a resource. + * + * Absolute semantic IRI, preserved exactly without URL normalization, case folding, or + * percent encoding. + */ +export enum MemberKind { + Association = "ASSOCIATION", + Axiom = "AXIOM", + Concept = "CONCEPT", + Ontology = "ONTOLOGY", + Property = "PROPERTY", + Shape = "SHAPE", +} + +/** + * A typed OWL 2 DL profile guardrail finding. + */ +export interface OntologyProfileViolation { + code: Code; + message: string; + path: string; + severity: ViolationSeverity; +} + +export enum Code { + IllegalPunning = "ILLEGAL_PUNNING", + InvalidAxiomShape = "INVALID_AXIOM_SHAPE", + InvalidClassKind = "INVALID_CLASS_KIND", + InvalidExpressionShape = "INVALID_EXPRESSION_SHAPE", + NonSimpleCardinality = "NON_SIMPLE_CARDINALITY", + NonSimpleProperty = "NON_SIMPLE_PROPERTY", + PropertyChainRegularity = "PROPERTY_CHAIN_REGULARITY", + ReservedSubjectIRI = "RESERVED_SUBJECT_IRI", + SubclassCycle = "SUBCLASS_CYCLE", + TransitiveCardinality = "TRANSITIVE_CARDINALITY", + UndeclaredClass = "UNDECLARED_CLASS", + UndeclaredProperty = "UNDECLARED_PROPERTY", +} + +export enum ViolationSeverity { + Error = "ERROR", + Warning = "WARNING", +}