[Master Data Management] Fixing synchronization of contacts, media, date, datetime, source watermarking, permission issue and minor UX issues

src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al

@@ -19,6 +19,7 @@ using Microsoft.Purchases.Vendor;
 using Microsoft.Sales.Customer;
 using Microsoft.Sales.Setup;
 using System.Environment;
+using System.Privacy;
 
 /// <summary>
 /// Assigned to the customer-registered Entra app on the SOURCE (Microsoft Entra Application Card) to grant
@@ -66,5 +67,8 @@ permissionset 7242 "MDM Cross-Env Read"
                   tabledata "Tax Area" = R,
                   tabledata "Tax Group" = R,
                   tabledata "Tax Jurisdiction" = R,
-                  tabledata "Tenant Media" = R;
+                  tabledata "Tenant Media" = R,
+                  // Read the consent tables so the source can answer "not consented" as a structured signal instead of throwing a permission error (surfaced as a bare HTTP 403).
+                  tabledata "Privacy Notice" = R,
+                  tabledata "Privacy Notice Approval" = R;
 }

src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al

@@ -258,8 +258,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source"
     begin
         SourceRecordRef.Close();
         SourceRecordRef.Open(IntegrationTableId, true);
-        InlineMedia.Reset(); // fresh fetch: drop any prior inline media bytes
-        SourceWatermark.Reset();
+        // Keep the batch's per-batch caches (inline media + source watermarks): GetBySystemId/GetById/GetByUidFilter
+        // run during the write loop (coupling, conflict checks), so resetting here would drop the pictures the apply
+        // step still needs and the real source timestamps GetRowLastModifiedOn reads back. Only the batch fetch
+        // (GetModifiedBatch) resets them.
         if SystemIds.Count() = 0 then
             exit;
         Transport := GetTransport();
@@ -281,8 +283,8 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source"
     begin
         SourceRecordRef.Close();
         SourceRecordRef.Open(TableId, true);
-        InlineMedia.Reset();
-        SourceWatermark.Reset();
+        // Keep the current batch's per-batch caches (see FetchBySystemIds): a related-table read during the write
+        // loop must not drop the batch's inline media or source watermarks.
         Transport := GetTransport();
         SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok);
         // Full read from the start (selector '{}' = no watermark), narrowed to the related rows by the row filter.
@@ -303,8 +305,8 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source"
         NotIndexed := false;
         SourceRecordRef.Close();
         SourceRecordRef.Open(TableId, true);
-        InlineMedia.Reset();
-        SourceWatermark.Reset();
+        // Keep the current batch's per-batch caches (see FetchBySystemIds): a related-table read during the write
+        // loop must not drop the batch's inline media or source watermarks.
         Transport := GetTransport();
         SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok);
         if not TryParseCleanResponse(Transport.GetRecords(TableId, BuildFieldIdsForTable(TableId), '{}', PageSize(), RowFilter), Response, NotIndexed) then

src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al

@@ -30,6 +30,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport"
         InvalidSourceUrlErr: Label 'The source environment URL is not a valid Business Central endpoint.';
         InvalidSourceUrlAuditTxt: Label 'Blocked a cross-environment request: the configured source environment URL host ''%1'' is not a valid Business Central endpoint.', Comment = '%1 = the rejected host';
         HttpErr: Label 'The source environment returned HTTP %1.', Comment = '%1 = HTTP status code';
+        AccessDeniedErr: Label 'The source environment %1 denied access (HTTP %2). On the source environment, assign the registered app the ''Master Data Mgt. - Cross Environment'' permission set and approve the cross-environment privacy notice, then try again.', Comment = '%1 = source environment name, %2 = HTTP status code';
         ServiceNameTok: Label 'MDMCrossEnvSource', Locked = true;
         ScopeTok: Label 'https://api.businesscentral.dynamics.com/.default', Locked = true;
         ScopePPETok: Label 'https://api.businesscentral.dynamics-tie.com/.default', Locked = true;
@@ -97,7 +98,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport"
                     exit(UnwrapODataValue(ResponseBodyText));
                 if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin
                     LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage);
-                    Error(SetupNavigationError(StrSubstNo(HttpErr, ResponseMessage.HttpStatusCode())));
+                    Error(SetupNavigationError(HttpStatusErrorMessage(MasterDataManagementSetup, ResponseMessage.HttpStatusCode())));
                 end;
                 Sleep(RetryAfter);
             end else begin
@@ -111,6 +112,15 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport"
             end;
     end;
 
+    // 401/403 is an authorization failure on the SOURCE (missing permission set or unapproved privacy notice), so
+    // return a remedy the source admin can act on; other status codes keep the generic message.
+    local procedure HttpStatusErrorMessage(var MasterDataManagementSetup: Record "Master Data Management Setup"; StatusCode: Integer): Text
+    begin
+        if StatusCode in [401, 403] then
+            exit(StrSubstNo(AccessDeniedErr, MasterDataManagementSetup."Source Environment Name", StatusCode));
+        exit(StrSubstNo(HttpErr, StatusCode));
+    end;
+
     local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage)
     var
         AuditLog: Codeunit "Audit Log";

src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al

@@ -74,6 +74,7 @@ codeunit 7233 "Master Data Management"
         EmptyIntegrationRecordSystemIdErr: Label 'Empty integration record system id.', Locked = true;
         DeletionConflictHandledRemoveCouplingTxt: Label 'Deletion conflict handled by removing the coupling to the deleted record.', Locked = true;
         DeletionConflictHandledRestoreRecordTxt: Label 'Deletion conflict handled by restoring the deleted record.', Locked = true;
+        DeletionConflictSourceRecordErr: Label 'The source record %1 cannot be synchronized because the record it is coupled to has been deleted. Restore the deleted record, or choose a Deletion-Conflict Resolution for this table on the Synchronization Tables page.', Comment = '%1 = the source record identifier';
         ResetAllCustomIntegrationTableMappingsLbl: Label 'One or more of the selected integration table mappings is custom. \\To restore a custom table mapping, you must subscribe to the event OnBeforeResetTableMapping in codeunit "Master Data Mgt. Setup Default" and implement the defaults for each custom table mapping. \\Do you want to continue?';
         DeletedRecordWithZeroTableIdTxt: Label 'CRM Integration Record with zero Table ID has been deleted. Integration ID: %1, CRM ID: %2', Locked = true;
         AllRecordsMarkedAsSkippedTxt: Label 'All of selected %1 records are marked as skipped.', Comment = '%1 = table caption';
@@ -1181,7 +1182,12 @@ codeunit 7233 "Master Data Management"
     end;
 
     [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Rec. Synch. Invoke", 'OnDeletionConflictDetected', '', false, false)]
-    local procedure HandleOnDeletionConflictDetected(var IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef; var DeletionConflictHandled: Boolean)
+    local procedure OnDeletionConflictDetected(var IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef; var DeletionConflictHandled: Boolean)
+    begin
+        HandleOnDeletionConflictDetected(IntegrationTableMapping, SourceRecordRef, DeletionConflictHandled);
+    end;
+
+    internal procedure HandleOnDeletionConflictDetected(var IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef; var DeletionConflictHandled: Boolean)
     var
         IntegrationSystemId: Guid;
     begin
@@ -1215,6 +1221,11 @@ codeunit 7233 "Master Data Management"
                     Session.LogMessage('0000J89', DeletionConflictHandledRestoreRecordTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', CategoryTok);
                 end;
         end;
+
+        // The platform's default deletion-conflict error names only the table; when MDM hasn't resolved the conflict
+        // (e.g. Deletion-Conflict Resolution = None), fail with a message that identifies the specific source record.
+        if (not DeletionConflictHandled) and (IntegrationTableMapping.Type = IntegrationTableMapping.Type::"Master Data Management") then
+            Error(DeletionConflictSourceRecordErr, Format(SourceRecordRef.RecordId(), 0, 1));
     end;
 
     [EventSubscriber(ObjectType::Page, Page::"My Notifications", 'OnInitializingNotificationWithDefaultState', '', false, false)]

src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al

@@ -269,6 +269,15 @@ codeunit 7237 "Master Data Mgt. Subscribers"
             end;
         end;
 
+        // The BaseApp sync engine does not transfer a blanked (0D/0DT) Date/DateTime, so clearing it on the source
+        // leaves the subsidiary value stale; force the blank through so the clear is mirrored (bug 648540).
+        if SourceFieldRef.Type in [SourceFieldRef.Type::Date, SourceFieldRef.Type::DateTime] then
+            if IsBlankDateTime(SourceFieldRef) and not IsBlankDateTime(DestinationFieldRef) then begin
+                NewValue := SourceFieldRef.Value();
+                IsValueFound := true;
+                NeedsConversion := false;
+            end;
+
         if SourceFieldRef.Value() <> DestinationFieldRef.Value() then begin
             DestinationRecordRef := DestinationFieldRef.Record();
             DestinationRecCreatedAt := DestinationRecordRef.Field(DestinationRecordRef.SystemCreatedAtNo()).Value();
@@ -311,6 +320,27 @@ codeunit 7237 "Master Data Mgt. Subscribers"
         end;
     end;
 
+    // A blanked Date/DateTime reads as 0D/0DT; the BaseApp transfer treats that as "no value" and skips it.
+    local procedure IsBlankDateTime(FieldReference: FieldRef): Boolean
+    var
+        DateValue: Date;
+        DateTimeValue: DateTime;
+    begin
+        case FieldReference.Type of
+            FieldReference.Type::Date:
+                begin
+                    DateValue := FieldReference.Value();
+                    exit(DateValue = 0D);
+                end;
+            FieldReference.Type::DateTime:
+                begin
+                    DateTimeValue := FieldReference.Value();
+                    exit(DateTimeValue = 0DT);
+                end;
+        end;
+        exit(false);
+    end;
+
     local procedure UpdateMedia(var SourceFieldRef: FieldRef; var DestinationFieldRef: FieldRef; var NewValue: Variant): Boolean
     var
         SourceTenantMedia, DestinationTenantMedia : Record "Tenant Media";

src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al

@@ -12,7 +12,7 @@ page 7232 "MDM Connection Details"
     PageType = NavigatePage;
     ApplicationArea = Suite;
     UsageCategory = None;
-    Permissions = tabledata "Master Data Management Setup" = imd;
+    Permissions = tabledata "Master Data Management Setup" = rimd;
 
     layout
     {

src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataFullSynchReview.Page.al

@@ -11,7 +11,7 @@ page 7234 "Master Data Full Synch. Review"
     InsertAllowed = false;
     ModifyAllowed = false;
     DeleteAllowed = true;
-    Permissions = tabledata "Master Data Full Synch. R. Ln." = imd;
+    Permissions = tabledata "Master Data Full Synch. R. Ln." = rimd;
 
     layout
     {

src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al

@@ -17,7 +17,7 @@ page 7230 "Master Data Management Setup"
     SourceTable = "Master Data Management Setup";
     UsageCategory = Administration;
     AdditionalSearchTerms = 'mdm,master data';
-    Permissions = tabledata "Master Data Management Setup" = imd;
+    Permissions = tabledata "Master Data Management Setup" = rimd;
 
     layout
     {
@@ -29,7 +29,7 @@ page 7230 "Master Data Management Setup"
                 {
                     ApplicationArea = Suite;
                     Editable = IsEditable;
-                    Visible = not CrossEnvConfigured; // same-environment source company; hidden once a source environment is configured
+                    Visible = not (CrossEnvConfigured and Rec."Is Enabled"); // same-env source company: blank under cross-env, so hide it once cross-env is set up and enabled (view cross-env details via the wizard)
                     ToolTip = 'Specifies the name of the source company that you synchronize data from.';
                 }
                 field("Source Environment Name"; Rec."Source Environment Name")

src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtCoupling.Table.al

@@ -221,6 +221,7 @@ table 7231 "Master Data Mgt. Coupling"
 
     internal procedure InsertRecord(IntegrationSysID: Guid; SysId: Guid; TableId: Integer)
     var
+        ExistingCoupling: Record "Master Data Mgt. Coupling";
         LocalRecordRef: RecordRef;
         EmptyGuid: Guid;
     begin
@@ -234,6 +235,11 @@ table 7231 "Master Data Mgt. Coupling"
                 exit;
         end;
 
+        // Another sync job (e.g. the customer/vendor run that auto-created and coupled the company contact) can create
+        // this exact coupling first; treat that as done instead of failing the whole record on the duplicate key.
+        if ExistingCoupling.Get(IntegrationSysID, SysId) then
+            exit;
+
         Reset();
         Init();
         "Integration System ID" := IntegrationSysID;

src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al

@@ -16,6 +16,15 @@ codeunit 139757 "Library - Master Data Mgt."
         MasterDataMgtSubscribers.HandleOnTransferFieldData(SourceFieldRef, DestinationFieldRef, NewValue, IsValueFound, NeedsConversion);
     end;
 
+    /// <summary>Invokes the deletion-conflict subscriber logic that resolves or fails a coupled-to-deleted-record conflict.</summary>
+    /// <param name="IntegrationTableMapping">The integration table mapping being synchronized.</param>
+    /// <param name="SourceRecordRef">The source record whose coupled record was deleted.</param>
+    /// <param name="DeletionConflictHandled">Returns whether the conflict was resolved.</param>
+    procedure HandleOnDeletionConflictDetected(var IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef; var DeletionConflictHandled: Boolean)
+    begin
+        MasterDataManagement.HandleOnDeletionConflictDetected(IntegrationTableMapping, SourceRecordRef, DeletionConflictHandled);
+    end;
+
     /// <summary>Renames the destination record before modification when the source primary key has changed.</summary>
     /// <param name="IntegrationTableMapping">The integration table mapping being synchronized.</param>
     /// <param name="SourceRecordRef">The source record providing the new primary key.</param>
@@ -102,6 +111,17 @@ codeunit 139757 "Library - Master Data Mgt."
         exit(MasterDataManagement.GetIntegrationRecordRef(IntegrationTableID, MasterDataMgtCoupling, RecRef));
     end;
 
+    /// <summary>Creates a Master Data Mgt. coupling via the table's InsertRecord (used to exercise duplicate-coupling resilience).</summary>
+    /// <param name="IntegrationSystemId">The source record SystemId.</param>
+    /// <param name="LocalSystemId">The local record SystemId.</param>
+    /// <param name="TableId">The coupled local table ID.</param>
+    procedure InsertCoupling(IntegrationSystemId: Guid; LocalSystemId: Guid; TableId: Integer)
+    var
+        MasterDataMgtCoupling: Record "Master Data Mgt. Coupling";
+    begin
+        MasterDataMgtCoupling.InsertRecord(IntegrationSystemId, LocalSystemId, TableId);
+    end;
+
     /// <summary>Gets the integration record reference identified by a coupling ID.</summary>
     /// <param name="IntegrationTableMapping">The integration table mapping to resolve against.</param>
     /// <param name="ID">The coupling ID to resolve.</param>

src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al

@@ -29,6 +29,14 @@ table 139757 "MDM Test Table A"
         {
             Caption = 'Test Image';
         }
+        field(6; "Test Date"; Date)
+        {
+            Caption = 'Test Date';
+        }
+        field(7; "Test DateTime"; DateTime)
+        {
+            Caption = 'Test DateTime';
+        }
     }
 
     keys