diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h
index 5f9c01f5ec0170..cdb5dbdb124fb3 100644
--- a/be/src/service/backend_service.h
+++ b/be/src/service/backend_service.h
@@ -73,6 +73,16 @@ class BaseBackendService : public BackendServiceIf {
_agent_server->submit_tasks(return_value, tasks);
}
+ // One-shot Lance index mutation dispatch. The isolated worker lands in a later
+ // slice; until then the request is answered as definitively NOT enqueued, so the
+ // FE classifies a trusted pre-invocation rejection (terminal NOT_COMMITTED)
+ // instead of an ambiguous result.
+ void submit_lance_index_job(TStatus& _return,
+ const TLanceIndexJobDispatch& dispatch) override {
+ _return.__set_status_code(TStatusCode::NOT_IMPLEMENTED_ERROR);
+ _return.__set_error_msgs({"lance index worker is not available in this build"});
+ }
+
void publish_cluster_state(TAgentResult& result, const TAgentPublishRequest& request) override {
_agent_server->publish_cluster_state(result, request);
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 08e14071a7f9fc..1ef5cfa2e08eda 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -4265,4 +4265,48 @@ public void handle(Field field, String value) throws Exception {
"Static upper bound for num_sub_vectors of Lance IVF_PQ indexes."})
public static int lance_index_max_num_sub_vectors = 256;
+ @ConfField(mutable = true, masterOnly = true,
+ callback = LanceIndexConfigValidator.PositiveIntConfigHandler.class,
+ description = {"Lance 索引 job 派发器(含 deadline/possible-live 扫掠与 refresh 驱动)的轮询周期(秒)。",
+ "Polling interval in seconds of the Lance index job dispatcher "
+ + "(dispatch sweep, deadline/possible-live sweeps, and refresh driver)."})
+ public static int lance_index_job_dispatch_interval_second = 10;
+
+ @ConfField(mutable = true, masterOnly = true,
+ callback = LanceIndexConfigValidator.PositiveLongConfigHandler.class,
+ description = {"单个 Lance 索引 job 派发后的结果等待上限(秒)。到期仍无完整可信结果即收敛为 UNKNOWN;"
+ + "该期限只限定等待,不证明终止,也不释放 possible-live 槽位。",
+ "Wait bound in seconds for the result of one dispatched Lance index job. Expiry without "
+ + "a complete trusted result converges the job to UNKNOWN; the deadline bounds the wait "
+ + "only, never proves termination, and never releases a possible-live slot."})
+ public static long lance_index_job_execute_deadline_second = 3600;
+
+ @ConfField(mutable = true, masterOnly = true,
+ callback = LanceIndexConfigValidator.PositiveIntConfigHandler.class,
+ description = {"派发器单轮最多新派发的 Lance 索引 job 数(背压上限)。",
+ "Maximum number of Lance index jobs newly dispatched per dispatcher round (backpressure)."})
+ public static int lance_index_job_max_dispatch_per_round = 16;
+
+ @ConfField(mutable = true, masterOnly = true,
+ callback = LanceIndexConfigValidator.PositiveIntConfigHandler.class,
+ description = {"单个 BE 上允许同时在途(RUNNING)的 Lance 索引 job 数上限。",
+ "Maximum number of in-flight (RUNNING) Lance index jobs per backend."})
+ public static int lance_index_job_max_inflight_per_backend = 2;
+
+ @ConfField(mutable = true, masterOnly = true,
+ callback = LanceIndexConfigValidator.PositiveIntConfigHandler.class,
+ description = {"refresh 失败的 Lance 索引 job 的最小重试间隔(秒);首次刷新不受此间隔限制。",
+ "Minimum retry interval in seconds for a terminal Lance index job whose metadata "
+ + "refresh FAILED; the first refresh attempt is never delayed by this interval."})
+ public static int lance_index_job_refresh_retry_second = 300;
+
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "是否允许 file:// 本地路径上的 Lance 索引变更派发(运维断言,默认关闭)。开启后派发仍要求"
+ + "集群恰一台 FE 且目标 BE 是唯一存活 BE;对象存储是生产形态。",
+ "Operator assertion allowing dispatch of Lance index mutations on local file:// datasets "
+ + "(disabled by default). When enabled, dispatch still requires exactly one FE in the "
+ + "cluster and the target backend to be the only alive backend. Object storage is the "
+ + "production mode."})
+ public static boolean enable_lance_index_local_file_mutation = false;
+
}
diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java
index c9e0de341acd16..abbdaacf9a3791 100644
--- a/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java
+++ b/fe/fe-common/src/test/java/org/apache/doris/common/LanceIndexConfigValidatorTest.java
@@ -44,6 +44,16 @@ public void testReviewedDefaults() {
Assertions.assertEquals(256, Config.lance_index_max_num_sub_vectors);
}
+ @Test
+ public void testDispatcherReviewedDefaults() {
+ Assertions.assertEquals(10, Config.lance_index_job_dispatch_interval_second);
+ Assertions.assertEquals(3600L, Config.lance_index_job_execute_deadline_second);
+ Assertions.assertEquals(16, Config.lance_index_job_max_dispatch_per_round);
+ Assertions.assertEquals(2, Config.lance_index_job_max_inflight_per_backend);
+ Assertions.assertEquals(300, Config.lance_index_job_refresh_retry_second);
+ Assertions.assertFalse(Config.enable_lance_index_local_file_mutation);
+ }
+
@Test
public void testConfFieldWiring() throws Exception {
assertCallbackWiring("lance_index_job_max_unresolved_per_table", true,
@@ -65,6 +75,29 @@ public void testConfFieldWiring() throws Exception {
Assertions.assertEquals(VariableAnnotation.EXPERIMENTAL, gate.varType());
}
+ @Test
+ public void testDispatcherConfFieldWiring() throws Exception {
+ assertCallbackWiring("lance_index_job_dispatch_interval_second", true,
+ LanceIndexConfigValidator.PositiveIntConfigHandler.class);
+ assertCallbackWiring("lance_index_job_execute_deadline_second", true,
+ LanceIndexConfigValidator.PositiveLongConfigHandler.class);
+ assertCallbackWiring("lance_index_job_max_dispatch_per_round", true,
+ LanceIndexConfigValidator.PositiveIntConfigHandler.class);
+ assertCallbackWiring("lance_index_job_max_inflight_per_backend", true,
+ LanceIndexConfigValidator.PositiveIntConfigHandler.class);
+ assertCallbackWiring("lance_index_job_refresh_retry_second", true,
+ LanceIndexConfigValidator.PositiveIntConfigHandler.class);
+
+ // The local-file operator assertion is a plain mutable master-only boolean: no
+ // numeric validator is attached, so any boolean the ADMIN SET path accepts is legal.
+ ConfigBase.ConfField gate = Config.class.getField("enable_lance_index_local_file_mutation")
+ .getAnnotation(ConfigBase.ConfField.class);
+ Assertions.assertNotNull(gate);
+ Assertions.assertTrue(gate.mutable());
+ Assertions.assertTrue(gate.masterOnly());
+ Assertions.assertEquals(ConfigBase.DefaultConfHandler.class, gate.callback());
+ }
+
private static void assertCallbackWiring(String fieldName, boolean masterOnly, Class callback)
throws Exception {
ConfigBase.ConfField anno = Config.class.getField(fieldName).getAnnotation(ConfigBase.ConfField.class);
@@ -160,6 +193,38 @@ private static void assertIntRejected(String fieldName, String value) throws Exc
* End-to-end through the ADMIN SET FRONTEND CONFIG machinery: the annotation callback
* must both validate and assign, and a rejected value must leave the field untouched.
*/
+ @Test
+ public void testDispatcherPositiveHandlersAssignAcceptedValues() throws Exception {
+ assertIntAssigns("lance_index_job_dispatch_interval_second");
+ assertIntAssigns("lance_index_job_max_dispatch_per_round");
+ assertIntAssigns("lance_index_job_max_inflight_per_backend");
+ assertIntAssigns("lance_index_job_refresh_retry_second");
+ // The long handler must also be exercised on its one long dispatcher field.
+ Field deadlineField = Config.class.getField("lance_index_job_execute_deadline_second");
+ long originalDeadline = deadlineField.getLong(null);
+ try {
+ new LanceIndexConfigValidator.PositiveLongConfigHandler().handle(deadlineField, " 7200 ");
+ Assertions.assertEquals(7200L, deadlineField.getLong(null));
+ } finally {
+ deadlineField.setLong(null, originalDeadline);
+ }
+ }
+
+ @Test
+ public void testDispatcherPositiveHandlersRejectInvalidValues() throws Exception {
+ assertIntRejected("lance_index_job_dispatch_interval_second", "0");
+ assertIntRejected("lance_index_job_dispatch_interval_second", "-10");
+ assertIntRejected("lance_index_job_max_dispatch_per_round", "0");
+ assertIntRejected("lance_index_job_max_dispatch_per_round", "-1");
+ assertIntRejected("lance_index_job_max_inflight_per_backend", "0");
+ assertIntRejected("lance_index_job_max_inflight_per_backend", "-3");
+ assertIntRejected("lance_index_job_refresh_retry_second", "0");
+ assertIntRejected("lance_index_job_refresh_retry_second", "-300");
+ assertLongRejected("lance_index_job_execute_deadline_second", "0");
+ assertLongRejected("lance_index_job_execute_deadline_second", "-3600");
+ assertLongRejected("lance_index_job_execute_deadline_second", "soon");
+ }
+
@Test
public void testSetMutableConfigPath() throws Exception {
Config config = new Config();
@@ -170,6 +235,8 @@ public void testSetMutableConfigPath() throws Exception {
long originalQuota = Config.lance_index_job_max_unresolved_per_catalog;
int originalBound = Config.lance_index_max_num_partitions;
boolean originalGate = Config.enable_lance_index_mutation;
+ int originalInterval = Config.lance_index_job_dispatch_interval_second;
+ boolean originalLocalFile = Config.enable_lance_index_local_file_mutation;
try {
ConfigBase.setMutableConfig("lance_index_job_max_unresolved_per_catalog", "96");
Assertions.assertEquals(96L, Config.lance_index_job_max_unresolved_per_catalog);
@@ -187,10 +254,23 @@ public void testSetMutableConfigPath() throws Exception {
ConfigBase.setMutableConfig("enable_lance_index_mutation", "true");
Assertions.assertTrue(Config.enable_lance_index_mutation);
+
+ ConfigBase.setMutableConfig("lance_index_job_dispatch_interval_second", "60");
+ Assertions.assertEquals(60, Config.lance_index_job_dispatch_interval_second);
+ Assertions.assertThrows(ConfigException.class,
+ () -> ConfigBase.setMutableConfig("lance_index_job_dispatch_interval_second", "0"));
+ Assertions.assertEquals(60, Config.lance_index_job_dispatch_interval_second);
+
+ ConfigBase.setMutableConfig("enable_lance_index_local_file_mutation", "true");
+ Assertions.assertTrue(Config.enable_lance_index_local_file_mutation);
+ ConfigBase.setMutableConfig("enable_lance_index_local_file_mutation", "false");
+ Assertions.assertFalse(Config.enable_lance_index_local_file_mutation);
} finally {
Config.lance_index_job_max_unresolved_per_catalog = originalQuota;
Config.lance_index_max_num_partitions = originalBound;
Config.enable_lance_index_mutation = originalGate;
+ Config.lance_index_job_dispatch_interval_second = originalInterval;
+ Config.enable_lance_index_local_file_mutation = originalLocalFile;
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 43969050b50013..38af43a1c99d26 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -111,6 +111,7 @@
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.datasource.iceberg.IcebergSysExternalTable;
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
+import org.apache.doris.datasource.lance.job.LanceIndexJobDispatcher;
import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
import org.apache.doris.datasource.paimon.PaimonExternalTable;
import org.apache.doris.datasource.paimon.PaimonSysExternalTable;
@@ -573,6 +574,8 @@ public class Env {
private LanceIndexJobManager lanceIndexJobManager;
+ private LanceIndexJobDispatcher lanceIndexJobDispatcher;
+
private DNSCache dnsCache;
private final NereidsSqlCacheManager sqlCacheManager;
@@ -859,6 +862,7 @@ public Env(boolean isCheckpointCatalog) {
this.eventProcessor = new EventProcessor(mtmvService);
this.insertOverwriteManager = new InsertOverwriteManager();
this.lanceIndexJobManager = new LanceIndexJobManager();
+ this.lanceIndexJobDispatcher = new LanceIndexJobDispatcher(lanceIndexJobManager);
this.dnsCache = new DNSCache();
this.sqlCacheManager = new NereidsSqlCacheManager();
this.sortedPartitionsCacheManager = new NereidsSortedPartitionsCacheManager();
@@ -2030,6 +2034,8 @@ protected void startMasterOnlyDaemonThreads() {
keyManager.init();
}
agentTaskCleanupDaemon.start();
+ // lance index job dispatcher: dispatch sweep, deadline/possible-live sweeps, refresh driver
+ lanceIndexJobDispatcher.start();
}
// start threads that should run on all FE
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobDispatcher.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobDispatcher.java
new file mode 100644
index 00000000000000..f2319446f127be
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobDispatcher.java
@@ -0,0 +1,522 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ClientPool;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.lance.LanceExternalCatalog;
+import org.apache.doris.datasource.lance.LanceStorageOptions;
+import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.system.Backend;
+import org.apache.doris.system.BeSelectionPolicy;
+import org.apache.doris.system.SystemInfoService;
+import org.apache.doris.thrift.BackendService;
+import org.apache.doris.thrift.TLanceIndexJobDispatch;
+import org.apache.doris.thrift.TLanceIndexMutationType;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
+
+import com.google.common.collect.Maps;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Master-only daemon that drives the durable Lance index job records through
+ * the lifecycle after admission. Each round runs in a fixed order: converge
+ * expired RUNNING jobs to UNKNOWN, release possible-live slots whose backend
+ * process was replaced, drive the refresh a terminal job still owes, then
+ * dispatch PENDING jobs. Every durable transition goes through
+ * {@link LanceIndexJobManager} under its own lock; the daemon holds no catalog
+ * or manager lock across any call.
+ *
+ * The daemon does not read the admission gate: a job that is already durable
+ * must be driven to its terminal state, whatever the gate says now, so the
+ * thread runs unconditionally on the master and simply finds nothing to do
+ * while no jobs exist. An idle round writes no journal record.
+ *
+ *
Dispatch follows the durable-before-send boundary: the markRunning edit
+ * log is written and re-read before the first byte of network I/O, and the
+ * invocation id of an attempt that lost the compare-and-set is never reused.
+ * After a successful markRunning there is exactly one send; from that point a
+ * job converges only through a matching result callback, the deadline sweep,
+ * or the epoch sweep, never through a resend.
+ */
+public class LanceIndexJobDispatcher extends MasterDaemon {
+ private static final Logger LOG = LogManager.getLogger(LanceIndexJobDispatcher.class);
+
+ private final LanceIndexJobManager jobManager;
+
+ public LanceIndexJobDispatcher(LanceIndexJobManager jobManager) {
+ super("lance index job dispatcher", dispatchIntervalMs());
+ this.jobManager = jobManager;
+ }
+
+ /**
+ * Values loaded from fe.conf bypass the config validator (only ADMIN SET runs
+ * it), so the positive invariant is re-asserted where a non-positive value
+ * would break the loop: a non-positive interval would kill this thread inside
+ * {@code Thread.sleep} or busy-spin it, a non-positive deadline would sweep
+ * every dispatched job UNKNOWN on the next round, and a zero cap would stall
+ * dispatch forever. The refresh retry interval needs no such defense: a
+ * non-positive value simply disengages the throttle.
+ */
+ private static long dispatchIntervalMs() {
+ return Math.max(1, Config.lance_index_job_dispatch_interval_second) * 1000L;
+ }
+
+ private static long executeDeadlineMs(long nowMs) {
+ long second = Math.max(1L, Config.lance_index_job_execute_deadline_second);
+ return second > (Long.MAX_VALUE - nowMs) / 1000L ? Long.MAX_VALUE : nowMs + second * 1000L;
+ }
+
+ @Override
+ protected void runAfterCatalogReady() {
+ if (!Env.getCurrentEnv().isMaster()) {
+ return;
+ }
+ if (Env.isCheckpointThread()) {
+ return;
+ }
+ setInterval(dispatchIntervalMs());
+ try {
+ runOneRound();
+ } catch (Throwable t) {
+ LOG.warn("Failed to process one round of the lance index job dispatcher", t);
+ }
+ }
+
+ private void runOneRound() {
+ long nowMs = System.currentTimeMillis();
+ sweepExpiredRunningJobs(nowMs);
+ sweepReplacedProcessEpochs();
+ driveRequiredRefreshes(nowMs);
+ dispatchPendingJobs();
+ }
+
+ /**
+ * Deadline sweep. A RUNNING job past its wait deadline has produced no
+ * complete trusted result, so it converges to UNKNOWN through the same
+ * completeWithResult channel a callback would use. Expiry bounds the wait
+ * only: it never proves termination, so the possible-live slot, the
+ * same-name fence, and the unresolved quota all stay held.
+ */
+ private void sweepExpiredRunningJobs(long nowMs) {
+ for (LanceIndexJob job : jobManager.getExpiredRunningJobs(nowMs)) {
+ try {
+ boolean completed = jobManager.completeWithResult(job.getJobId(),
+ dispatchRevisionOf(job), job.getInvocationId(), job.getBeProcessEpoch(),
+ new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE,
+ "execute deadline expired without a complete trusted result", false));
+ if (completed) {
+ LOG.info("lance index job {} converged RUNNING -> UNKNOWN on deadline expiry",
+ job.getJobId());
+ } else {
+ LOG.warn("deadline sweep skipped lance index job {}: already converged by a callback or sweep",
+ job.getJobId());
+ }
+ } catch (Throwable t) {
+ LOG.warn("failed to sweep expired lance index job " + job.getJobId(), t);
+ }
+ }
+ }
+
+ /**
+ * Possible-live sweep. The only slot-release proof this daemon produces is
+ * that the recorded backend process epoch no longer exists: a backend entry
+ * reporting a different epoch proves the process that received the dispatch
+ * was replaced. A missing backend entry or heartbeat loss proves nothing
+ * (the worker may still be running behind a partition), so such a job keeps
+ * its slot until a stronger proof or an operator force release. An epoch
+ * change also proves nothing about the outcome, so the mutation state is
+ * never touched here.
+ */
+ private void sweepReplacedProcessEpochs() {
+ for (LanceIndexJob job : jobManager.getJobsHoldingPossibleLiveSlot()) {
+ try {
+ Backend backend = Env.getCurrentSystemInfo().getBackend(job.getBackendId());
+ if (backend == null || backend.getProcessEpoch() == job.getBeProcessEpoch()) {
+ continue;
+ }
+ boolean recorded = jobManager.recordTerminationProof(job.getJobId(),
+ dispatchRevisionOf(job), job.getBackendId(), job.getBeProcessEpoch(),
+ job.getInvocationId(), LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE);
+ if (recorded) {
+ LOG.info("released possible-live slot of lance index job {}: backend process epoch was replaced",
+ job.getJobId());
+ } else {
+ LOG.warn("epoch sweep skipped lance index job {}: dispatch identity already moved",
+ job.getJobId());
+ }
+ } catch (Throwable t) {
+ LOG.warn("failed to sweep possible-live slot of lance index job " + job.getJobId(), t);
+ }
+ }
+ }
+
+ /**
+ * Refresh driver for terminal jobs with an unfinished refresh obligation.
+ * Completing the refresh is the protocol duty that releases the same-name
+ * fence and the unresolved quota once DONE; it is not a read-visibility
+ * action, because index metadata is never cached. Each job is driven
+ * through markRefreshRunning, the idempotent external-table refresh, then
+ * DONE or FAILED: a FAILED job keeps its fence and is retried, throttled to
+ * one attempt per retry interval, while a first REQUIRED refresh is never
+ * delayed. UNKNOWN jobs never appear here; they owe no refresh.
+ */
+ private void driveRequiredRefreshes(long nowMs) {
+ for (LanceIndexJob job : jobManager.getJobsNeedingRefresh()) {
+ try {
+ if (job.getRefreshState() == LanceIndexJobRefreshState.RUNNING) {
+ // In flight elsewhere; the master-transfer sweep downgrades a stale
+ // RUNNING back to REQUIRED, so a lost driver cannot strand it.
+ continue;
+ }
+ if (job.getRefreshState() == LanceIndexJobRefreshState.FAILED
+ && nowMs - job.getUpdateTimeMs()
+ < Config.lance_index_job_refresh_retry_second * 1000L) {
+ continue;
+ }
+ if (!jobManager.markRefreshRunning(job.getJobId(), job.getRevision())) {
+ // A concurrent driver won the compare-and-set; nothing to do here.
+ continue;
+ }
+ driveOneRefresh(job);
+ } catch (Throwable t) {
+ LOG.warn("failed to drive the refresh of lance index job " + job.getJobId(), t);
+ }
+ }
+ }
+
+ private void driveOneRefresh(LanceIndexJob job) {
+ long refreshRevision = job.getRevision() + 1;
+ CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(job.getCatalogId());
+ if (catalog == null) {
+ // Unreachable while the unresolved-job guard blocks catalog drops; kept as a
+ // fail-closed fallback so the job still transitions and retries later.
+ LOG.warn("catalog of lance index job {} is gone; marking its refresh FAILED", job.getJobId());
+ finishRefreshTransition(job.getJobId(), refreshRevision, false);
+ return;
+ }
+ try {
+ // A half-orphan target (its db or table already dropped externally) is a
+ // silent no-op: nothing is left to invalidate, and DONE is the correct end
+ // state for the job.
+ Env.getCurrentEnv().getRefreshManager().handleRefreshTable(catalog.getName(),
+ job.getDbName(), job.getTableName(), true);
+ } catch (Throwable t) {
+ // The typed DdlException is the expected failure; an unchecked exception out
+ // of the metadata path must still leave the durable refresh state, or the
+ // job would strand in refresh RUNNING until the next master transfer.
+ LOG.warn("refresh of lance index job {} failed; keeping the fence for a retry",
+ job.getJobId(), t);
+ finishRefreshTransition(job.getJobId(), refreshRevision, false);
+ return;
+ }
+ finishRefreshTransition(job.getJobId(), refreshRevision, true);
+ }
+
+ /**
+ * Applies the DONE/FAILED transition with a bounded revision retry. A concurrent
+ * termination-proof write can bump the revision after markRefreshRunning succeeded,
+ * and silently losing that compare-and-set would leave the refresh RUNNING — a
+ * state only the master-transfer sweep downgrades. Re-reading the revision and
+ * retrying a few times converges it; a persistent loss is escalated.
+ */
+ private void finishRefreshTransition(long jobId, long expectedRevision, boolean done) {
+ long revision = expectedRevision;
+ for (int attempt = 0; attempt < 3; attempt++) {
+ boolean transitioned = done ? jobManager.markRefreshDone(jobId, revision)
+ : jobManager.markRefreshFailed(jobId, revision);
+ if (transitioned) {
+ return;
+ }
+ LanceIndexJob fresh = jobManager.getJob(jobId);
+ if (fresh == null) {
+ break;
+ }
+ revision = fresh.getRevision();
+ }
+ LOG.error("lance index job {} kept its refresh RUNNING: the DONE/FAILED transition kept losing the"
+ + " compare-and-set; the master-transfer sweep will downgrade it", jobId);
+ }
+
+ /**
+ * PENDING dispatch. Attempts at most
+ * {@link Config#lance_index_job_max_dispatch_per_round} fresh dispatches per
+ * round, and never more than {@link Config#lance_index_job_max_inflight_per_backend}
+ * in-flight jobs per backend, counted from the RUNNING snapshot plus the
+ * jobs this round already made RUNNING. A job that cannot be dispatched
+ * keeps waiting as PENDING: there is no dispatch-exhaustion terminal state
+ * and no backoff beyond the daemon period.
+ */
+ private void dispatchPendingJobs() {
+ int maxPerRound = Math.max(1, Config.lance_index_job_max_dispatch_per_round);
+ Map inflightByBackend = countInflightByBackend();
+ int attempted = 0;
+ for (LanceIndexJob job : jobManager.getJobsNeedingDispatch(maxPerRound)) {
+ if (++attempted > maxPerRound) {
+ break;
+ }
+ try {
+ tryDispatch(job, inflightByBackend);
+ } catch (Throwable t) {
+ LOG.warn("failed to dispatch lance index job " + job.getJobId(), t);
+ }
+ }
+ }
+
+ private Map countInflightByBackend() {
+ Map inflightByBackend = Maps.newHashMap();
+ for (LanceIndexJob job : jobManager.getAllJobsSnapshot()) {
+ if (job.getMutationState() == LanceIndexJobMutationState.RUNNING && job.getBackendId() != null) {
+ inflightByBackend.merge(job.getBackendId(), 1, Integer::sum);
+ }
+ }
+ return inflightByBackend;
+ }
+
+ /**
+ * One dispatch attempt for one PENDING job. Every early return before
+ * markRunning leaves the job PENDING for a later round. Once markRunning
+ * succeeds the job is durable RUNNING and this invocation id gets exactly
+ * one send attempt; after that only a matching callback, the deadline
+ * sweep, or the epoch sweep can converge the job.
+ */
+ private void tryDispatch(LanceIndexJob job, Map inflightByBackend) {
+ boolean localDataset = isLocalFileDataset(job.getNormalizedLocator());
+ if (localDataset && !Config.enable_lance_index_local_file_mutation) {
+ // Operator assertion is off: a local-filesystem mutation stays PENDING.
+ return;
+ }
+ if (localDataset && Env.getCurrentEnv().getFrontends(null).size() != 1) {
+ // Local files are only shared by a single-node deployment.
+ return;
+ }
+ SystemInfoService systemInfo = Env.getCurrentSystemInfo();
+ List backendIds = systemInfo.selectBackendIdsByPolicy(
+ new BeSelectionPolicy.Builder().needScheduleAvailable().build(), 1);
+ if (backendIds.isEmpty()) {
+ return;
+ }
+ Backend backend = systemInfo.getBackend(backendIds.get(0));
+ if (backend == null) {
+ return;
+ }
+ if (localDataset && !isOnlyAliveBackend(systemInfo, backend.getId())) {
+ return;
+ }
+ Integer inflight = inflightByBackend.get(backend.getId());
+ if (inflight != null && inflight >= Math.max(1, Config.lance_index_job_max_inflight_per_backend)) {
+ return;
+ }
+ String invocationId = UUID.randomUUID().toString();
+ // The process epoch is captured once, and the same value goes to the
+ // durable record and the wire: a heartbeat landing between the two reads
+ // must not split the dispatch identity (the callback matches the durable
+ // value, and the epoch sweep releases the slot against it).
+ long beProcessEpoch = backend.getProcessEpoch();
+ long deadlineMs = executeDeadlineMs(System.currentTimeMillis());
+ if (!jobManager.markRunning(job.getJobId(), job.getRevision(), backend.getId(),
+ beProcessEpoch, invocationId, deadlineMs)) {
+ // The compare-and-set lost: this attempt's dispatch identity is void and its
+ // invocation id is discarded. A fresh identity is built from scratch next round.
+ return;
+ }
+ inflightByBackend.merge(backend.getId(), 1, Integer::sum);
+ long expectedDispatchRevision = job.getRevision() + 1;
+ LanceIndexJob fresh = jobManager.getJob(job.getJobId());
+ if (!Env.getCurrentEnv().isMaster() || fresh == null
+ || fresh.getMutationState() != LanceIndexJobMutationState.RUNNING
+ || fresh.getDispatchRevision() == null
+ || fresh.getDispatchRevision() != expectedDispatchRevision
+ || !invocationId.equals(fresh.getInvocationId())) {
+ // The recheck failed right before the send: no send, and no resend either.
+ // The job is durable RUNNING, so the deadline sweep or a matching callback
+ // converges it.
+ LOG.warn("lance index job {} did not survive the pre-send recheck; not sending", job.getJobId());
+ return;
+ }
+ TLanceIndexJobDispatch dispatch;
+ try {
+ dispatch = buildDispatch(fresh, invocationId, deadlineMs, beProcessEpoch,
+ resolveStorageOptions(fresh));
+ } catch (Exception e) {
+ // An FE-side resolution failure is not a trusted worker rejection, so it must
+ // not fabricate NOT_COMMITTED. The job is already RUNNING without a send, and
+ // the send may never happen, so converge it to UNKNOWN fail-closed.
+ LOG.warn("failed to prepare the dispatch of lance index job {}: {}", job.getJobId(), e.getMessage());
+ completeNoTrusted(fresh, "dispatch preparation failed before send");
+ return;
+ }
+ TStatus status;
+ try {
+ status = sendExecuteRequest(backend, dispatch);
+ } catch (Exception e) {
+ // The request may have reached the backend, so its outcome cannot be trusted.
+ LOG.warn("dispatch send of lance index job {} failed: {}", job.getJobId(), e.getMessage());
+ completeNoTrusted(fresh, "dispatch send failed; the result cannot be trusted");
+ return;
+ }
+ if (status == null || status.getStatusCode() == null) {
+ // Absence of a status is the absence of a trusted answer, not a clean
+ // rejection; only a complete error status proves the dispatch was not
+ // enqueued.
+ LOG.warn("dispatch send of lance index job {} returned no status", job.getJobId());
+ completeNoTrusted(fresh, "dispatch send returned no status");
+ return;
+ }
+ if (status.getStatusCode() != TStatusCode.OK) {
+ // A clean error status proves the backend did not enqueue the dispatch, so
+ // this invocation is known never to have executed.
+ LOG.warn("backend {} rejected the dispatch of lance index job {} before enqueueing",
+ backend.getId(), job.getJobId());
+ completePreInvocationRejected(fresh);
+ }
+ // OK: enqueued exactly once. The result arrives through the report callback;
+ // nothing more is done here, and the deadline sweep bounds the wait.
+ }
+
+ /**
+ * Sends one dispatch to the backend's thrift service and returns its status.
+ * The connection is borrowed per send, returned only when the call completed,
+ * and invalidated after a failed call. Test seam: subclasses override this
+ * method to record the request or inject faults without a live client pool.
+ */
+ protected TStatus sendExecuteRequest(Backend backend, TLanceIndexJobDispatch dispatch) throws Exception {
+ TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBePort());
+ BackendService.Client client = null;
+ boolean callCompleted = false;
+ try {
+ client = ClientPool.backendPool.borrowObject(address);
+ TStatus status = client.submitLanceIndexJob(dispatch);
+ callCompleted = true;
+ return status;
+ } finally {
+ if (client != null) {
+ if (callCompleted) {
+ ClientPool.backendPool.returnObject(address, client);
+ } else {
+ ClientPool.backendPool.invalidateObject(address, client);
+ }
+ }
+ }
+ }
+
+ /**
+ * Builds the wire request from the durable record. Definition fields a DROP
+ * never carries travel as the empty string: the wire marks them required,
+ * and the worker only reads them for CREATE and REPLACE.
+ */
+ private TLanceIndexJobDispatch buildDispatch(LanceIndexJob job, String invocationId, long deadlineMs,
+ long beProcessEpoch, Map storageOptions) {
+ TLanceIndexJobDispatch dispatch = new TLanceIndexJobDispatch();
+ dispatch.setJobId(job.getJobId());
+ dispatch.setDispatchRevision(job.getDispatchRevision());
+ dispatch.setInvocationId(invocationId);
+ dispatch.setBeProcessEpoch(beProcessEpoch);
+ dispatch.setDeadlineMs(deadlineMs);
+ dispatch.setMutationType(TLanceIndexMutationType.valueOf(job.getMutationType().name()));
+ dispatch.setIndexName(job.getDisplayIndexName());
+ dispatch.setColumnName(job.getColumnName() == null ? "" : job.getColumnName());
+ dispatch.setIndexType(job.getIndexType() == null ? "" : job.getIndexType());
+ if (job.getPropertiesJson() != null) {
+ dispatch.setPropertiesJson(job.getPropertiesJson());
+ }
+ dispatch.setIfNotExists(job.isIfNotExists());
+ dispatch.setIfExists(job.isIfExists());
+ dispatch.setDatasetUri(job.getNormalizedLocator());
+ dispatch.setAdmittedDatasetVersion(job.getAdmittedDatasetVersion());
+ dispatch.setSchemaContractJson(job.getSchemaContract() == null ? ""
+ : GsonUtils.GSON.toJson(job.getSchemaContract()));
+ if (!storageOptions.isEmpty()) {
+ dispatch.setStorageOptions(storageOptions);
+ }
+ return dispatch;
+ }
+
+ /**
+ * Resolves the storage options of one dataset at send time from the
+ * catalog's current storage properties, in the vocabulary of the provider
+ * the dataset URI routes to. The result is used for this dispatch only: it
+ * is never persisted in the job record and never logged, so a rotated
+ * credential takes effect on the next dispatch without any journal record.
+ */
+ private Map resolveStorageOptions(LanceIndexJob job) {
+ CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(job.getCatalogId());
+ if (!(catalog instanceof LanceExternalCatalog)) {
+ throw new IllegalStateException(
+ "catalog of lance index job " + job.getJobId() + " does not resolve to a Lance catalog");
+ }
+ return LanceStorageOptions.fromDorisStorageProperties(job.getNormalizedLocator(),
+ ((LanceExternalCatalog) catalog).getCatalogProperty().getOrderedStoragePropertiesList());
+ }
+
+ private void completeNoTrusted(LanceIndexJob job, String reason) {
+ boolean completed = jobManager.completeWithResult(job.getJobId(),
+ dispatchRevisionOf(job), job.getInvocationId(), job.getBeProcessEpoch(),
+ new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE, reason, false));
+ if (!completed) {
+ LOG.warn("no-trusted-result convergence skipped for lance index job {}: already converged by a callback"
+ + " or sweep", job.getJobId());
+ }
+ }
+
+ private void completePreInvocationRejected(LanceIndexJob job) {
+ boolean completed = jobManager.completeWithResult(job.getJobId(),
+ dispatchRevisionOf(job), job.getInvocationId(), job.getBeProcessEpoch(),
+ new LanceIndexJobResult(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED,
+ LanceIndexJobCompletionReason.NONE,
+ "backend returned a clean error status before enqueueing the dispatch", false));
+ if (!completed) {
+ LOG.warn("rejection convergence skipped for lance index job {}: already converged by a callback or sweep",
+ job.getJobId());
+ }
+ }
+
+ /**
+ * True for datasets on the local filesystem: a scheme-less absolute path or
+ * a {@code file://} URI, matching the provider routing of the dataset URL.
+ */
+ private static boolean isLocalFileDataset(String normalizedLocator) {
+ int separator = normalizedLocator.indexOf("://");
+ if (separator < 0) {
+ return true;
+ }
+ return "file".equals(normalizedLocator.substring(0, separator).toLowerCase(Locale.ROOT));
+ }
+
+ private static boolean isOnlyAliveBackend(SystemInfoService systemInfo, long backendId) {
+ List aliveBackendIds = systemInfo.getAllBackendIds(true);
+ return aliveBackendIds.size() == 1 && aliveBackendIds.get(0) == backendId;
+ }
+
+ private static long dispatchRevisionOf(LanceIndexJob job) {
+ return job.getDispatchRevision() == null ? job.getRevision() : job.getDispatchRevision();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
index a34c9c1a087018..100096066c9689 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
@@ -596,6 +596,32 @@ private static boolean hasFenceIdentity(LanceIndexJob job) {
&& job.getNormalizedIndexName() != null;
}
+ /**
+ * True when the durable record carries the complete target identity the
+ * dispatcher needs to address a job: catalog, database, table, normalized
+ * locator, and the index/mutation identity. This is the eligibility of a
+ * PENDING job for dispatch; the dispatch quad (backend id, BE process
+ * epoch, dispatch revision, invocation id) does not exist before
+ * {@code markRunning} and is not required here. Corrupt identity-less
+ * records are never dispatchable.
+ */
+ private static boolean hasDispatchTarget(LanceIndexJob job) {
+ return job.getProvider() != null && job.getNormalizedLocator() != null
+ && job.getNormalizedIndexName() != null && job.getDisplayIndexName() != null
+ && job.getDbName() != null && job.getTableName() != null
+ && job.getMutationType() != null && job.getCatalogId() > 0;
+ }
+
+ /**
+ * True when the record additionally carries the full dispatch quad recorded
+ * by {@code markRunning}. The possible-live sweep requires the quad: only
+ * with it can {@code recordTerminationProof} address and release the slot.
+ */
+ private static boolean hasDispatchIdentity(LanceIndexJob job) {
+ return hasDispatchTarget(job) && job.getBackendId() != null && job.getBeProcessEpoch() != null
+ && job.getDispatchRevision() != null && job.getInvocationId() != null;
+ }
+
// ------------------------------------------------------------------
// Queries
// ------------------------------------------------------------------
@@ -673,7 +699,8 @@ public List getUnresolvedJobs() {
* run, including jobs downgraded by the master-transfer sweep) or FAILED
* (waiting for a retry). Both resume through the idempotent refresh path via
* {@link #markRefreshRunning}; a FAILED job invisible here would hold its
- * fence forever with no retry channel.
+ * fence forever with no retry channel. Force-released UNKNOWN jobs are excluded:
+ * their fence is already released and re-driving refresh would only add audit noise.
*/
public List getJobsNeedingRefresh() {
readLock();
@@ -682,6 +709,7 @@ public List getJobsNeedingRefresh() {
for (LanceIndexJob job : jobs.values()) {
if (job != null && job.getMutationState() != null && job.getMutationState().isTerminal()
&& hasFenceIdentity(job)
+ && !job.isForceReleased()
&& (job.getRefreshState() == LanceIndexJobRefreshState.REQUIRED
|| job.getRefreshState() == LanceIndexJobRefreshState.FAILED)) {
result.add(new LanceIndexJob(job));
@@ -693,6 +721,75 @@ && hasFenceIdentity(job)
}
}
+ /**
+ * PENDING jobs eligible for the dispatcher, in job id order (FIFO fairness),
+ * at most {@code limit} of them. Target identity must be complete: corrupt
+ * identity-less records are never dispatchable (their only exit is the
+ * force-release transition by job id) and are skipped here. The dispatch
+ * quad is deliberately not required: it is written by {@code markRunning},
+ * which is the step this query feeds. All matches are collected and ordered
+ * before truncating, so a stable subset of permanently undispatchable jobs
+ * can never crowd out later ids.
+ */
+ public List getJobsNeedingDispatch(int limit) {
+ readLock();
+ try {
+ List result = new ArrayList<>();
+ for (LanceIndexJob job : jobs.values()) {
+ if (job != null && job.getMutationState() == LanceIndexJobMutationState.PENDING
+ && hasDispatchTarget(job)) {
+ result.add(new LanceIndexJob(job));
+ }
+ }
+ result.sort(Comparator.comparingLong(LanceIndexJob::getJobId));
+ return result.size() <= limit ? result : new ArrayList<>(result.subList(0, limit));
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * RUNNING jobs whose wait deadline has expired. Expiry converges the job to
+ * UNKNOWN via completeWithResult(NO_TRUSTED_RESULT); it never proves
+ * termination and never releases a possible-live slot.
+ */
+ public List getExpiredRunningJobs(long nowMs) {
+ readLock();
+ try {
+ List result = new ArrayList<>();
+ for (LanceIndexJob job : jobs.values()) {
+ if (job != null && job.getMutationState() == LanceIndexJobMutationState.RUNNING
+ && job.getDeadlineMs() != null && job.getDeadlineMs() < nowMs) {
+ result.add(new LanceIndexJob(job));
+ }
+ }
+ return result;
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * Jobs still holding a possible-live slot with complete dispatch identity,
+ * regardless of mutation state: the slot-release proof (BE process epoch no
+ * longer exists) is independent of the outcome, so UNKNOWN jobs swept by the
+ * deadline or master transfer are released here exactly like RUNNING ones.
+ */
+ public List getJobsHoldingPossibleLiveSlot() {
+ readLock();
+ try {
+ List result = new ArrayList<>();
+ for (LanceIndexJob job : jobs.values()) {
+ if (job != null && job.holdsPossibleLiveSlot() && hasDispatchIdentity(job)) {
+ result.add(new LanceIndexJob(job));
+ }
+ }
+ return result;
+ } finally {
+ readUnlock();
+ }
+ }
+
public boolean isFenceHeld(LanceIndexFenceKey fenceKey) {
readLock();
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobReportHandler.java
new file mode 100644
index 00000000000000..bc3519784f92fa
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobReportHandler.java
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.thrift.TLanceIndexJobReport;
+import org.apache.doris.thrift.TLanceIndexTerminationProof;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Objects;
+
+/**
+ * Applies one typed result envelope reported by a backend to the durable job
+ * record. This is a thin shim over the manager transitions: dispatch-identity
+ * checking and result classification all live in {@link LanceIndexJobManager},
+ * so a stale or identity-mismatched report only logs a warning and changes
+ * nothing. A malformed envelope (missing result code, a code this FE does not
+ * know, or a sanitized message past the durable bound) is dropped rather than
+ * trusted; the job then converges through the dispatcher's deadline sweep.
+ * Only the typed codes are read: message text is never inspected to infer an
+ * outcome.
+ *
+ * The handler runs on the report RPC thread and performs no I/O beyond the
+ * manager's own edit-log write. It starts no refresh: the metadata refresh a
+ * completed job may owe is driven by the dispatcher daemon, not here.
+ */
+public class LanceIndexJobReportHandler {
+
+ private static final Logger LOG = LogManager.getLogger(LanceIndexJobReportHandler.class);
+
+ private final LanceIndexJobManager jobManager;
+
+ public LanceIndexJobReportHandler(LanceIndexJobManager jobManager) {
+ this.jobManager = Objects.requireNonNull(jobManager, "jobManager");
+ }
+
+ /**
+ * Handles one report: a matched report completes the job with its
+ * classified result, and a CHILD_REAPED termination proof additionally
+ * releases the possible-live slot, because reaping the exact child process
+ * proves that process ended (which still says nothing about the outcome).
+ */
+ public void handle(TLanceIndexJobReport report) {
+ if (report == null) {
+ LOG.warn("dropping null lance index job report");
+ return;
+ }
+ LanceIndexJobResult result;
+ try {
+ result = toResult(report);
+ } catch (IllegalArgumentException e) {
+ LOG.warn("dropping malformed lance index job report for job {}: {}", report.getJobId(), e.getMessage());
+ return;
+ }
+ boolean completed = jobManager.completeWithResult(report.getJobId(), report.getDispatchRevision(),
+ report.getInvocationId(), report.getBeProcessEpoch(), result);
+ if (!completed) {
+ LOG.warn("dropping stale lance index job report for job {}", report.getJobId());
+ }
+ if (report.getTerminationProof() == TLanceIndexTerminationProof.CHILD_REAPED) {
+ recordChildReaped(report);
+ }
+ }
+
+ /**
+ * Releases the possible-live slot on a CHILD_REAPED proof. The report
+ * carries the invocation identity but not the backend id, so the durable
+ * record is its source; the quad match inside recordTerminationProof still
+ * rejects anything stale.
+ */
+ private void recordChildReaped(TLanceIndexJobReport report) {
+ LanceIndexJob job = jobManager.getJob(report.getJobId());
+ if (job == null || job.getBackendId() == null) {
+ LOG.warn("dropping CHILD_REAPED proof of lance index job {}: no durable dispatch identity",
+ report.getJobId());
+ return;
+ }
+ boolean recorded = jobManager.recordTerminationProof(report.getJobId(), report.getDispatchRevision(),
+ job.getBackendId(), report.getBeProcessEpoch(), report.getInvocationId(),
+ LanceIndexTerminationProof.CHILD_REAPED);
+ if (!recorded) {
+ LOG.warn("dropping stale CHILD_REAPED proof of lance index job {}", report.getJobId());
+ }
+ }
+
+ /**
+ * Converts the wire envelope to the durable result value, rejecting
+ * anything that cannot be represented: a missing result code, a code this
+ * FE does not know, or a sanitized message past the durable bound.
+ * NO_TRUSTED_RESULT is FE-side only and absent from the wire enum, so it
+ * can never arrive here.
+ */
+ private static LanceIndexJobResult toResult(TLanceIndexJobReport report) {
+ if (report.getResultCode() == null) {
+ throw new IllegalArgumentException("report carries no result code");
+ }
+ LanceIndexJobResultCode resultCode;
+ try {
+ resultCode = LanceIndexJobResultCode.valueOf(report.getResultCode().name());
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("report carries an unknown result code " + report.getResultCode());
+ }
+ LanceIndexJobCompletionReason completionReason = LanceIndexJobCompletionReason.NONE;
+ if (report.isSetCompletionReason() && report.getCompletionReason() != null) {
+ completionReason = LanceIndexJobCompletionReason.valueOf(report.getCompletionReason().name());
+ }
+ boolean externalMetadataAdvanced =
+ report.isSetExternalMetadataAdvanced() && report.isExternalMetadataAdvanced();
+ // Throws IllegalArgumentException when the message is past the durable bound.
+ return new LanceIndexJobResult(resultCode, completionReason,
+ report.getSanitizedMessage(), externalMetadataAdvanced);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 54101ad5640031..77459d882a65a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -94,6 +94,7 @@
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.SplitSource;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.datasource.lance.job.LanceIndexJobReportHandler;
import org.apache.doris.datasource.maxcompute.MCTransaction;
import org.apache.doris.encryption.EncryptionKey;
import org.apache.doris.ha.FrontendNodeType;
@@ -232,6 +233,7 @@
import org.apache.doris.thrift.TInitExternalCtlMetaRequest;
import org.apache.doris.thrift.TInitExternalCtlMetaResult;
import org.apache.doris.thrift.TInvalidateFollowerStatsCacheRequest;
+import org.apache.doris.thrift.TLanceIndexJobReport;
import org.apache.doris.thrift.TListPrivilegesResult;
import org.apache.doris.thrift.TListTableMetadataNameIdsResult;
import org.apache.doris.thrift.TListTableStatusResult;
@@ -1139,6 +1141,23 @@ public TMasterResult finishTask(TFinishTaskRequest request) throws TException {
return masterImpl.finishTask(request);
}
+ /**
+ * Typed result envelope of one Lance index mutation invocation. Stale or
+ * identity-mismatched reports are logged and dropped; a complete matched
+ * report is classified into the durable job state. This layer stays thin:
+ * only the master accepts reports, and everything beyond identity checking
+ * and classification lives in the report handler.
+ */
+ @Override
+ public TStatus reportLanceIndexJobResult(TLanceIndexJobReport report) throws TException {
+ TStatus status = checkMaster();
+ if (status.getStatusCode() != TStatusCode.OK) {
+ return status;
+ }
+ new LanceIndexJobReportHandler(Env.getCurrentEnv().getLanceIndexJobManager()).handle(report);
+ return new TStatus(TStatusCode.OK);
+ }
+
@Override
public TMasterResult report(TReportRequest request) throws TException {
TStatus status = checkMaster();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
index f02dc7f6d7b8ce..8db9cf665e653b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java
@@ -32,6 +32,7 @@
import org.apache.doris.thrift.TGetTopNHotPartitionsResponse;
import org.apache.doris.thrift.TIngestBinlogRequest;
import org.apache.doris.thrift.TIngestBinlogResult;
+import org.apache.doris.thrift.TLanceIndexJobDispatch;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPublishTopicRequest;
import org.apache.doris.thrift.TPublishTopicResult;
@@ -120,6 +121,11 @@ public InternalProcessor() {
//
}
+ @Override
+ public TStatus submitLanceIndexJob(TLanceIndexJobDispatch dispatch) throws TException {
+ return null;
+ }
+
@Override
public TAgentResult submitTasks(List tasks) throws TException {
return null;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java
index 331872d1552a9a..4d63d707d88eef 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/LanceThriftContractTest.java
@@ -20,11 +20,19 @@
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileScanRangeParams;
import org.apache.doris.thrift.TLanceFileDesc;
+import org.apache.doris.thrift.TLanceIndexCompletionReason;
+import org.apache.doris.thrift.TLanceIndexJobDispatch;
+import org.apache.doris.thrift.TLanceIndexJobReport;
+import org.apache.doris.thrift.TLanceIndexJobResultCode;
+import org.apache.doris.thrift.TLanceIndexMutationType;
+import org.apache.doris.thrift.TLanceIndexTerminationProof;
import org.apache.doris.thrift.TLanceScanParams;
import org.apache.doris.thrift.TTableFormatFileDesc;
import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TFieldIdEnum;
import org.apache.thrift.TSerializer;
+import org.apache.thrift.meta_data.FieldMetaData;
import org.apache.thrift.protocol.TCompactProtocol;
import org.junit.Assert;
import org.junit.Test;
@@ -124,4 +132,291 @@ public void testLanceStorageOptionsAreOptional() throws Exception {
// A local dataset needs no storage configuration at all.
Assert.assertFalse(restored.isSetLanceScanParams());
}
+
+ @Test
+ public void testLanceIndexJobDispatchCompactRoundTrip() throws Exception {
+ TLanceIndexJobDispatch source = new TLanceIndexJobDispatch()
+ .setJobId(11L)
+ .setDispatchRevision(3L)
+ .setInvocationId("0f1e2d3c-dispatch")
+ .setBeProcessEpoch(55L)
+ .setDeadlineMs(1760000000000L)
+ .setMutationType(TLanceIndexMutationType.REPLACE)
+ .setIndexName("OrdersIdx")
+ .setColumnName("vec")
+ .setIndexType("IVF_PQ")
+ .setPropertiesJson("{\"num_partitions\":4096}")
+ .setIfNotExists(false)
+ .setIfExists(true)
+ .setDatasetUri("s3://warehouse/db/table.lance")
+ .setAdmittedDatasetVersion(42L)
+ .setSchemaContractJson("{\"version\":1}")
+ .setStorageOptions(lanceStorageOptions());
+
+ TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
+ byte[] bytes = serializer.serialize(source);
+
+ TLanceIndexJobDispatch restored = new TLanceIndexJobDispatch();
+ new TDeserializer(new TCompactProtocol.Factory()).deserialize(restored, bytes);
+
+ Assert.assertEquals(11L, restored.getJobId());
+ Assert.assertEquals(3L, restored.getDispatchRevision());
+ Assert.assertEquals("0f1e2d3c-dispatch", restored.getInvocationId());
+ Assert.assertEquals(55L, restored.getBeProcessEpoch());
+ Assert.assertEquals(1760000000000L, restored.getDeadlineMs());
+ Assert.assertEquals(TLanceIndexMutationType.REPLACE, restored.getMutationType());
+ Assert.assertEquals("OrdersIdx", restored.getIndexName());
+ Assert.assertEquals("vec", restored.getColumnName());
+ Assert.assertEquals("IVF_PQ", restored.getIndexType());
+ Assert.assertEquals("{\"num_partitions\":4096}", restored.getPropertiesJson());
+ // Optional booleans must distinguish both states, not just "true".
+ Assert.assertTrue(restored.isSetIfNotExists());
+ Assert.assertFalse(restored.isIfNotExists());
+ Assert.assertTrue(restored.isSetIfExists());
+ Assert.assertTrue(restored.isIfExists());
+ Assert.assertEquals("s3://warehouse/db/table.lance", restored.getDatasetUri());
+ Assert.assertEquals(42L, restored.getAdmittedDatasetVersion());
+ Assert.assertEquals("{\"version\":1}", restored.getSchemaContractJson());
+ Assert.assertTrue(restored.isSetStorageOptions());
+ Assert.assertEquals(lanceStorageOptions(), restored.getStorageOptions());
+ }
+
+ @Test
+ public void testLanceIndexJobDispatchLeavesOptionalFieldsUnset() throws Exception {
+ TLanceIndexJobDispatch source = new TLanceIndexJobDispatch()
+ .setJobId(1L)
+ .setDispatchRevision(1L)
+ .setInvocationId("inv")
+ .setBeProcessEpoch(1L)
+ .setDeadlineMs(1L)
+ .setMutationType(TLanceIndexMutationType.CREATE)
+ .setIndexName("idx")
+ .setColumnName("v")
+ .setIndexType("IVF_PQ")
+ .setDatasetUri("file:///data/ds")
+ .setAdmittedDatasetVersion(1L)
+ .setSchemaContractJson("{}");
+
+ TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
+ byte[] bytes = serializer.serialize(source);
+
+ TLanceIndexJobDispatch restored = new TLanceIndexJobDispatch();
+ new TDeserializer(new TCompactProtocol.Factory()).deserialize(restored, bytes);
+
+ // A dispatch without properties, IF flags, or credentials must round-trip with those
+ // fields unset: the worker treats each absence as its own meaning, and a local
+ // dataset carries no storage options at all.
+ Assert.assertFalse(restored.isSetPropertiesJson());
+ Assert.assertFalse(restored.isSetIfNotExists());
+ Assert.assertFalse(restored.isSetIfExists());
+ Assert.assertFalse(restored.isSetStorageOptions());
+ Assert.assertEquals(TLanceIndexMutationType.CREATE, restored.getMutationType());
+ Assert.assertEquals("file:///data/ds", restored.getDatasetUri());
+ }
+
+ @Test
+ public void testLanceIndexJobReportCompactRoundTrip() throws Exception {
+ TLanceIndexJobReport source = new TLanceIndexJobReport()
+ .setJobId(9L)
+ .setDispatchRevision(4L)
+ .setInvocationId("0f1e2d3c-report")
+ .setBeProcessEpoch(66L)
+ .setResultCode(TLanceIndexJobResultCode.NATIVE_NOT_FOUND)
+ .setCompletionReason(TLanceIndexCompletionReason.IF_CONDITION_NOOP)
+ .setSanitizedMessage("index absent on the provider")
+ .setExternalMetadataAdvanced(true)
+ .setTerminationProof(TLanceIndexTerminationProof.CHILD_REAPED);
+
+ TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
+ byte[] bytes = serializer.serialize(source);
+
+ TLanceIndexJobReport restored = new TLanceIndexJobReport();
+ new TDeserializer(new TCompactProtocol.Factory()).deserialize(restored, bytes);
+
+ Assert.assertEquals(9L, restored.getJobId());
+ Assert.assertEquals(4L, restored.getDispatchRevision());
+ Assert.assertEquals("0f1e2d3c-report", restored.getInvocationId());
+ Assert.assertEquals(66L, restored.getBeProcessEpoch());
+ Assert.assertEquals(TLanceIndexJobResultCode.NATIVE_NOT_FOUND, restored.getResultCode());
+ Assert.assertEquals(TLanceIndexCompletionReason.IF_CONDITION_NOOP, restored.getCompletionReason());
+ Assert.assertEquals("index absent on the provider", restored.getSanitizedMessage());
+ Assert.assertTrue(restored.isSetExternalMetadataAdvanced());
+ Assert.assertTrue(restored.isExternalMetadataAdvanced());
+ Assert.assertEquals(TLanceIndexTerminationProof.CHILD_REAPED, restored.getTerminationProof());
+ }
+
+ @Test
+ public void testLanceIndexJobReportLeavesOptionalFieldsUnset() throws Exception {
+ TLanceIndexJobReport source = new TLanceIndexJobReport()
+ .setJobId(9L)
+ .setDispatchRevision(4L)
+ .setInvocationId("inv")
+ .setBeProcessEpoch(66L)
+ .setResultCode(TLanceIndexJobResultCode.NATIVE_OK);
+
+ TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
+ byte[] bytes = serializer.serialize(source);
+
+ TLanceIndexJobReport restored = new TLanceIndexJobReport();
+ new TDeserializer(new TCompactProtocol.Factory()).deserialize(restored, bytes);
+
+ // The minimal honest result: a code and nothing else. Each absent optional field is
+ // its own meaning (NONE reason, no message, no advancement, no proof).
+ Assert.assertFalse(restored.isSetCompletionReason());
+ Assert.assertFalse(restored.isSetSanitizedMessage());
+ Assert.assertFalse(restored.isSetExternalMetadataAdvanced());
+ Assert.assertFalse(restored.isSetTerminationProof());
+ Assert.assertEquals(TLanceIndexJobResultCode.NATIVE_OK, restored.getResultCode());
+ }
+
+ @Test
+ public void testLanceIndexJobEnumNumberingMatchesTheIdl() {
+ // Explicit wire numbers are a permanent contract for the worker side; drift here is a
+ // protocol break, not a rename.
+ Assert.assertEquals(1, TLanceIndexMutationType.CREATE.getValue());
+ Assert.assertEquals(2, TLanceIndexMutationType.REPLACE.getValue());
+ Assert.assertEquals(3, TLanceIndexMutationType.DROP.getValue());
+
+ Assert.assertEquals(1, TLanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION.getValue());
+ Assert.assertEquals(2, TLanceIndexJobResultCode.PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT.getValue());
+ Assert.assertEquals(3, TLanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED.getValue());
+ Assert.assertEquals(4, TLanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED.getValue());
+ Assert.assertEquals(5, TLanceIndexJobResultCode.NATIVE_OK.getValue());
+ Assert.assertEquals(6, TLanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT.getValue());
+ Assert.assertEquals(7, TLanceIndexJobResultCode.NATIVE_NOT_FOUND.getValue());
+ Assert.assertEquals(8, TLanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT.getValue());
+ Assert.assertEquals(9, TLanceIndexJobResultCode.NATIVE_NOT_SUPPORTED.getValue());
+ Assert.assertEquals(10, TLanceIndexJobResultCode.NATIVE_INDEX.getValue());
+ Assert.assertEquals(11, TLanceIndexJobResultCode.NATIVE_IO.getValue());
+ Assert.assertEquals(12, TLanceIndexJobResultCode.NATIVE_INTERNAL.getValue());
+
+ Assert.assertEquals(1, TLanceIndexCompletionReason.NONE.getValue());
+ Assert.assertEquals(2, TLanceIndexCompletionReason.IF_CONDITION_NOOP.getValue());
+
+ Assert.assertEquals(1, TLanceIndexTerminationProof.NONE.getValue());
+ Assert.assertEquals(2, TLanceIndexTerminationProof.CHILD_REAPED.getValue());
+
+ // NO_TRUSTED_RESULT is FE-side only and must never gain a wire number.
+ Assert.assertEquals(12, TLanceIndexJobResultCode.values().length);
+ for (TLanceIndexJobResultCode code : TLanceIndexJobResultCode.values()) {
+ Assert.assertNotEquals("NO_TRUSTED_RESULT", code.name());
+ }
+ }
+
+ @Test
+ public void testLanceIndexJobEnumFindByValueResolvesEveryConstant() {
+ for (TLanceIndexMutationType value : TLanceIndexMutationType.values()) {
+ Assert.assertSame(value, TLanceIndexMutationType.findByValue(value.getValue()));
+ }
+ for (TLanceIndexJobResultCode value : TLanceIndexJobResultCode.values()) {
+ Assert.assertSame(value, TLanceIndexJobResultCode.findByValue(value.getValue()));
+ }
+ for (TLanceIndexCompletionReason value : TLanceIndexCompletionReason.values()) {
+ Assert.assertSame(value, TLanceIndexCompletionReason.findByValue(value.getValue()));
+ }
+ for (TLanceIndexTerminationProof value : TLanceIndexTerminationProof.values()) {
+ Assert.assertSame(value, TLanceIndexTerminationProof.findByValue(value.getValue()));
+ }
+ // Unknown numbers (including 0, the thrift default) resolve to null, never to a
+ // wrong constant; the report handler drops such envelopes.
+ Assert.assertNull(TLanceIndexMutationType.findByValue(0));
+ Assert.assertNull(TLanceIndexMutationType.findByValue(4));
+ Assert.assertNull(TLanceIndexJobResultCode.findByValue(0));
+ Assert.assertNull(TLanceIndexJobResultCode.findByValue(13));
+ Assert.assertNull(TLanceIndexCompletionReason.findByValue(0));
+ Assert.assertNull(TLanceIndexCompletionReason.findByValue(3));
+ Assert.assertNull(TLanceIndexTerminationProof.findByValue(0));
+ Assert.assertNull(TLanceIndexTerminationProof.findByValue(3));
+ }
+
+ @Test
+ public void testLanceIndexJobStructFieldIdsMatchTheIdl() {
+ // Explicit field ids are the wire-drift defense of the dedicated channel. A
+ // symmetric round-trip cannot catch renumbering (reader and writer move
+ // together), so the ids are pinned against the generated metadata directly.
+ Map expectedDispatchIds = new HashMap<>();
+ expectedDispatchIds.put("job_id", 1);
+ expectedDispatchIds.put("dispatch_revision", 2);
+ expectedDispatchIds.put("invocation_id", 3);
+ expectedDispatchIds.put("be_process_epoch", 4);
+ expectedDispatchIds.put("deadline_ms", 5);
+ expectedDispatchIds.put("mutation_type", 6);
+ expectedDispatchIds.put("index_name", 7);
+ expectedDispatchIds.put("column_name", 8);
+ expectedDispatchIds.put("index_type", 9);
+ expectedDispatchIds.put("properties_json", 10);
+ expectedDispatchIds.put("if_not_exists", 11);
+ expectedDispatchIds.put("if_exists", 12);
+ expectedDispatchIds.put("dataset_uri", 13);
+ expectedDispatchIds.put("admitted_dataset_version", 14);
+ expectedDispatchIds.put("schema_contract_json", 15);
+ expectedDispatchIds.put("storage_options", 16);
+ Assert.assertEquals(expectedDispatchIds, fieldIdsByName(TLanceIndexJobDispatch.metaDataMap));
+
+ Map expectedReportIds = new HashMap<>();
+ expectedReportIds.put("job_id", 1);
+ expectedReportIds.put("dispatch_revision", 2);
+ expectedReportIds.put("invocation_id", 3);
+ expectedReportIds.put("be_process_epoch", 4);
+ expectedReportIds.put("result_code", 5);
+ expectedReportIds.put("completion_reason", 6);
+ expectedReportIds.put("sanitized_message", 7);
+ expectedReportIds.put("external_metadata_advanced", 8);
+ expectedReportIds.put("termination_proof", 9);
+ Assert.assertEquals(expectedReportIds, fieldIdsByName(TLanceIndexJobReport.metaDataMap));
+ }
+
+ private static Map fieldIdsByName(
+ Map metaDataMap) {
+ Map ids = new HashMap<>();
+ for (Map.Entry entry : metaDataMap.entrySet()) {
+ ids.put(entry.getValue().fieldName, (int) entry.getKey().getThriftFieldId());
+ }
+ return ids;
+ }
+
+ @Test
+ public void testDispatchStorageOptionsReachTheWireUntouched() throws Exception {
+ // Whatever the namespace vends has to reach the worker untranslated inside the
+ // dispatch too, including values Doris itself assigns no meaning to (empty strings).
+ Map options = new HashMap<>();
+ options.put("access_key_id", "ak");
+ options.put("endpoint", "http://127.0.0.1:9000");
+ options.put("azure_storage_sas_token", "");
+ TLanceIndexJobDispatch source = minimalDispatch().setStorageOptions(options);
+
+ TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
+ byte[] bytes = serializer.serialize(source);
+
+ TLanceIndexJobDispatch restored = new TLanceIndexJobDispatch();
+ new TDeserializer(new TCompactProtocol.Factory()).deserialize(restored, bytes);
+
+ Assert.assertEquals(options, restored.getStorageOptions());
+ Assert.assertEquals("", restored.getStorageOptions().get("azure_storage_sas_token"));
+ }
+
+ private static Map lanceStorageOptions() {
+ Map options = new HashMap<>();
+ options.put("access_key_id", "ak");
+ options.put("secret_access_key", "sk");
+ options.put("endpoint", "http://127.0.0.1:9000");
+ options.put("expires_at_millis", "1760000000000");
+ return options;
+ }
+
+ private static TLanceIndexJobDispatch minimalDispatch() {
+ return new TLanceIndexJobDispatch()
+ .setJobId(1L)
+ .setDispatchRevision(1L)
+ .setInvocationId("inv")
+ .setBeProcessEpoch(1L)
+ .setDeadlineMs(1L)
+ .setMutationType(TLanceIndexMutationType.CREATE)
+ .setIndexName("idx")
+ .setColumnName("v")
+ .setIndexType("IVF_PQ")
+ .setDatasetUri("s3://bucket/dataset")
+ .setAdmittedDatasetVersion(1L)
+ .setSchemaContractJson("{}");
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobDispatcherTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobDispatcherTest.java
new file mode 100644
index 00000000000000..14088c2653a64a
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobDispatcherTest.java
@@ -0,0 +1,926 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.RefreshManager;
+import org.apache.doris.common.Config;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.CatalogMgr;
+import org.apache.doris.datasource.lance.LanceExternalCatalog;
+import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
+import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.system.Backend;
+import org.apache.doris.system.BeSelectionPolicy;
+import org.apache.doris.system.Frontend;
+import org.apache.doris.system.SystemInfoService;
+import org.apache.doris.thrift.TLanceIndexJobDispatch;
+import org.apache.doris.thrift.TLanceIndexMutationType;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Fault-matrix coverage for {@link LanceIndexJobDispatcher}. The two test seams
+ * are the manager's edit-log capture (the 3B pattern) and the send seam: a
+ * subclass records every journal record and every send with their ordering,
+ * injects clean error statuses or transport failures, and never opens a real
+ * client pool. The pinned invariants: one round runs the five phases in the
+ * fixed order (deadline sweep, epoch sweep, refresh, dispatch); the markRunning
+ * journal record always precedes the send (durable-before-send); an attempt
+ * whose compare-and-set lost never sends and never reuses its invocation id; a
+ * clean send error converges NOT_COMMITTED while a transport failure converges
+ * UNKNOWN with the possible-live slot still held; only a changed backend
+ * process epoch releases a slot; a local-filesystem dataset is dispatched only
+ * on the asserted single-node topology; and an idle round writes no journal
+ * record. Storage options reach the wire but never a durable record.
+ */
+public class LanceIndexJobDispatcherTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR = "s3://bucket/dataset";
+ private static final long BE1_ID = 1001L;
+ private static final long BE2_ID = 1002L;
+ private static final long BE_EPOCH = 55L;
+ private static final long REPLACED_BE_EPOCH = 77L;
+ private static final long FAR_DEADLINE_MS = System.currentTimeMillis() + 3600_000L;
+ /** Fake markers that must never surface in any durable or logged form. */
+ private static final String FAKE_ACCESS_KEY = "test-ak-marker-not-a-real-credential";
+ private static final String FAKE_SECRET_KEY = "test-sk-marker-not-a-real-credential";
+
+ private final List events = new ArrayList<>();
+ private MockedStatic mockedEnv;
+ private Env env;
+ private SystemInfoService systemInfo;
+ private RefreshManager refreshManager;
+ private LanceExternalCatalog catalog;
+ private TestManager manager;
+ private TestDispatcher dispatcher;
+
+ private int originalIntervalSecond;
+ private int originalMaxDispatchPerRound;
+ private int originalMaxInflightPerBackend;
+ private long originalExecuteDeadlineSecond;
+ private boolean originalLocalFileMutation;
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ events.clear();
+ mockedEnv = Mockito.mockStatic(Env.class);
+ env = Mockito.mock(Env.class);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Mockito.when(env.isMaster()).thenReturn(true);
+ mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo = Mockito.mock(SystemInfoService.class));
+ Mockito.when(systemInfo.selectBackendIdsByPolicy(Mockito.any(BeSelectionPolicy.class), Mockito.eq(1)))
+ .thenReturn(Collections.singletonList(BE1_ID));
+ Mockito.when(systemInfo.getBackend(BE1_ID)).thenReturn(backend(BE1_ID, BE_EPOCH));
+ Mockito.when(systemInfo.getBackend(BE2_ID)).thenReturn(backend(BE2_ID, BE_EPOCH));
+ Mockito.when(systemInfo.getAllBackendIds(true)).thenReturn(Collections.singletonList(BE1_ID));
+ Mockito.when(env.getFrontends(Mockito.any()))
+ .thenReturn(Collections.singletonList(Mockito.mock(Frontend.class)));
+
+ catalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(CATALOG_ID);
+ Mockito.when(catalog.getName()).thenReturn("lance_cat");
+ AbstractS3CompatibleProperties storageProperties = Mockito.mock(AbstractS3CompatibleProperties.class);
+ Mockito.when(storageProperties.getAccessKey()).thenReturn(FAKE_ACCESS_KEY);
+ Mockito.when(storageProperties.getSecretKey()).thenReturn(FAKE_SECRET_KEY);
+ Mockito.when(storageProperties.getEndpoint()).thenReturn("http://minio.example:9000");
+ Mockito.when(storageProperties.getRegion()).thenReturn("us-east-1");
+ org.apache.doris.datasource.CatalogProperty catalogProperty =
+ Mockito.mock(org.apache.doris.datasource.CatalogProperty.class);
+ Mockito.when(catalogProperty.getOrderedStoragePropertiesList())
+ .thenReturn(Collections.singletonList(storageProperties));
+ Mockito.when(catalog.getCatalogProperty()).thenReturn(catalogProperty);
+ CatalogMgr catalogMgr = new CatalogMgr();
+ java.lang.reflect.Field catalogs = CatalogMgr.class.getDeclaredField("idToCatalog");
+ catalogs.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map registered = (Map) catalogs.get(catalogMgr);
+ registered.put(CATALOG_ID, catalog);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+
+ refreshManager = Mockito.mock(RefreshManager.class);
+ Mockito.doAnswer(invocation -> {
+ events.add("refresh:" + invocation.getArgument(1) + "." + invocation.getArgument(2));
+ return null;
+ }).when(refreshManager).handleRefreshTable(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyBoolean());
+ Mockito.when(env.getRefreshManager()).thenReturn(refreshManager);
+
+ manager = new TestManager(events);
+ dispatcher = new TestDispatcher(manager, events);
+
+ originalIntervalSecond = Config.lance_index_job_dispatch_interval_second;
+ originalMaxDispatchPerRound = Config.lance_index_job_max_dispatch_per_round;
+ originalMaxInflightPerBackend = Config.lance_index_job_max_inflight_per_backend;
+ originalExecuteDeadlineSecond = Config.lance_index_job_execute_deadline_second;
+ originalLocalFileMutation = Config.enable_lance_index_local_file_mutation;
+ }
+
+ @AfterEach
+ public void tearDown() {
+ Config.lance_index_job_dispatch_interval_second = originalIntervalSecond;
+ Config.lance_index_job_max_dispatch_per_round = originalMaxDispatchPerRound;
+ Config.lance_index_job_max_inflight_per_backend = originalMaxInflightPerBackend;
+ Config.lance_index_job_execute_deadline_second = originalExecuteDeadlineSecond;
+ Config.enable_lance_index_local_file_mutation = originalLocalFileMutation;
+ mockedEnv.close();
+ }
+
+ // ------------------------------------------------------------------
+ // Round structure
+ // ------------------------------------------------------------------
+
+ @Test
+ public void oneRoundRunsTheFivePhasesInFixedOrder() throws Exception {
+ // Job 1: expired RUNNING, converged by the deadline sweep. Its backend keeps
+ // the recorded epoch so the epoch sweep leaves it alone.
+ admit(1L, "IdxDeadline", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BE1_ID, BE_EPOCH, "inv-1",
+ System.currentTimeMillis() - 1_000L));
+ // Job 2: RUNNING on a backend whose process epoch was replaced.
+ admit(2L, "IdxEpoch", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(2L, 0L, BE2_ID, BE_EPOCH, "inv-2", FAR_DEADLINE_MS));
+ Mockito.when(systemInfo.getBackend(BE2_ID)).thenReturn(backend(BE2_ID, REPLACED_BE_EPOCH));
+ // Job 3: terminal with a refresh obligation.
+ admit(3L, "IdxRefresh", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(3L, 0L, BE1_ID, BE_EPOCH, "inv-3", FAR_DEADLINE_MS));
+ Assertions.assertTrue(manager.completeWithResult(3L, 1L, "inv-3", BE_EPOCH, okResult()));
+ // Job 4: PENDING, dispatched this round.
+ admit(4L, "IdxPending", LOCATOR);
+
+ dispatcher.runAfterCatalogReady();
+
+ int deadlineIdx = events.indexOf(journal(1L, "UNKNOWN", "NOT_REQUIRED", true));
+ int epochIdx = events.indexOf(journal(2L, "RUNNING", "NOT_REQUIRED", false));
+ int refreshRunningIdx = events.indexOf(journal(3L, "COMMITTED", "RUNNING", true));
+ int refreshDoneIdx = events.indexOf(journal(3L, "COMMITTED", "DONE", true));
+ int dispatchIdx = events.indexOf(journal(4L, "RUNNING", "NOT_REQUIRED", true));
+ int sendIdx = events.indexOf("send:4");
+ Assertions.assertTrue(deadlineIdx >= 0, events.toString());
+ Assertions.assertTrue(epochIdx >= 0, events.toString());
+ Assertions.assertTrue(refreshRunningIdx >= 0, events.toString());
+ Assertions.assertTrue(refreshDoneIdx >= 0, events.toString());
+ Assertions.assertTrue(dispatchIdx >= 0, events.toString());
+ Assertions.assertTrue(sendIdx >= 0, events.toString());
+ Assertions.assertTrue(deadlineIdx < epochIdx, events.toString());
+ Assertions.assertTrue(epochIdx < refreshRunningIdx, events.toString());
+ Assertions.assertTrue(refreshRunningIdx < events.indexOf("refresh:db1.tbl1"), events.toString());
+ Assertions.assertTrue(events.indexOf("refresh:db1.tbl1") < refreshDoneIdx, events.toString());
+ Assertions.assertTrue(refreshDoneIdx < dispatchIdx, events.toString());
+ Assertions.assertTrue(dispatchIdx < sendIdx, events.toString());
+
+ // Phase outcomes: the deadline sweep keeps fence and slot, the epoch sweep only
+ // releases the slot, the refresh completes, and the dispatch is durable RUNNING.
+ LanceIndexJob swept = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, swept.getMutationState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT, swept.getResult().getResultCode());
+ Assertions.assertTrue(swept.holdsPossibleLiveSlot());
+ Assertions.assertTrue(manager.isFenceHeld(swept.fenceKey()));
+ LanceIndexJob epochSwept = manager.getJob(2L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, epochSwept.getMutationState());
+ Assertions.assertFalse(epochSwept.holdsPossibleLiveSlot());
+ Assertions.assertEquals(LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE,
+ epochSwept.getTerminationProof());
+ Assertions.assertTrue(manager.isFenceHeld(epochSwept.fenceKey()));
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(3L).getRefreshState());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(4L).getMutationState());
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ }
+
+ @Test
+ public void markRunningJournalPrecedesTheSend() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+
+ dispatcher.runAfterCatalogReady();
+
+ int journalIdx = events.indexOf(journal(1L, "RUNNING", "NOT_REQUIRED", true));
+ int sendIdx = events.indexOf("send:1");
+ Assertions.assertTrue(journalIdx >= 0, events.toString());
+ Assertions.assertTrue(sendIdx >= 0, events.toString());
+ // Direct durable-before-send evidence: the only network path is the send seam,
+ // and it fired strictly after the markRunning journal record.
+ Assertions.assertTrue(journalIdx < sendIdx, events.toString());
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ }
+
+ @Test
+ public void dispatchRequestCarriesTheDurableDispatchIdentity() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ TLanceIndexJobDispatch request = dispatcher.sends.get(0);
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(1L, request.getJobId());
+ Assertions.assertEquals(1L, request.getDispatchRevision());
+ Assertions.assertEquals(stored.getInvocationId(), request.getInvocationId());
+ Assertions.assertEquals(BE_EPOCH, request.getBeProcessEpoch());
+ Assertions.assertTrue(request.getDeadlineMs() > System.currentTimeMillis());
+ Assertions.assertEquals(TLanceIndexMutationType.CREATE, request.getMutationType());
+ Assertions.assertEquals("IdxA", request.getIndexName());
+ Assertions.assertEquals("v", request.getColumnName());
+ Assertions.assertEquals("IVF_PQ", request.getIndexType());
+ Assertions.assertEquals(LOCATOR, request.getDatasetUri());
+ Assertions.assertEquals(7L, request.getAdmittedDatasetVersion());
+ Assertions.assertFalse(request.isIfNotExists());
+ Assertions.assertFalse(request.isIfExists());
+ }
+
+ @Test
+ public void idleRoundWritesNoJournalRecord() {
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(manager.editLog.isEmpty(), manager.editLog.toString());
+ Assertions.assertTrue(dispatcher.sends.isEmpty());
+ Assertions.assertEquals(0, manager.getJobCount());
+ }
+
+ @Test
+ public void intervalIsRereadFromMutableConfigEachRound() {
+ Config.lance_index_job_dispatch_interval_second = 7;
+ dispatcher.runAfterCatalogReady();
+ Assertions.assertEquals(7_000L, dispatcher.getInterval());
+
+ Config.lance_index_job_dispatch_interval_second = 9;
+ dispatcher.runAfterCatalogReady();
+ Assertions.assertEquals(9_000L, dispatcher.getInterval());
+ }
+
+ @Test
+ public void nonPositiveDispatchIntervalIsClampedAtConsumption() {
+ // fe.conf bypasses the config validator; the consumption clamp keeps the daemon
+ // thread alive instead of dying inside Thread.sleep.
+ Config.lance_index_job_dispatch_interval_second = -5;
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1_000L, dispatcher.getInterval());
+ }
+
+ @Test
+ public void zeroDispatchCapsAreClampedSoProgressContinues() throws Exception {
+ Config.lance_index_job_max_dispatch_per_round = 0;
+ Config.lance_index_job_max_inflight_per_backend = 0;
+ admit(1L, "IdxA", LOCATOR);
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void nonPositiveExecuteDeadlineIsClamped() throws Exception {
+ Config.lance_index_job_execute_deadline_second = 0L;
+ admit(1L, "IdxA", LOCATOR);
+ long beforeMs = System.currentTimeMillis();
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ long deadlineMs = dispatcher.sends.get(0).getDeadlineMs();
+ Assertions.assertTrue(deadlineMs >= beforeMs + 1_000L, "deadline=" + deadlineMs + " before=" + beforeMs);
+ Assertions.assertTrue(deadlineMs <= System.currentTimeMillis() + 1_000L);
+ }
+
+ @Test
+ public void dispatchCarriesTheEpochCapturedAtMarkRunningNotALaterHeartbeat() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ Backend selected = systemInfo.getBackend(BE1_ID);
+ manager.afterMarkRunning = () -> selected.setLastStartTime(REPLACED_BE_EPOCH);
+
+ dispatcher.runAfterCatalogReady();
+
+ // A heartbeat landing between the durable record and the send must not split
+ // the dispatch identity: the wire carries the same epoch the journal recorded,
+ // which is what the callback matches and the epoch sweep releases against.
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Assertions.assertEquals(BE_EPOCH, dispatcher.sends.get(0).getBeProcessEpoch());
+ Assertions.assertEquals(BE_EPOCH, manager.getJob(1L).getBeProcessEpoch().longValue());
+ }
+
+ @Test
+ public void leadershipLossBeforeTheSendSendsNothing() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ // The entry check passes, then mastership is lost before the pre-send recheck.
+ Mockito.when(env.isMaster()).thenReturn(true, false);
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty(), events.toString());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void checkpointThreadSkipsTheRound() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ mockedEnv.when(Env::isCheckpointThread).thenReturn(true);
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty(), events.toString());
+ // Only the admission record exists: the round never ran.
+ Assertions.assertEquals(1, manager.editLog.size(), manager.editLog.toString());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ }
+
+ // ------------------------------------------------------------------
+ // No-second-dispatch
+ // ------------------------------------------------------------------
+
+ @Test
+ public void markRunningCasLossSkipsTheSendAndRetriesNextRound() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ manager.rejectNextMarkRunning = true;
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(events.contains("markRunningRejected:1"), events.toString());
+ Assertions.assertFalse(events.contains("send:1"), events.toString());
+ Assertions.assertEquals(1, manager.editLog.size());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+
+ // The next round builds a fresh identity from scratch and sends exactly once.
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(2, manager.editLog.size());
+ }
+
+ @Test
+ public void runningJobIsNeverRedispatchedAcrossRounds() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+
+ dispatcher.runAfterCatalogReady();
+ dispatcher.runAfterCatalogReady();
+ dispatcher.runAfterCatalogReady();
+
+ // After the one send of the one durable RUNNING record, later rounds neither
+ // resend nor rewrite anything: convergence belongs to the callback or sweeps.
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Assertions.assertEquals(2, manager.editLog.size());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void preSendRecheckFailureSendsNothing() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ manager.hijackedInvocationId = "concurrent-invocation";
+
+ dispatcher.runAfterCatalogReady();
+
+ // markRunning won, but the job was advanced concurrently before the send: the
+ // recheck fails, nothing is sent, and no resend ever happens for the old identity.
+ Assertions.assertTrue(events.contains("hijacked:1"), events.toString());
+ Assertions.assertTrue(dispatcher.sends.isEmpty(), events.toString());
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertEquals("concurrent-invocation", stored.getInvocationId());
+ // Exactly the create and markRunning records: the dispatcher neither converged
+ // the job nor journaled anything else; the deadline sweep owns it now.
+ Assertions.assertEquals(2, manager.editLog.size());
+ }
+
+ // ------------------------------------------------------------------
+ // Send outcomes
+ // ------------------------------------------------------------------
+
+ @Test
+ public void cleanSendErrorConvergesNotCommitted() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ dispatcher.statusToReturn = new TStatus(TStatusCode.INTERNAL_ERROR);
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ // A clean error status proves the dispatch was never enqueued, so the
+ // invocation is known never to have executed: NOT_COMMITTED, no refresh owed.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED,
+ stored.getResult().getResultCode());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ }
+
+ @Test
+ public void sendWithoutAStatusConvergesUnknownNotCommitted() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ dispatcher.statusToReturn = null;
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ // The absence of a status is the absence of a trusted answer, not a clean
+ // rejection: only a complete error status proves the dispatch was not
+ // enqueued, so this converges UNKNOWN with everything still held.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ stored.getResult().getResultCode());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(stored.holdsPossibleLiveSlot());
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ }
+
+ @Test
+ public void sendFailureConvergesUnknownAndKeepsThePossibleLiveSlot() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ dispatcher.throwOnSend = true;
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ // The request may have reached the backend, so nothing about the outcome is
+ // trusted: UNKNOWN through the only channel, with fence, quota, and the
+ // possible-live slot all still held.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT, stored.getResult().getResultCode());
+ Assertions.assertTrue(stored.holdsPossibleLiveSlot());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(containsJob(manager.getJobsHoldingPossibleLiveSlot(), 1L));
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ }
+
+ @Test
+ public void preparationFailureConvergesUnknownWithoutASend() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ // The job's catalog resolves to nothing: storage options cannot be resolved,
+ // which is an FE-side failure, never a trusted worker rejection. Built before
+ // the stubbing: constructing it inside when(...) triggers Mockito's
+ // unfinished-stubbing detection.
+ CatalogMgr catalogless = new CatalogMgr();
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogless);
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty(), events.toString());
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT, stored.getResult().getResultCode());
+ Assertions.assertTrue(stored.holdsPossibleLiveSlot());
+ }
+
+ // ------------------------------------------------------------------
+ // Backpressure
+ // ------------------------------------------------------------------
+
+ @Test
+ public void perRoundCapDefersDispatchesBeyondTheLimit() throws Exception {
+ Config.lance_index_job_max_dispatch_per_round = 2;
+ // Roomy per-backend cap so only the per-round limit binds in this test.
+ Config.lance_index_job_max_inflight_per_backend = 8;
+ admit(1L, "IdxA", LOCATOR);
+ admit(2L, "IdxB", LOCATOR);
+ admit(3L, "IdxC", LOCATOR);
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(2, dispatcher.sends.size());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(2L).getMutationState());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(3L).getMutationState());
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(3, dispatcher.sends.size());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(3L).getMutationState());
+ }
+
+ @Test
+ public void perBackendInflightCapDefersDispatchUntilASlotFrees() throws Exception {
+ Config.lance_index_job_max_inflight_per_backend = 2;
+ admit(1L, "IdxOccupied1", LOCATOR);
+ admit(2L, "IdxOccupied2", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BE1_ID, BE_EPOCH, "place-1", FAR_DEADLINE_MS));
+ Assertions.assertTrue(manager.markRunning(2L, 0L, BE1_ID, BE_EPOCH, "place-2", FAR_DEADLINE_MS));
+ admit(3L, "IdxWaiting", LOCATOR);
+ int journalBefore = manager.editLog.size();
+
+ dispatcher.runAfterCatalogReady();
+
+ // Both in-flight slots of the only selectable backend are taken: the round
+ // attempts nothing and the job keeps waiting as PENDING.
+ Assertions.assertTrue(dispatcher.sends.isEmpty(), events.toString());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(3L).getMutationState());
+ Assertions.assertEquals(journalBefore, manager.editLog.size());
+
+ // One placeholder converges, so the snapshot count drops and the next round sends.
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, "place-1", BE_EPOCH, okResult()));
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(3L).getMutationState());
+ }
+
+ @Test
+ public void noSelectableBackendKeepsTheJobPending() throws Exception {
+ Mockito.when(systemInfo.selectBackendIdsByPolicy(Mockito.any(BeSelectionPolicy.class), Mockito.eq(1)))
+ .thenReturn(Collections.emptyList());
+ admit(1L, "IdxA", LOCATOR);
+ int journalBefore = manager.editLog.size();
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(journalBefore, manager.editLog.size());
+ }
+
+ @Test
+ public void identityLessPendingJobIsNeverPickedUpForDispatch() throws Exception {
+ admit(1L, "IdxHealthy", LOCATOR);
+ // A corrupt identity-less PENDING record: queryable, but never dispatchable.
+ manager.replayUpsertJob(GsonUtils.GSON.fromJson(
+ "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class));
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Assertions.assertEquals(1L, dispatcher.sends.get(0).getJobId());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(5L).getMutationState());
+ Assertions.assertEquals(Collections.singletonList(5L), manager.getCorruptUnresolvedJobIds());
+ }
+
+ // ------------------------------------------------------------------
+ // Deadline sweep
+ // ------------------------------------------------------------------
+
+ @Test
+ public void deadlineSweepConvergesOnlyTheExpiredRunningJob() throws Exception {
+ admit(1L, "IdxExpired", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BE1_ID, BE_EPOCH, "inv-1",
+ System.currentTimeMillis() - 1_000L));
+ admit(2L, "IdxCurrent", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(2L, 0L, BE1_ID, BE_EPOCH, "inv-2", FAR_DEADLINE_MS));
+
+ dispatcher.runAfterCatalogReady();
+
+ LanceIndexJob expired = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, expired.getMutationState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT, expired.getResult().getResultCode());
+ // Expiry bounds the wait only: slot, fence, and quota all stay held.
+ Assertions.assertTrue(expired.holdsPossibleLiveSlot());
+ Assertions.assertTrue(manager.isFenceHeld(expired.fenceKey()));
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(2L).getMutationState());
+ Assertions.assertTrue(dispatcher.sends.isEmpty());
+ }
+
+ @Test
+ public void callbackArrivingBeforeTheDeadlineSweepOnlyWarns() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BE1_ID, BE_EPOCH, "inv-1",
+ System.currentTimeMillis() - 1_000L));
+ LanceIndexJob staleSnapshot = manager.getJob(1L);
+ // The matching callback converges the job first, and its refresh duty is
+ // settled too, so the late sweep is the only remaining actor.
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, "inv-1", BE_EPOCH, okResult()));
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
+ manager.staleExpiredJob = staleSnapshot;
+ int journalBefore = manager.editLog.size();
+
+ dispatcher.runAfterCatalogReady();
+
+ // The sweep's late completeWithResult loses the identity check and only warns:
+ // no state change, no journal record, and the round itself must not throw.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_OK, stored.getResult().getResultCode());
+ Assertions.assertEquals(journalBefore, manager.editLog.size());
+ }
+
+ // ------------------------------------------------------------------
+ // Epoch sweep
+ // ------------------------------------------------------------------
+
+ @Test
+ public void epochSweepReleasesTheSlotOfAReplacedBackendProcess() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BE2_ID, BE_EPOCH, "inv-1", FAR_DEADLINE_MS));
+ Mockito.when(systemInfo.getBackend(BE2_ID)).thenReturn(backend(BE2_ID, REPLACED_BE_EPOCH));
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ // The proof releases only the slot: the mutation state, fence, and quota stay.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertFalse(stored.holdsPossibleLiveSlot());
+ Assertions.assertEquals(LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE, stored.getTerminationProof());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void epochSweepReleasesTheSlotOfAnUnknownJobToo() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BE2_ID, BE_EPOCH, "inv-1", FAR_DEADLINE_MS));
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, "inv-1", BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE, "ambiguous", false)));
+ Mockito.when(systemInfo.getBackend(BE2_ID)).thenReturn(backend(BE2_ID, REPLACED_BE_EPOCH));
+
+ dispatcher.runAfterCatalogReady();
+
+ // The slot-release proof is independent of the outcome, so an UNKNOWN job's
+ // slot is released exactly like a RUNNING one's.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertFalse(stored.holdsPossibleLiveSlot());
+ Assertions.assertEquals(LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE, stored.getTerminationProof());
+ }
+
+ @Test
+ public void missingBackendEntrySameEpochOrHeartbeatLossNeverReleasesTheSlot() throws Exception {
+ // A backend entry that disappeared proves nothing: the worker may still run
+ // behind a partition, so the slot stays until a stronger proof.
+ admit(1L, "IdxGone", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BE1_ID, BE_EPOCH, "inv-1", FAR_DEADLINE_MS));
+ Mockito.when(systemInfo.getBackend(BE1_ID)).thenReturn(null);
+ dispatcher.runAfterCatalogReady();
+ Assertions.assertTrue(manager.getJob(1L).holdsPossibleLiveSlot(), "backend entry must not release the slot");
+
+ // Same epoch: the recorded process is still the one that received the dispatch.
+ admit(2L, "IdxSameEpoch", LOCATOR);
+ Assertions.assertTrue(manager.markRunning(2L, 0L, BE2_ID, BE_EPOCH, "inv-2", FAR_DEADLINE_MS));
+ Mockito.when(systemInfo.getBackend(BE1_ID)).thenReturn(backend(BE1_ID, BE_EPOCH));
+ dispatcher.runAfterCatalogReady();
+ Assertions.assertTrue(manager.getJob(1L).holdsPossibleLiveSlot());
+ Assertions.assertTrue(manager.getJob(2L).holdsPossibleLiveSlot(), "same epoch must not release the slot");
+
+ // Heartbeat loss without a process restart: same epoch, dead marker, no release.
+ Backend notAlive = backend(BE2_ID, BE_EPOCH);
+ notAlive.setAlive(false);
+ Mockito.when(systemInfo.getBackend(BE2_ID)).thenReturn(notAlive);
+ dispatcher.runAfterCatalogReady();
+ Assertions.assertTrue(manager.getJob(1L).holdsPossibleLiveSlot());
+ Assertions.assertTrue(manager.getJob(2L).holdsPossibleLiveSlot(), "heartbeat loss must not release the slot");
+
+ for (LanceIndexJob record : manager.editLog) {
+ Assertions.assertNotEquals(LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE,
+ record.getTerminationProof());
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // Local-filesystem datasets
+ // ------------------------------------------------------------------
+
+ @Test
+ public void localFileDatasetStaysPendingWhileTheAssertionIsOff() throws Exception {
+ admit(1L, "IdxLocal", "file:///data/dataset");
+ int journalBefore = manager.editLog.size();
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(journalBefore, manager.editLog.size());
+ }
+
+ @Test
+ public void schemelessAbsolutePathIsAlsoALocalDataset() throws Exception {
+ admit(1L, "IdxLocal", "/data/dataset");
+ int journalBefore = manager.editLog.size();
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(journalBefore, manager.editLog.size());
+ }
+
+ @Test
+ public void assertedLocalMutationIsRefusedWithMultipleFrontends() throws Exception {
+ Config.enable_lance_index_local_file_mutation = true;
+ Mockito.when(env.getFrontends(Mockito.any()))
+ .thenReturn(Arrays.asList(Mockito.mock(Frontend.class), Mockito.mock(Frontend.class)));
+ admit(1L, "IdxLocal", "file:///data/dataset");
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void assertedLocalMutationIsRefusedWithMultipleAliveBackends() throws Exception {
+ Config.enable_lance_index_local_file_mutation = true;
+ Mockito.when(systemInfo.getAllBackendIds(true)).thenReturn(Arrays.asList(BE1_ID, BE2_ID));
+ admit(1L, "IdxLocal", "file:///data/dataset");
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertTrue(dispatcher.sends.isEmpty());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void assertedLocalMutationDispatchesOnASingleNodeTopology() throws Exception {
+ Config.enable_lance_index_local_file_mutation = true;
+ admit(1L, "IdxLocal", "file:///data/dataset");
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Assertions.assertEquals("file:///data/dataset", dispatcher.sends.get(0).getDatasetUri());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ }
+
+ // ------------------------------------------------------------------
+ // Storage-option confidentiality
+ // ------------------------------------------------------------------
+
+ @Test
+ public void storageOptionsReachTheWireButNeverADurableRecord() throws Exception {
+ admit(1L, "IdxA", LOCATOR);
+
+ dispatcher.runAfterCatalogReady();
+
+ // The resolved credentials really did reach the wire request...
+ Assertions.assertEquals(1, dispatcher.sends.size());
+ Map wireOptions = dispatcher.sends.get(0).getStorageOptions();
+ Assertions.assertEquals(FAKE_ACCESS_KEY, wireOptions.get("aws_access_key_id"));
+ Assertions.assertEquals(FAKE_SECRET_KEY, wireOptions.get("aws_secret_access_key"));
+ // ...while no durable record, in journal form or in its log/toString rendering,
+ // carries the credential key or value: storage options are never persisted.
+ for (LanceIndexJob record : manager.editLog) {
+ String json = GsonUtils.GSON.toJson(record);
+ Assertions.assertFalse(json.contains(FAKE_ACCESS_KEY), "journal json leaked the access key");
+ Assertions.assertFalse(json.contains(FAKE_SECRET_KEY), "journal json leaked the secret key");
+ Assertions.assertFalse(record.toString().contains(FAKE_ACCESS_KEY), "toString leaked the access key");
+ Assertions.assertFalse(record.toString().contains(FAKE_SECRET_KEY), "toString leaked the secret key");
+ }
+ String storedJson = GsonUtils.GSON.toJson(manager.getJob(1L));
+ Assertions.assertFalse(storedJson.contains(FAKE_ACCESS_KEY));
+ Assertions.assertFalse(storedJson.contains(FAKE_SECRET_KEY));
+ Assertions.assertFalse(manager.getJob(1L).toString().contains("aws_access_key_id"));
+ }
+
+ // ------------------------------------------------------------------
+ // Fixtures
+ // ------------------------------------------------------------------
+
+ private static Backend backend(long id, long processEpoch) {
+ Backend backend = new Backend(id, "host-" + id, 9050);
+ backend.setAlive(true);
+ backend.setLastStartTime(processEpoch);
+ return backend;
+ }
+
+ private void admit(long jobId, String displayName, String locator) throws Exception {
+ manager.createJob(new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, locator,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
+ null, 7L, null), 100, 100, 100);
+ }
+
+ private static String journal(long jobId, String mutationState, String refreshState, boolean slot) {
+ return "journal:" + jobId + ":" + mutationState + ":" + refreshState + ":" + (slot ? "slot" : "noslot");
+ }
+
+ private static boolean containsJob(List jobs, long jobId) {
+ for (LanceIndexJob job : jobs) {
+ if (job.getJobId() == jobId) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static LanceIndexJobResult okResult() {
+ return new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, "ok", false);
+ }
+
+ /**
+ * Edit-log seam plus dispatch-boundary injections: captures every durable
+ * record, can lose one markRunning compare-and-set, can replay a concurrent
+ * higher-revision record right after a won markRunning (pre-send recheck), and
+ * can hand the deadline sweep one stale snapshot after a callback converged
+ * the job first.
+ */
+ private static class TestManager extends LanceIndexJobManager {
+ private final List editLog = new ArrayList<>();
+ private final List events;
+ private boolean rejectNextMarkRunning;
+ private String hijackedInvocationId;
+ private Runnable afterMarkRunning;
+ private LanceIndexJob staleExpiredJob;
+
+ TestManager(List events) {
+ this.events = events;
+ }
+
+ @Override
+ protected void writeEditLog(LanceIndexJob job) {
+ editLog.add(job);
+ events.add("journal:" + job.getJobId() + ":" + job.getMutationState() + ":" + job.getRefreshState()
+ + ":" + (job.isPossibleLiveOwned() ? "slot" : "noslot"));
+ }
+
+ @Override
+ public boolean markRunning(long jobId, long expectedRevision, long backendId, long beProcessEpoch,
+ String invocationId, long deadlineMs) {
+ if (rejectNextMarkRunning) {
+ rejectNextMarkRunning = false;
+ events.add("markRunningRejected:" + jobId);
+ return false;
+ }
+ boolean marked = super.markRunning(jobId, expectedRevision, backendId, beProcessEpoch, invocationId,
+ deadlineMs);
+ if (marked && afterMarkRunning != null) {
+ // Simulates a heartbeat landing right after the durable record, before
+ // the dispatcher reads the backend again for the wire request.
+ Runnable hook = afterMarkRunning;
+ afterMarkRunning = null;
+ hook.run();
+ }
+ if (marked && hijackedInvocationId != null) {
+ String hijacker = hijackedInvocationId;
+ hijackedInvocationId = null;
+ LanceIndexJob concurrent = getJob(jobId);
+ concurrent.setRevision(expectedRevision + 5);
+ concurrent.setDispatchRevision(expectedRevision + 5);
+ concurrent.setInvocationId(hijacker);
+ replayUpsertJob(concurrent);
+ events.add("hijacked:" + jobId);
+ }
+ return marked;
+ }
+
+ @Override
+ public List getExpiredRunningJobs(long nowMs) {
+ if (staleExpiredJob != null) {
+ LanceIndexJob stale = staleExpiredJob;
+ staleExpiredJob = null;
+ return new ArrayList<>(Collections.singletonList(stale));
+ }
+ return super.getExpiredRunningJobs(nowMs);
+ }
+ }
+
+ /**
+ * Send seam: records the call order against the journal events and injects the
+ * per-case send outcome. No client pool is ever touched, so a send event is the
+ * earliest possible network activity of a dispatch attempt.
+ */
+ private static class TestDispatcher extends LanceIndexJobDispatcher {
+ private final List events;
+ private final List sends = new ArrayList<>();
+ private TStatus statusToReturn = new TStatus(TStatusCode.OK);
+ private boolean throwOnSend;
+
+ TestDispatcher(LanceIndexJobManager jobManager, List events) {
+ super(jobManager);
+ this.events = events;
+ }
+
+ @Override
+ protected TStatus sendExecuteRequest(Backend backend, TLanceIndexJobDispatch dispatch) throws Exception {
+ events.add("send:" + dispatch.getJobId());
+ sends.add(dispatch);
+ if (throwOnSend) {
+ throw new RuntimeException("injected transport failure");
+ }
+ return statusToReturn;
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java
index 7675d05ab18c5f..5f69cfbc4af5c8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerQueryTest.java
@@ -17,17 +17,27 @@
package org.apache.doris.datasource.lance.job;
+import org.apache.doris.persist.gson.GsonUtils;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
/**
- * Coverage for the two read-side queries added to {@link LanceIndexJobManager} for the
- * admission slice: {@link LanceIndexJobManager#getAllJobsSnapshot()} (the data source of
- * SHOW LANCE INDEX JOBS: every job, copies only, ordered by job id) and
+ * Coverage for the read-side queries of {@link LanceIndexJobManager}: the admission-slice
+ * queries {@link LanceIndexJobManager#getAllJobsSnapshot()} (the data source of SHOW LANCE
+ * INDEX JOBS: every job, copies only, ordered by job id) and
* {@link LanceIndexJobManager#hasUnresolvedJobsForCatalog(long)} (the catalog DDL guard
- * probe: any job still holding its fence and unresolved quota for the catalog).
+ * probe), plus the dispatcher-slice queries: {@link LanceIndexJobManager#getJobsNeedingDispatch(int)}
+ * (only PENDING records whose dispatch identity is complete, in job id order, at most
+ * limit), {@link LanceIndexJobManager#getExpiredRunningJobs(long)} (only RUNNING past the
+ * deadline), {@link LanceIndexJobManager#getJobsHoldingPossibleLiveSlot()} (slot holders
+ * with complete identity, regardless of mutation state), and the force-release filter of
+ * {@link LanceIndexJobManager#getJobsNeedingRefresh()}.
*/
public class LanceIndexJobManagerQueryTest {
private static final long CATALOG_ID = 10L;
@@ -114,6 +124,166 @@ public void unresolvedJobInAnotherCatalogDoesNotLeak() throws Exception {
Assertions.assertTrue(manager.hasUnresolvedJobsForCatalog(OTHER_CATALOG_ID));
}
+ // ------------------------------------------------------------------
+ // dispatcher-slice queries
+ // ------------------------------------------------------------------
+
+ @Test
+ public void dispatchQueryReturnsPendingJobsWithCompleteTargetIdentity() throws Exception {
+ TestManager manager = new TestManager();
+ // A durable PENDING record carrying the full dispatch identity is dispatchable.
+ manager.replayUpsertJob(dispatchablePending(1L, "IdxDispatchable"));
+ // An admitted PENDING record (the form createJob produces) carries no dispatch quad
+ // yet and is dispatchable too: the quad is written by markRunning, the very step
+ // this query feeds, so only the target identity is required here.
+ manager.createJob(newCreateJob(2L, "IdxAdmitted", CATALOG_ID), 100, 100, 100);
+ // Non-PENDING states are invisible to the dispatch sweep even with full identity.
+ manager.replayUpsertJob(runningRecord(4L, "IdxRunning"));
+ LanceIndexJob terminal = dispatchablePending(5L, "IdxTerminal");
+ terminal.setMutationState(LanceIndexJobMutationState.COMMITTED);
+ terminal.setRefreshState(LanceIndexJobRefreshState.DONE);
+ manager.replayUpsertJob(terminal);
+ // A corrupt identity-less PENDING record is never dispatchable (replayed last: it
+ // also fail-closes new admissions, which the createJob above must not hit).
+ manager.replayUpsertJob(GsonUtils.GSON.fromJson(
+ "{\"jid\":3,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class));
+
+ List dispatchable = manager.getJobsNeedingDispatch(10);
+ Assertions.assertEquals(2, dispatchable.size());
+ Assertions.assertEquals(1L, dispatchable.get(0).getJobId());
+ Assertions.assertEquals(2L, dispatchable.get(1).getJobId());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, dispatchable.get(0).getMutationState());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, dispatchable.get(1).getMutationState());
+ }
+
+ @Test
+ public void dispatchQueryHonorsLimitAndJobIdOrder() {
+ TestManager manager = new TestManager();
+ // Insertion order deliberately scrambled; the sweep returns job id order (FIFO).
+ for (long jobId : new long[]{5L, 1L, 4L, 2L, 3L}) {
+ manager.replayUpsertJob(dispatchablePending(jobId, "Idx" + jobId));
+ }
+
+ List all = manager.getJobsNeedingDispatch(10);
+ Assertions.assertEquals(5, all.size());
+ for (int i = 0; i < all.size(); i++) {
+ Assertions.assertEquals(i + 1L, all.get(i).getJobId());
+ }
+
+ // The limit keeps the smallest ids: ordering happens before truncation, so a
+ // stable subset of undispatchable jobs can never crowd out later ids.
+ List capped = manager.getJobsNeedingDispatch(3);
+ Assertions.assertEquals(3, capped.size());
+ for (int i = 0; i < capped.size(); i++) {
+ Assertions.assertEquals(i + 1L, capped.get(i).getJobId());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, capped.get(i).getMutationState());
+ }
+
+ Assertions.assertTrue(manager.getJobsNeedingDispatch(0).isEmpty());
+ }
+
+ @Test
+ public void expiredQueryReturnsOnlyRunningJobsPastTheirDeadline() throws Exception {
+ TestManager manager = new TestManager();
+ long now = 1_000L;
+ // RUNNING with an expired deadline is the sweep's only input.
+ manager.replayUpsertJob(runningRecord(1L, "IdxExpired", 999L));
+ // The boundary is strict: a deadline exactly at "now" has not expired.
+ manager.replayUpsertJob(runningRecord(2L, "IdxAtBoundary", now));
+ manager.replayUpsertJob(runningRecord(3L, "IdxStillWaiting", 1_001L));
+ // RUNNING without a deadline (an old record) never expires on its own.
+ LanceIndexJob deadlineless = runningRecord(4L, "IdxDeadlineless");
+ deadlineless.setDeadlineMs(null);
+ manager.replayUpsertJob(deadlineless);
+ // Non-RUNNING states are invisible even with an expired deadline in the record.
+ LanceIndexJob pending = dispatchablePending(5L, "IdxPending");
+ pending.setDeadlineMs(1L);
+ manager.replayUpsertJob(pending);
+ LanceIndexJob unknown = runningRecord(6L, "IdxUnknown", 1L);
+ unknown.setMutationState(LanceIndexJobMutationState.UNKNOWN);
+ manager.replayUpsertJob(unknown);
+
+ List expired = manager.getExpiredRunningJobs(now);
+ Assertions.assertEquals(1, expired.size());
+ Assertions.assertEquals(1L, expired.get(0).getJobId());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, expired.get(0).getMutationState());
+ }
+
+ @Test
+ public void possibleLiveQueryReturnsSlotHoldersWithCompleteIdentity() throws Exception {
+ TestManager manager = new TestManager();
+ // A RUNNING dispatch holds a slot.
+ manager.replayUpsertJob(runningRecord(1L, "IdxRunning"));
+ // An UNKNOWN converged from RUNNING holds its slot exactly the same way: the release
+ // proof is independent of the outcome.
+ LanceIndexJob unknown = runningRecord(2L, "IdxUnknown");
+ unknown.setMutationState(LanceIndexJobMutationState.UNKNOWN);
+ unknown.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE, "deadline expired", false));
+ unknown.setRevision(2L);
+ manager.replayUpsertJob(unknown);
+ // A slot already released by a termination proof is gone.
+ LanceIndexJob reaped = runningRecord(3L, "IdxReaped");
+ reaped.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED);
+ reaped.setPossibleLiveOwned(false);
+ manager.replayUpsertJob(reaped);
+ // A force-released record holds nothing.
+ LanceIndexJob forced = runningRecord(4L, "IdxForced");
+ forced.setMutationState(LanceIndexJobMutationState.UNKNOWN);
+ forced.setForceReleased(true);
+ manager.replayUpsertJob(forced);
+ // A corrupt record that claims a slot but lacks dispatch identity cannot be matched
+ // by the epoch sweep and is skipped.
+ manager.replayUpsertJob(GsonUtils.GSON.fromJson(
+ "{\"jid\":5,\"rev\":1,\"ms\":\"RUNNING\",\"plo\":true}", LanceIndexJob.class));
+
+ List holders = manager.getJobsHoldingPossibleLiveSlot();
+ Assertions.assertEquals(2, holders.size());
+ // The query does not promise an order; assert membership and each state.
+ List holderIds = new ArrayList<>();
+ for (LanceIndexJob holder : holders) {
+ holderIds.add(holder.getJobId());
+ Assertions.assertTrue(holder.holdsPossibleLiveSlot());
+ }
+ Collections.sort(holderIds);
+ Assertions.assertEquals(Arrays.asList(1L, 2L), holderIds);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, manager.getJob(2L).getMutationState());
+ }
+
+ @Test
+ public void refreshQueryExcludesForceReleasedJobs() throws Exception {
+ TestManager manager = new TestManager();
+ // A terminal job owing its first refresh is the driver's input.
+ manager.createJob(newCreateJob(1L, "IdxOwed", CATALOG_ID), 100, 100, 100);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, "ok", false)));
+
+ // The same terminal shape, but force-released: its fence is already gone and the
+ // driver must never pick it up again.
+ LanceIndexJob forced = newCreateJob(7L, "IdxForced", CATALOG_ID);
+ forced.setMutationState(LanceIndexJobMutationState.COMMITTED);
+ forced.setRefreshState(LanceIndexJobRefreshState.REQUIRED);
+ forced.setRevision(2L);
+ forced.setForceReleased(true);
+ manager.replayUpsertJob(forced);
+ // And the FAILED variant of a forced job, equally invisible.
+ LanceIndexJob forcedFailed = newCreateJob(8L, "IdxForcedFailed", CATALOG_ID);
+ forcedFailed.setMutationState(LanceIndexJobMutationState.NOT_COMMITTED);
+ forcedFailed.setRefreshState(LanceIndexJobRefreshState.FAILED);
+ forcedFailed.setRevision(3L);
+ forcedFailed.setForceReleased(true);
+ manager.replayUpsertJob(forcedFailed);
+
+ List needing = manager.getJobsNeedingRefresh();
+ Assertions.assertEquals(1, needing.size());
+ Assertions.assertEquals(1L, needing.get(0).getJobId());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, needing.get(0).getRefreshState());
+ Assertions.assertFalse(needing.get(0).isForceReleased());
+ }
+
private static LanceIndexJob newCreateJob(long jobId, String displayName, long catalogId) {
return new LanceIndexJob(jobId, "tester", catalogId, "db1", "tbl1",
LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
@@ -122,6 +292,35 @@ private static LanceIndexJob newCreateJob(long jobId, String displayName, long c
null, 7L, null);
}
+ /**
+ * A durable PENDING record whose dispatch identity is complete, the form the dispatch
+ * sweep accepts. Built through setters and replayed verbatim, like a follower applying
+ * a journal record.
+ */
+ private static LanceIndexJob dispatchablePending(long jobId, String displayName) {
+ LanceIndexJob job = newCreateJob(jobId, displayName, CATALOG_ID);
+ job.setMutationState(LanceIndexJobMutationState.PENDING);
+ job.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
+ job.setBackendId(BACKEND_ID);
+ job.setBeProcessEpoch(BE_EPOCH);
+ job.setInvocationId(INVOCATION_ID);
+ job.setDispatchRevision(1L);
+ job.setRevision(1L);
+ return job;
+ }
+
+ private static LanceIndexJob runningRecord(long jobId, String displayName) {
+ return runningRecord(jobId, displayName, DEADLINE_MS);
+ }
+
+ private static LanceIndexJob runningRecord(long jobId, String displayName, long deadlineMs) {
+ LanceIndexJob job = dispatchablePending(jobId, displayName);
+ job.setMutationState(LanceIndexJobMutationState.RUNNING);
+ job.setPossibleLiveOwned(true);
+ job.setDeadlineMs(deadlineMs);
+ return job;
+ }
+
private static class TestManager extends LanceIndexJobManager {
@Override
protected void writeEditLog(LanceIndexJob job) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshDriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshDriverTest.java
new file mode 100644
index 00000000000000..a2fcf90eafab0d
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshDriverTest.java
@@ -0,0 +1,462 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.RefreshManager;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.CatalogMgr;
+import org.apache.doris.datasource.lance.LanceExternalCatalog;
+import org.apache.doris.system.Backend;
+import org.apache.doris.system.BeSelectionPolicy;
+import org.apache.doris.system.Frontend;
+import org.apache.doris.system.SystemInfoService;
+import org.apache.doris.thrift.TLanceIndexJobDispatch;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Refresh-phase coverage for {@link LanceIndexJobDispatcher}. The external-table
+ * refresh is mocked, the manager's edit-log seam captures every durable record,
+ * and each round is one direct {@code runAfterCatalogReady} call, so the pinned
+ * invariants are all observable: the driver runs markRefreshRunning, then
+ * {@code handleRefreshTable(catalogName, db, table, ignoreIfNotExists=true)},
+ * then DONE, which releases the fence and the unresolved quota; a
+ * {@link DdlException} keeps the fence and retries on a later round; the retry
+ * throttle delays only FAILED refreshes, never a first REQUIRED one; a silent
+ * no-op refresh (a half-orphan target) still completes to DONE; force-released
+ * jobs owe nothing; and a refresh stranded at RUNNING by a lost driver is
+ * downgraded by the master-transfer sweep and then driven to DONE here.
+ */
+public class LanceIndexJobRefreshDriverTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR = "s3://bucket/dataset";
+ private static final long BACKEND_ID = 1001L;
+ private static final long BE_EPOCH = 55L;
+ private static final long FAR_DEADLINE_MS = System.currentTimeMillis() + 3600_000L;
+
+ private final List events = new ArrayList<>();
+ private MockedStatic mockedEnv;
+ private Env env;
+ private RefreshManager refreshManager;
+ private TestManager manager;
+ private TestDispatcher dispatcher;
+
+ private int originalRetrySecond;
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ events.clear();
+ mockedEnv = Mockito.mockStatic(Env.class);
+ env = Mockito.mock(Env.class);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ Mockito.when(env.isMaster()).thenReturn(true);
+ SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class);
+ Mockito.when(systemInfo.selectBackendIdsByPolicy(Mockito.any(BeSelectionPolicy.class), Mockito.eq(1)))
+ .thenReturn(Collections.emptyList());
+ mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo);
+ Mockito.when(env.getFrontends(Mockito.any()))
+ .thenReturn(Collections.singletonList(Mockito.mock(Frontend.class)));
+
+ LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class);
+ Mockito.when(catalog.getId()).thenReturn(CATALOG_ID);
+ Mockito.when(catalog.getName()).thenReturn("lance_cat");
+ CatalogMgr catalogMgr = new CatalogMgr();
+ java.lang.reflect.Field catalogs = CatalogMgr.class.getDeclaredField("idToCatalog");
+ catalogs.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Map registered = (Map) catalogs.get(catalogMgr);
+ registered.put(CATALOG_ID, catalog);
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
+
+ refreshManager = Mockito.mock(RefreshManager.class);
+ Mockito.doAnswer(invocation -> {
+ events.add("refresh:" + invocation.getArgument(1) + "." + invocation.getArgument(2));
+ return null;
+ }).when(refreshManager).handleRefreshTable(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyBoolean());
+ Mockito.when(env.getRefreshManager()).thenReturn(refreshManager);
+
+ manager = new TestManager(events);
+ dispatcher = new TestDispatcher(manager, events);
+
+ originalRetrySecond = Config.lance_index_job_refresh_retry_second;
+ }
+
+ @AfterEach
+ public void tearDown() {
+ Config.lance_index_job_refresh_retry_second = originalRetrySecond;
+ mockedEnv.close();
+ }
+
+ @Test
+ public void refreshRunsMarkRunningThenRefreshTableThenDoneWithExactParameters() throws Exception {
+ admitTerminalCommitted(1L, "IdxA");
+
+ dispatcher.runAfterCatalogReady();
+
+ // Order: the durable refresh-RUNNING record precedes the external refresh,
+ // and the DONE record follows it.
+ int refreshRunningIdx = events.indexOf(journal(1L, "COMMITTED", "RUNNING", true));
+ int refreshTableIdx = events.indexOf("refresh:db1.tbl1");
+ int refreshDoneIdx = events.indexOf(journal(1L, "COMMITTED", "DONE", true));
+ Assertions.assertTrue(refreshRunningIdx >= 0, events.toString());
+ Assertions.assertTrue(refreshTableIdx >= 0, events.toString());
+ Assertions.assertTrue(refreshDoneIdx >= 0, events.toString());
+ Assertions.assertTrue(refreshRunningIdx < refreshTableIdx, events.toString());
+ Assertions.assertTrue(refreshTableIdx < refreshDoneIdx, events.toString());
+
+ ArgumentCaptor catalogName = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor dbName = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor tableName = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor ignoreIfNotExists = ArgumentCaptor.forClass(Boolean.class);
+ Mockito.verify(refreshManager, Mockito.times(1)).handleRefreshTable(catalogName.capture(), dbName.capture(),
+ tableName.capture(), ignoreIfNotExists.capture());
+ Assertions.assertEquals("lance_cat", catalogName.getValue());
+ Assertions.assertEquals("db1", dbName.getValue());
+ Assertions.assertEquals("tbl1", tableName.getValue());
+ // A half-orphan target is a legal input to the refresh call, not an error.
+ Assertions.assertEquals(Boolean.TRUE, ignoreIfNotExists.getValue());
+
+ // Completing the refresh duty releases the fence and the unresolved quota,
+ // and a refresh round never dispatches anything.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, stored.getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(stored.fenceKey()));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
+ Assertions.assertTrue(dispatcher.sendJobIds.isEmpty());
+ }
+
+ @Test
+ public void ddlExceptionMarksRefreshFailedAndKeepsTheFenceForARetry() throws Exception {
+ Config.lance_index_job_refresh_retry_second = 0;
+ admitTerminalCommitted(1L, "IdxA");
+ Mockito.doThrow(new DdlException("refresh exploded")).doAnswer(invocation -> {
+ events.add("refresh:" + invocation.getArgument(1) + "." + invocation.getArgument(2));
+ return null;
+ }).when(refreshManager).handleRefreshTable(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyBoolean());
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ LanceIndexJob failed = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, failed.getRefreshState());
+ // The fence and quota survive the failure: DONE is the only release.
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(containsJob(manager.getUnresolvedJobs(), 1L));
+ Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), 1L));
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(1L).getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Mockito.verify(refreshManager, Mockito.times(2)).handleRefreshTable(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyBoolean());
+ }
+
+ @Test
+ public void uncheckedExceptionStillMarksRefreshFailedInsteadOfStrandingRunning() throws Exception {
+ Config.lance_index_job_refresh_retry_second = 0;
+ admitTerminalCommitted(1L, "IdxA");
+ Mockito.doThrow(new IllegalStateException("metadata path exploded")).doAnswer(invocation -> {
+ events.add("refresh:" + invocation.getArgument(1) + "." + invocation.getArgument(2));
+ return null;
+ }).when(refreshManager).handleRefreshTable(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyBoolean());
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ // The failure face of the refresh path is not only the typed DdlException: an
+ // unchecked exception must still leave the durable refresh state FAILED, or the
+ // job would strand in refresh RUNNING until a master transfer.
+ Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, manager.getJob(1L).getRefreshState());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), 1L));
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(1L).getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ }
+
+ @Test
+ public void doneTransitionLosingTheCompareAndSetIsRetriedWithTheFreshRevision() throws Exception {
+ Config.lance_index_job_refresh_retry_second = 0;
+ FlakyDoneTestManager flaky = new FlakyDoneTestManager(events);
+ manager = flaky;
+ dispatcher = new TestDispatcher(flaky, events);
+ admitTerminalCommitted(1L, "IdxA");
+ flaky.failNextDone = 1;
+
+ dispatcher.runAfterCatalogReady();
+
+ // Losing the DONE compare-and-set once (as a concurrent revision bump would)
+ // must not strand the refresh in RUNNING: the driver re-reads the revision and
+ // retries within the same round.
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, stored.getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(stored.fenceKey()));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
+ }
+
+ @Test
+ public void freshFailedRefreshIsThrottledUntilTheRetryIntervalElapses() throws Exception {
+ Config.lance_index_job_refresh_retry_second = 300;
+ admitTerminalCommitted(1L, "IdxA");
+ Mockito.doThrow(new DdlException("refresh exploded")).when(refreshManager)
+ .handleRefreshTable(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyBoolean());
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, manager.getJob(1L).getRefreshState());
+ int journalAfterFailure = manager.editLog.size();
+
+ // The FAILED transition just bumped updateTimeMs: the immediately following
+ // round is inside the retry window and must not even attempt the CAS.
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, manager.getJob(1L).getRefreshState());
+ Assertions.assertEquals(journalAfterFailure, manager.editLog.size());
+ Mockito.verify(refreshManager, Mockito.times(1)).handleRefreshTable(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyBoolean());
+ Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), 1L));
+ }
+
+ @Test
+ public void staleFailedRefreshIsRetriedOncePastTheThrottleWindow() throws Exception {
+ Config.lance_index_job_refresh_retry_second = 300;
+ // A FAILED refresh whose last transition aged past the retry window, built as
+ // a replayed durable record so its updateTimeMs is controllable.
+ LanceIndexJob stale = newCreateJob(1L, "IdxA");
+ stale.setRevision(2L);
+ stale.setMutationState(LanceIndexJobMutationState.COMMITTED);
+ stale.setRefreshState(LanceIndexJobRefreshState.FAILED);
+ long staleTime = System.currentTimeMillis()
+ - (Config.lance_index_job_refresh_retry_second * 1000L + 60_000L);
+ stale.setCreateTimeMs(staleTime);
+ stale.setUpdateTimeMs(staleTime);
+ manager.replayUpsertJob(stale);
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(1L).getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Mockito.verify(refreshManager, Mockito.times(1)).handleRefreshTable(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyBoolean());
+ }
+
+ @Test
+ public void halfOrphanTargetRefreshesSilentlyAndStillCompletes() throws Exception {
+ admitTerminalCommitted(1L, "IdxA");
+ // The target db/table was already dropped externally: the refresh call is a
+ // silent no-op (nothing is left to invalidate), which is success here.
+ Mockito.doNothing().when(refreshManager).handleRefreshTable(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyBoolean());
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, stored.getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
+ }
+
+ @Test
+ public void forceReleasedJobsOweNoRefresh() throws Exception {
+ LanceIndexJob forced = newCreateJob(1L, "IdxA");
+ forced.setRevision(2L);
+ forced.setMutationState(LanceIndexJobMutationState.UNKNOWN);
+ forced.setRefreshState(LanceIndexJobRefreshState.REQUIRED);
+ forced.setForceReleased(true);
+ manager.replayUpsertJob(forced);
+
+ dispatcher.runAfterCatalogReady();
+
+ Mockito.verify(refreshManager, Mockito.never()).handleRefreshTable(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyBoolean());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
+ }
+
+ @Test
+ public void downgradedRunningRefreshIsDrivenToDoneAfterMasterTransfer() throws Exception {
+ // A driver crashed between markRefreshRunning and the refresh call: the job
+ // is terminal with a refresh stuck at RUNNING.
+ TestManager source = new TestManager(new ArrayList<>());
+ source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, "inv-1", FAR_DEADLINE_MS);
+ source.completeWithResult(1L, 1L, "inv-1", BE_EPOCH, okResult());
+ source.markRefreshRunning(1L, 2L);
+ Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, source.getJob(1L).getRefreshState());
+
+ // A new master replays the journal, then its election sweep downgrades the
+ // stranded RUNNING refresh back to REQUIRED before any daemon could start.
+ for (LanceIndexJob record : source.editLog) {
+ manager.replayUpsertJob(record);
+ }
+ manager.onTransferToMaster();
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
+
+ dispatcher.runAfterCatalogReady();
+
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(1L).getRefreshState());
+ Assertions.assertFalse(manager.isFenceHeld(manager.getJob(1L).fenceKey()));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void missingCatalogFailsTheRefreshClosedAndKeepsRetrying() throws Exception {
+ Config.lance_index_job_refresh_retry_second = 0;
+ admitTerminalCommitted(1L, "IdxA");
+ // Built before the stubbing: constructing it inside when(...) triggers
+ // Mockito's unfinished-stubbing detection.
+ CatalogMgr catalogless = new CatalogMgr();
+ Mockito.when(env.getCatalogMgr()).thenReturn(catalogless);
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ dispatcher.runAfterCatalogReady();
+ dispatcher.runAfterCatalogReady();
+
+ // Unreachable while the unresolved-job guard blocks catalog drops, but the
+ // driver still transitions and retries instead of stranding the job.
+ Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, manager.getJob(1L).getRefreshState());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Mockito.verify(refreshManager, Mockito.never()).handleRefreshTable(Mockito.anyString(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyBoolean());
+ }
+
+ // ------------------------------------------------------------------
+ // Fixtures
+ // ------------------------------------------------------------------
+
+ /** Creates a job and walks it to COMMITTED with a REQUIRED refresh, the driver's input. */
+ private void admitTerminalCommitted(long jobId, String displayName) throws Exception {
+ manager.createJob(newCreateJob(jobId, displayName), 100, 100, 100);
+ Assertions.assertTrue(manager.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, "inv-" + jobId,
+ FAR_DEADLINE_MS));
+ Assertions.assertTrue(manager.completeWithResult(jobId, 1L, "inv-" + jobId, BE_EPOCH, okResult()));
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(jobId).getRefreshState());
+ }
+
+ private static LanceIndexJob newCreateJob(long jobId, String displayName) {
+ return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
+ null, 7L, null);
+ }
+
+ private static String journal(long jobId, String mutationState, String refreshState, boolean slot) {
+ return "journal:" + jobId + ":" + mutationState + ":" + refreshState + ":" + (slot ? "slot" : "noslot");
+ }
+
+ private static boolean containsJob(List jobs, long jobId) {
+ for (LanceIndexJob job : jobs) {
+ if (job.getJobId() == jobId) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static LanceIndexJobResult okResult() {
+ return new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, "ok", false);
+ }
+
+ /** Edit-log seam: captures every durable record with its phase-observable label. */
+ private static class TestManager extends LanceIndexJobManager {
+ private final List editLog = new ArrayList<>();
+ private final List events;
+
+ TestManager(List events) {
+ this.events = events;
+ }
+
+ @Override
+ protected void writeEditLog(LanceIndexJob job) {
+ editLog.add(job);
+ events.add("journal:" + job.getJobId() + ":" + job.getMutationState() + ":" + job.getRefreshState()
+ + ":" + (job.isPossibleLiveOwned() ? "slot" : "noslot"));
+ }
+ }
+
+ /** Fails the DONE transition a bounded number of times, as a concurrent revision bump would. */
+ private static class FlakyDoneTestManager extends TestManager {
+ private int failNextDone = 0;
+
+ FlakyDoneTestManager(List events) {
+ super(events);
+ }
+
+ @Override
+ public boolean markRefreshDone(long jobId, long expectedRevision) {
+ if (failNextDone > 0) {
+ failNextDone--;
+ return false;
+ }
+ return super.markRefreshDone(jobId, expectedRevision);
+ }
+ }
+
+ /** Send seam: refresh rounds must never dispatch; any send is a test failure. */
+ private static class TestDispatcher extends LanceIndexJobDispatcher {
+ private final List events;
+ private final List sendJobIds = new ArrayList<>();
+
+ TestDispatcher(LanceIndexJobManager jobManager, List events) {
+ super(jobManager);
+ this.events = events;
+ }
+
+ @Override
+ protected TStatus sendExecuteRequest(Backend backend, TLanceIndexJobDispatch dispatch) throws Exception {
+ events.add("send:" + dispatch.getJobId());
+ sendJobIds.add(dispatch.getJobId());
+ return new TStatus(TStatusCode.OK);
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobReportHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobReportHandlerTest.java
new file mode 100644
index 00000000000000..6ce24ec8064f79
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobReportHandlerTest.java
@@ -0,0 +1,438 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.thrift.TLanceIndexCompletionReason;
+import org.apache.doris.thrift.TLanceIndexJobReport;
+import org.apache.doris.thrift.TLanceIndexJobResultCode;
+import org.apache.doris.thrift.TLanceIndexTerminationProof;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Coverage for {@link LanceIndexJobReportHandler}, the thin shim applying one typed
+ * result envelope to the durable job record. The pinned invariants: every wire result
+ * code lands the classified (mutationState, refreshState, completionReason) triple of
+ * the classification table; a stale report (wrong dispatch revision, invocation id, BE
+ * process epoch, or an already-terminal job) only warns and changes nothing; a
+ * CHILD_REAPED proof releases exactly the possible-live slot (never the outcome, never
+ * the fence); and a malformed envelope (missing or unknown result code, sanitized
+ * message past the durable bound) is dropped whole so the dispatcher's deadline sweep
+ * converges the job. Message text is stored verbatim and never inspected to infer an
+ * outcome, and NO_TRUSTED_RESULT never arrives on the wire.
+ */
+public class LanceIndexJobReportHandlerTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR = "s3://bucket/dataset";
+ private static final long BACKEND_ID = 1001L;
+ private static final long BE_EPOCH = 55L;
+ private static final String INVOCATION_ID = "invocation-1";
+ private static final long NOT_EXPIRED_DEADLINE_MS = Long.MAX_VALUE;
+
+ // ------------------------------------------------------------------
+ // result_code x mutation_type matrix
+ // ------------------------------------------------------------------
+
+ @Test
+ public void everyWireCodeLandsTheClassifiedTripleForEveryMutationType() throws DdlException {
+ for (TLanceIndexJobResultCode wireCode : TLanceIndexJobResultCode.values()) {
+ for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
+ for (boolean ifExists : new boolean[]{false, true}) {
+ for (boolean advanced : new boolean[]{false, true}) {
+ assertOneLandedTriple(wireCode, type, ifExists, advanced);
+ }
+ }
+ }
+ }
+ }
+
+ private void assertOneLandedTriple(TLanceIndexJobResultCode wireCode, LanceIndexJobMutationType type,
+ boolean ifExists, boolean advanced) throws DdlException {
+ TestManager manager = runningManager(1L, "Idx" + wireCode + type + ifExists + advanced, type, ifExists);
+ LanceIndexJobReportHandler handler = new LanceIndexJobReportHandler(manager);
+ handler.handle(matchingReport(wireCode)
+ .setExternalMetadataAdvanced(advanced));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ String context = "code=" + wireCode + ", type=" + type + ", ifExists=" + ifExists
+ + ", advanced=" + advanced;
+ // The FE enum mirrors the wire enum name for name; the landed triple must be exactly
+ // the classification of that code (the exhaustive restatement lives in
+ // LanceIndexJobResultClassifyTest, so here the oracle is the classifier itself and
+ // the pinned families below restate the load-bearing rows with literals).
+ LanceIndexJobResultCode feCode = LanceIndexJobResultCode.valueOf(wireCode.name());
+ LanceIndexJobResultCode.Classification expected =
+ LanceIndexJobResultCode.classify(type, feCode, ifExists, advanced);
+ Assertions.assertEquals(expected.getMutationState(), stored.getMutationState(), context);
+ Assertions.assertEquals(expected.getRefreshState(), stored.getRefreshState(), context);
+ Assertions.assertEquals(expected.getCompletionReason(), stored.getResult().getCompletionReason(), context);
+ Assertions.assertEquals(feCode, stored.getResult().getResultCode(), context);
+ Assertions.assertEquals(advanced, stored.getResult().isExternalMetadataAdvanced(), context);
+ // One completion journal record, exactly one.
+ Assertions.assertEquals(1, manager.editLog.size(), context);
+ }
+
+ @Test
+ public void preInvocationRejectionsAreNotCommittedAndOweRefreshOnlyOnAdvancement() throws DdlException {
+ for (TLanceIndexJobResultCode code : new TLanceIndexJobResultCode[]{
+ TLanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION,
+ TLanceIndexJobResultCode.PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT,
+ TLanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED,
+ TLanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED}) {
+ TestManager quiet = runningManager(1L, "IdxQuiet", LanceIndexJobMutationType.CREATE, false);
+ new LanceIndexJobReportHandler(quiet).handle(matchingReport(code));
+ LanceIndexJob stored = quiet.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState(),
+ "code=" + code);
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState(),
+ "code=" + code);
+
+ TestManager advanced = runningManager(1L, "IdxAdvanced", LanceIndexJobMutationType.CREATE, false);
+ new LanceIndexJobReportHandler(advanced).handle(matchingReport(code)
+ .setExternalMetadataAdvanced(true));
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED,
+ advanced.getJob(1L).getMutationState(), "code=" + code);
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED,
+ advanced.getJob(1L).getRefreshState(), "code=" + code);
+ }
+ }
+
+ @Test
+ public void nativeOkCommitsAndOwesRefresh() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxOk", LanceIndexJobMutationType.CREATE, false);
+ new LanceIndexJobReportHandler(manager).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_OK).setSanitizedMessage("built"));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_OK, stored.getResult().getResultCode());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, stored.getResult().getCompletionReason());
+ }
+
+ @Test
+ public void commitConflictIsNotCommittedButOwesRefresh() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxConflict", LanceIndexJobMutationType.REPLACE, false);
+ new LanceIndexJobReportHandler(manager).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
+ }
+
+ @Test
+ public void notFoundAttributionDependsOnMutationTypeAndIfExists() throws DdlException {
+ for (LanceIndexJobMutationType type : new LanceIndexJobMutationType[]{
+ LanceIndexJobMutationType.CREATE, LanceIndexJobMutationType.REPLACE}) {
+ TestManager manager = runningManager(1L, "Idx" + type, type, false);
+ new LanceIndexJobReportHandler(manager).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_NOT_FOUND));
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState(),
+ "type=" + type);
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState(),
+ "type=" + type);
+ }
+
+ TestManager plainDrop = runningManager(1L, "IdxDrop", LanceIndexJobMutationType.DROP, false);
+ new LanceIndexJobReportHandler(plainDrop).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_NOT_FOUND));
+ LanceIndexJob dropped = plainDrop.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, dropped.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, dropped.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, dropped.getResult().getCompletionReason());
+
+ TestManager ifExistsDrop = runningManager(1L, "IdxDropIf", LanceIndexJobMutationType.DROP, true);
+ new LanceIndexJobReportHandler(ifExistsDrop).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_NOT_FOUND)
+ .setCompletionReason(TLanceIndexCompletionReason.IF_CONDITION_NOOP));
+ LanceIndexJob noop = ifExistsDrop.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, noop.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, noop.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.IF_CONDITION_NOOP,
+ noop.getResult().getCompletionReason());
+ }
+
+ @Test
+ public void invalidArgumentIsUnknownWithoutRefresh() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxBadArg", LanceIndexJobMutationType.CREATE, false);
+ new LanceIndexJobReportHandler(manager).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT,
+ stored.getResult().getResultCode());
+ }
+
+ @Test
+ public void noTrustedResultNeverArrivesOnTheWire() {
+ // NO_TRUSTED_RESULT is FE-side only (deadline sweep, send failure, master-transfer
+ // sweep); the wire enum must not carry it, or a worker could report it and bypass
+ // the ambiguity rule.
+ Assertions.assertEquals(12, TLanceIndexJobResultCode.values().length);
+ for (TLanceIndexJobResultCode code : TLanceIndexJobResultCode.values()) {
+ Assertions.assertNotEquals("NO_TRUSTED_RESULT", code.name());
+ }
+ // The FE enum is exactly the wire enum plus that one FE-only code, so every wire
+ // name resolves through the name-based mapping in the handler.
+ Assertions.assertEquals(13, LanceIndexJobResultCode.values().length);
+ for (TLanceIndexJobResultCode code : TLanceIndexJobResultCode.values()) {
+ Assertions.assertEquals(code.name(), LanceIndexJobResultCode.valueOf(code.name()).name());
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // stale reports
+ // ------------------------------------------------------------------
+
+ @Test
+ public void staleReportsOnlyWarnAndChangeNothing() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxA", LanceIndexJobMutationType.CREATE, false);
+ LanceIndexJobReportHandler handler = new LanceIndexJobReportHandler(manager);
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ // Wrong dispatch revision, wrong invocation id, wrong BE process epoch, unknown job.
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_OK).setDispatchRevision(0L));
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_OK).setInvocationId("invocation-x"));
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_OK).setBeProcessEpoch(BE_EPOCH + 1));
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_OK).setJobId(404L));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertEquals(1L, stored.getRevision());
+ Assertions.assertNull(stored.getResult());
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+
+ // A matched report still completes afterwards; a duplicate then arrives for a job
+ // that is already terminal and is dropped the same way.
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_OK));
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, manager.getJob(1L).getMutationState());
+ handler.handle(matchingReport(TLanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED));
+ LanceIndexJob terminal = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, terminal.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, terminal.getRefreshState());
+ Assertions.assertEquals(2L, terminal.getRevision());
+ Assertions.assertEquals(1, manager.editLog.size());
+ }
+
+ // ------------------------------------------------------------------
+ // CHILD_REAPED termination proof
+ // ------------------------------------------------------------------
+
+ @Test
+ public void childReapedProofReleasesOnlyThePossibleLiveSlot() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxA", LanceIndexJobMutationType.CREATE, false);
+ LanceIndexJobReportHandler handler = new LanceIndexJobReportHandler(manager);
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT)
+ .setTerminationProof(TLanceIndexTerminationProof.CHILD_REAPED));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ // The proof says the child process ended, which still says nothing about the outcome.
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexTerminationProof.CHILD_REAPED, stored.getTerminationProof());
+ Assertions.assertFalse(stored.holdsPossibleLiveSlot());
+ // Only the slot was released: fence and quota survive until FORCE.
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertEquals(2, manager.editLog.size());
+ }
+
+ @Test
+ public void childReapedProofWithMismatchedIdentityReleasesNothing() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxA", LanceIndexJobMutationType.CREATE, false);
+ LanceIndexJobReportHandler handler = new LanceIndexJobReportHandler(manager);
+
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT)
+ .setInvocationId("invocation-x")
+ .setTerminationProof(TLanceIndexTerminationProof.CHILD_REAPED));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertTrue(stored.holdsPossibleLiveSlot());
+ Assertions.assertEquals(LanceIndexTerminationProof.NONE, stored.getTerminationProof());
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ }
+
+ @Test
+ public void childReapedProofForUnknownJobChangesNothing() {
+ TestManager manager = new TestManager();
+ LanceIndexJobReportHandler handler = new LanceIndexJobReportHandler(manager);
+
+ handler.handle(matchingReport(TLanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT)
+ .setJobId(404L)
+ .setTerminationProof(TLanceIndexTerminationProof.CHILD_REAPED));
+
+ Assertions.assertEquals(0, manager.getJobCount());
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ }
+
+ // ------------------------------------------------------------------
+ // malformed envelopes
+ // ------------------------------------------------------------------
+
+ @Test
+ public void malformedEnvelopesAreDroppedForTheDeadlineSweep() throws DdlException {
+ TestManager missingCode = runningManager(1L, "IdxMissing", LanceIndexJobMutationType.CREATE, false);
+ TLanceIndexJobReport withoutCode = matchingReport(TLanceIndexJobResultCode.NATIVE_OK);
+ withoutCode.unsetResultCode();
+ new LanceIndexJobReportHandler(missingCode).handle(withoutCode);
+ assertManagerUnchangedByDroppedEnvelope(missingCode);
+
+ TestManager overlong = runningManager(1L, "IdxOverlong", LanceIndexJobMutationType.CREATE, false);
+ StringBuilder over = new StringBuilder();
+ for (int i = 0; i < LanceIndexJobResult.MAX_MESSAGE_BYTES + 1; i++) {
+ over.append('x');
+ }
+ new LanceIndexJobReportHandler(overlong).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_OK).setSanitizedMessage(over.toString()));
+ assertManagerUnchangedByDroppedEnvelope(overlong);
+
+ // A later valid envelope for the same job still completes it: the drop was not a
+ // terminal transition, and the deadline sweep is only the fallback.
+ new LanceIndexJobReportHandler(overlong).handle(matchingReport(TLanceIndexJobResultCode.NATIVE_OK));
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, overlong.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void unknownWireResultCodeIsDroppedLikeAnyMalformedEnvelope() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxUnknownCode", LanceIndexJobMutationType.CREATE, false);
+ // A code this FE does not know can only exist as enum skew on the wire; model it by
+ // a wire-code instance whose name resolves to nothing on the FE side.
+ TLanceIndexJobResultCode skew = Mockito.mock(TLanceIndexJobResultCode.class);
+ Mockito.when(skew.name()).thenReturn("SOMETHING_ONLY_THE_BE_KNOWS");
+ Mockito.when(skew.getValue()).thenReturn(9999);
+
+ new LanceIndexJobReportHandler(manager).handle(matchingReport(skew));
+
+ assertManagerUnchangedByDroppedEnvelope(manager);
+ }
+
+ @Test
+ public void nullReportIsDroppedWithoutThrowing() {
+ TestManager manager = new TestManager();
+ new LanceIndexJobReportHandler(manager).handle(null);
+ Assertions.assertEquals(0, manager.getJobCount());
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ }
+
+ // ------------------------------------------------------------------
+ // toResult passthrough
+ // ------------------------------------------------------------------
+
+ @Test
+ public void messageAndOptionalFlagsPassThroughUntouched() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxPassthrough", LanceIndexJobMutationType.DROP, true);
+ new LanceIndexJobReportHandler(manager).handle(
+ matchingReport(TLanceIndexJobResultCode.NATIVE_NOT_FOUND)
+ .setCompletionReason(TLanceIndexCompletionReason.IF_CONDITION_NOOP)
+ .setSanitizedMessage("index absent on the provider")
+ .setExternalMetadataAdvanced(false));
+
+ LanceIndexJobResult result = manager.getJob(1L).getResult();
+ Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_NOT_FOUND, result.getResultCode());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.IF_CONDITION_NOOP, result.getCompletionReason());
+ Assertions.assertEquals("index absent on the provider", result.getSanitizedMessage());
+ Assertions.assertFalse(result.isExternalMetadataAdvanced());
+ }
+
+ @Test
+ public void unsetOptionalFlagsDefaultToNoneAndFalse() throws DdlException {
+ TestManager manager = runningManager(1L, "IdxDefaults", LanceIndexJobMutationType.CREATE, false);
+ new LanceIndexJobReportHandler(manager).handle(matchingReport(TLanceIndexJobResultCode.NATIVE_OK));
+
+ LanceIndexJobResult result = manager.getJob(1L).getResult();
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, result.getCompletionReason());
+ Assertions.assertFalse(result.isExternalMetadataAdvanced());
+ Assertions.assertNull(result.getSanitizedMessage());
+ }
+
+ // ------------------------------------------------------------------
+ // fixtures
+ // ------------------------------------------------------------------
+
+ private static TLanceIndexJobReport matchingReport(TLanceIndexJobResultCode resultCode) {
+ return new TLanceIndexJobReport()
+ .setJobId(1L)
+ .setDispatchRevision(1L)
+ .setInvocationId(INVOCATION_ID)
+ .setBeProcessEpoch(BE_EPOCH)
+ .setResultCode(resultCode);
+ }
+
+ private static void assertManagerUnchangedByDroppedEnvelope(TestManager manager) {
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertNotNull(stored, "the dropped envelope must not delete the job");
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertEquals(1L, stored.getRevision());
+ Assertions.assertNull(stored.getResult());
+ Assertions.assertTrue(stored.holdsPossibleLiveSlot());
+ // Zero change anywhere in the manager: no journal record, fence and quota intact.
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ Assertions.assertTrue(manager.isFenceHeld(stored.fenceKey()));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ }
+
+ /**
+ * A manager holding exactly one durable RUNNING job for the given mutation type,
+ * dispatched with the standard test identity (backend {@link #BACKEND_ID}, epoch
+ * {@link #BE_EPOCH}, invocation {@link #INVOCATION_ID}, dispatch revision 1). The
+ * setup journal records are cleared so tests can assert on post-setup writes alone.
+ */
+ private static TestManager runningManager(long jobId, String displayName,
+ LanceIndexJobMutationType type, boolean ifExists) throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newJob(jobId, displayName, type, ifExists), 100, 100, 100);
+ Assertions.assertTrue(manager.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ NOT_EXPIRED_DEADLINE_MS));
+ manager.editLog.clear();
+ return manager;
+ }
+
+ private static LanceIndexJob newJob(long jobId, String displayName,
+ LanceIndexJobMutationType type, boolean ifExists) {
+ return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ type, false, ifExists, "IVF_PQ", "v", null, 7L, null);
+ }
+
+ /**
+ * Edit-log seam: captures every durable record instead of writing the journal.
+ */
+ private static class TestManager extends LanceIndexJobManager {
+ private final List editLog = new ArrayList<>();
+
+ @Override
+ protected void writeEditLog(LanceIndexJob job) {
+ editLog.add(job);
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
index ecbd2b42799bb1..71996262881f90 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
@@ -19,17 +19,23 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.common.io.CountingDataOutputStream;
+import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.meta.MetaPersistMethod;
import org.apache.doris.persist.meta.PersistMetaModules;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
import java.io.DataInputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -75,4 +81,75 @@ public void lanceIndexJobManagerIsTheLastBaseImageModuleWithEnvBindings() throws
Assertions.assertEquals(long.class, persistMethod.readMethod.getReturnType());
Assertions.assertEquals(long.class, persistMethod.writeMethod.getReturnType());
}
+
+ /**
+ * The dispatcher is wired as a master-only daemon: an instance field on {@link Env}
+ * (assigned in the Env constructor from the job manager, so both share one durable
+ * image), of a {@link MasterDaemon} subclass constructible from exactly the manager.
+ */
+ @Test
+ public void lanceIndexJobDispatcherIsAWiredMasterDaemon() throws Exception {
+ Field field = Env.class.getDeclaredField("lanceIndexJobDispatcher");
+ Assertions.assertFalse(Modifier.isStatic(field.getModifiers()), "one dispatcher per Env instance");
+ Assertions.assertEquals(LanceIndexJobDispatcher.class, field.getType());
+ Assertions.assertTrue(MasterDaemon.class.isAssignableFrom(field.getType()),
+ "the dispatcher must start through the master-only MasterDaemon machinery");
+ Assertions.assertNotNull(
+ LanceIndexJobDispatcher.class.getDeclaredConstructor(LanceIndexJobManager.class),
+ "the dispatcher is constructed from the Env-owned job manager");
+ }
+
+ /**
+ * Source-order wiring of the dispatch lifecycle in Env.java: the constructor creates
+ * the dispatcher on the manager, only {@code startMasterOnlyDaemonThreads} starts it
+ * (never the non-master path), and the master-transfer sweep of the job manager runs
+ * before that start, so no dispatcher round can ever observe a durable RUNNING left by
+ * the old master. Reflection cannot see call sites, so this reads the source; it is
+ * skipped when sources are not next to the test run (jar-only environment).
+ */
+ @Test
+ public void dispatcherStartsInStartMasterOnlyDaemonThreadsAfterTheTransferSweep() throws Exception {
+ String source = readEnvSource();
+
+ Assertions.assertTrue(source.contains(
+ "this.lanceIndexJobDispatcher = new LanceIndexJobDispatcher(lanceIndexJobManager);"),
+ "the Env constructor must create the dispatcher on the Env-owned job manager");
+
+ String masterOnlyBody = methodBody(source, "protected void startMasterOnlyDaemonThreads() {");
+ Assertions.assertTrue(masterOnlyBody.contains("lanceIndexJobDispatcher.start();"),
+ "the dispatcher must be started in startMasterOnlyDaemonThreads");
+ Assertions.assertFalse(
+ methodBody(source, "protected void startNonMasterDaemonThreads() {").contains(
+ "lanceIndexJobDispatcher"),
+ "the dispatcher must never start on a non-master FE");
+
+ int sweep = source.indexOf("lanceIndexJobManager.onTransferToMaster();");
+ int daemonStart = source.indexOf("startMasterOnlyDaemonThreads();");
+ Assertions.assertTrue(sweep >= 0, "the master-transfer sweep call was not found");
+ Assertions.assertTrue(daemonStart > sweep,
+ "the RUNNING-to-UNKNOWN sweep must run before any master-only daemon starts");
+ }
+
+ private static String readEnvSource() throws Exception {
+ // Surefire runs with the module directory as the working directory; also try the
+ // checkout root so an IDE run from fe/ resolves the same file.
+ for (Path candidate : new Path[]{
+ Paths.get("src/main/java/org/apache/doris/catalog/Env.java"),
+ Paths.get("fe-core/src/main/java/org/apache/doris/catalog/Env.java")}) {
+ if (Files.exists(candidate)) {
+ return new String(Files.readAllBytes(candidate), StandardCharsets.UTF_8);
+ }
+ }
+ Assumptions.assumeTrue(false, "Env.java source is not available next to the test run; skipping");
+ throw new IllegalStateException("unreachable");
+ }
+
+ /** Extracts one method's source from its definition down to its closing brace. */
+ private static String methodBody(String source, String definition) {
+ int signature = source.indexOf(definition);
+ Assertions.assertTrue(signature >= 0, "method definition not found in Env.java: " + definition);
+ int end = source.indexOf("\n }", signature);
+ Assertions.assertTrue(end > signature, "no closing brace found for method " + definition);
+ return source.substring(signature, end);
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/LanceIndexReportFrontendServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/LanceIndexReportFrontendServiceTest.java
new file mode 100644
index 00000000000000..7f14a15679993e
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/service/LanceIndexReportFrontendServiceTest.java
@@ -0,0 +1,130 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.service;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.datasource.lance.job.LanceIndexJob;
+import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
+import org.apache.doris.datasource.lance.job.LanceIndexJobResult;
+import org.apache.doris.datasource.lance.job.LanceIndexJobResultCode;
+import org.apache.doris.datasource.lance.job.LanceIndexTerminationProof;
+import org.apache.doris.thrift.TLanceIndexJobReport;
+import org.apache.doris.thrift.TLanceIndexJobResultCode;
+import org.apache.doris.thrift.TLanceIndexTerminationProof;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+/**
+ * Coverage for the report entry point
+ * {@link FrontendServiceImpl#reportLanceIndexJobResult(TLanceIndexJobReport)}: the layer
+ * must stay thin. Only the master accepts a report (a non-master answers NOT_MASTER and
+ * never touches the job manager), and on the master the envelope is handed to the report
+ * handler verbatim: the identity quad and every typed field of the classified result must
+ * reach {@link LanceIndexJobManager#completeWithResult} unchanged, and a CHILD_REAPED
+ * proof must reach {@link LanceIndexJobManager#recordTerminationProof} with the durable
+ * backend id as its source.
+ */
+public class LanceIndexReportFrontendServiceTest {
+ private static final long JOB_ID = 1L;
+ private static final long DISPATCH_REVISION = 1L;
+ private static final long BACKEND_ID = 1001L;
+ private static final long BE_EPOCH = 55L;
+ private static final String INVOCATION_ID = "invocation-1";
+
+ @Test
+ public void nonMasterRejectsTheReportWithoutTouchingTheJobManager() throws Exception {
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.isMaster()).thenReturn(false);
+ LanceIndexJobManager manager = Mockito.mock(LanceIndexJobManager.class);
+ Mockito.when(env.getLanceIndexJobManager()).thenReturn(manager);
+ FrontendServiceImpl service = new FrontendServiceImpl(Mockito.mock(ExecuteEnv.class));
+
+ try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ TStatus status = service.reportLanceIndexJobResult(matchingReport());
+ Assertions.assertEquals(TStatusCode.NOT_MASTER, status.getStatusCode());
+ }
+ Mockito.verifyNoInteractions(manager);
+ }
+
+ @Test
+ public void masterDelegatesTheIdentityQuadAndTypedResultVerbatim() throws Exception {
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.isMaster()).thenReturn(true);
+ LanceIndexJobManager manager = Mockito.mock(LanceIndexJobManager.class);
+ Mockito.when(env.getLanceIndexJobManager()).thenReturn(manager);
+ FrontendServiceImpl service = new FrontendServiceImpl(Mockito.mock(ExecuteEnv.class));
+
+ try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ TStatus status = service.reportLanceIndexJobResult(matchingReport()
+ .setResultCode(TLanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT)
+ .setSanitizedMessage("commit conflict on the provider")
+ .setExternalMetadataAdvanced(true));
+ Assertions.assertEquals(TStatusCode.OK, status.getStatusCode());
+ }
+
+ ArgumentCaptor captor = ArgumentCaptor.forClass(LanceIndexJobResult.class);
+ Mockito.verify(manager).completeWithResult(Mockito.eq(JOB_ID), Mockito.eq(DISPATCH_REVISION),
+ Mockito.eq(INVOCATION_ID), Mockito.eq(BE_EPOCH), captor.capture());
+ LanceIndexJobResult result = captor.getValue();
+ Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT, result.getResultCode());
+ Assertions.assertEquals("commit conflict on the provider", result.getSanitizedMessage());
+ Assertions.assertTrue(result.isExternalMetadataAdvanced());
+ }
+
+ @Test
+ public void masterRecordsAChildReapedProofWithTheDurableBackendId() throws Exception {
+ Env env = Mockito.mock(Env.class);
+ Mockito.when(env.isMaster()).thenReturn(true);
+ LanceIndexJobManager manager = Mockito.mock(LanceIndexJobManager.class);
+ Mockito.when(env.getLanceIndexJobManager()).thenReturn(manager);
+ LanceIndexJob dispatched = Mockito.mock(LanceIndexJob.class);
+ Mockito.when(dispatched.getBackendId()).thenReturn(BACKEND_ID);
+ Mockito.when(manager.getJob(JOB_ID)).thenReturn(dispatched);
+ FrontendServiceImpl service = new FrontendServiceImpl(Mockito.mock(ExecuteEnv.class));
+
+ try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ TStatus status = service.reportLanceIndexJobResult(matchingReport()
+ .setTerminationProof(TLanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertEquals(TStatusCode.OK, status.getStatusCode());
+ }
+
+ Mockito.verify(manager).completeWithResult(Mockito.eq(JOB_ID), Mockito.eq(DISPATCH_REVISION),
+ Mockito.eq(INVOCATION_ID), Mockito.eq(BE_EPOCH), Mockito.any());
+ Mockito.verify(manager).recordTerminationProof(Mockito.eq(JOB_ID), Mockito.eq(DISPATCH_REVISION),
+ Mockito.eq(BACKEND_ID), Mockito.eq(BE_EPOCH), Mockito.eq(INVOCATION_ID),
+ Mockito.eq(LanceIndexTerminationProof.CHILD_REAPED));
+ }
+
+ private static TLanceIndexJobReport matchingReport() {
+ return new TLanceIndexJobReport()
+ .setJobId(JOB_ID)
+ .setDispatchRevision(DISPATCH_REVISION)
+ .setInvocationId(INVOCATION_ID)
+ .setBeProcessEpoch(BE_EPOCH)
+ .setResultCode(TLanceIndexJobResultCode.NATIVE_OK);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
index 5c20a2b64f6b90..dbe8b06584ebde 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java
@@ -51,6 +51,7 @@
import org.apache.doris.thrift.THeartbeatResult;
import org.apache.doris.thrift.TIngestBinlogRequest;
import org.apache.doris.thrift.TIngestBinlogResult;
+import org.apache.doris.thrift.TLanceIndexJobDispatch;
import org.apache.doris.thrift.TMasterInfo;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPublishTopicRequest;
@@ -160,6 +161,13 @@ public void setBackendInFe(Backend backendInFe) {
this.backendInFe = backendInFe;
}
+ // One-shot Lance index mutation dispatch: the default mock only acknowledges
+ // enqueue; fault-injecting tests override or drive the FE handler directly.
+ @Override
+ public TStatus submitLanceIndexJob(TLanceIndexJobDispatch dispatch) {
+ return new TStatus(TStatusCode.OK);
+ }
+
public abstract void init();
}
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index 9973d6fa1f890f..ffbcb882ebf8cd 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -653,3 +653,39 @@ struct TAgentPublishRequest {
1: required TAgentServiceVersion protocol_version
2: required list updates
}
+
+// Mutation intent of a one-shot Lance index job, mirroring the durable FE job record.
+enum TLanceIndexMutationType {
+ CREATE = 1,
+ REPLACE = 2,
+ DROP = 3
+}
+
+// One-shot Lance index mutation dispatch. The master FE records the durable RUNNING
+// state (invocation id, BE process epoch, deadline) BEFORE sending this request, and
+// sends it at most once per invocation id to one selected BE. The BE handler only
+// enqueues it into a bounded supervisor queue and answers immediately: OK means the
+// request was enqueued exactly once; an ERROR status means it was NOT enqueued and
+// this invocation id will never be executed. Execution and results are reported only
+// via FrontendService.reportLanceIndexJobResult.
+struct TLanceIndexJobDispatch {
+ 1: required i64 job_id
+ 2: required i64 dispatch_revision
+ 3: required string invocation_id
+ 4: required i64 be_process_epoch
+ 5: required i64 deadline_ms
+ 6: required TLanceIndexMutationType mutation_type
+ 7: required string index_name
+ 8: required string column_name
+ 9: required string index_type
+ 10: optional string properties_json
+ 11: optional bool if_not_exists
+ 12: optional bool if_exists
+ 13: required string dataset_uri
+ 14: required i64 admitted_dataset_version
+ 15: required string schema_contract_json
+ // Lance-native storage options, handed to the worker untranslated, resolved from
+ // current catalog properties at send time. Same provider-opaque contract as
+ // TLanceScanParams.lance_storage_options. Never persisted, logged, or echoed back.
+ 16: optional map storage_options
+}
diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift
index 58bd4294d97f5b..3dffacc26a31fd 100644
--- a/gensrc/thrift/BackendService.thrift
+++ b/gensrc/thrift/BackendService.thrift
@@ -399,6 +399,11 @@ struct TPythonPackageInfo {
service BackendService {
AgentService.TAgentResult submit_tasks(1:list tasks);
+ // Enqueue one one-shot Lance index mutation dispatch (see TLanceIndexJobDispatch).
+ // OK means enqueued exactly once; an ERROR status means NOT enqueued and never
+ // executed for this invocation id. Result arrives via reportLanceIndexJobResult.
+ Status.TStatus submit_lance_index_job(1:AgentService.TLanceIndexJobDispatch dispatch);
+
AgentService.TAgentResult make_snapshot(1:AgentService.TSnapshotRequest snapshot_request);
AgentService.TAgentResult release_snapshot(1:string snapshot_path);
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 2fe8e3e1ffe887..c0563ad93f2688 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1823,6 +1823,9 @@ service FrontendService {
TReportExecStatusResult reportExecStatus(1: TReportExecStatusParams params)
MasterService.TMasterResult finishTask(1: MasterService.TFinishTaskRequest request)
+ // Report the typed result envelope of one Lance index mutation invocation.
+ // Stale or identity-mismatched reports are logged and dropped on the FE side.
+ Status.TStatus reportLanceIndexJobResult(1: MasterService.TLanceIndexJobReport report)
MasterService.TMasterResult report(1: MasterService.TReportRequest request)
// Deprecated
MasterService.TFetchResourceResult fetchResource()
diff --git a/gensrc/thrift/MasterService.thrift b/gensrc/thrift/MasterService.thrift
index 2caf3c7696512d..3d22e86b6b1015 100644
--- a/gensrc/thrift/MasterService.thrift
+++ b/gensrc/thrift/MasterService.thrift
@@ -164,3 +164,56 @@ struct TFetchResourceResult {
2: required i64 resourceVersion
3: required map resourceByUser
}
+
+// Typed result code of one Lance index mutation invocation. PRE_INVOCATION_* codes are
+// complete trusted rejections before the native call (dataset version / schema
+// contract / credential / resource revalidation) and prove NOT_COMMITTED. NATIVE_*
+// codes are the saved Lance error code of the single native invocation, read before
+// the consuming error message. NO_TRUSTED_RESULT is produced FE-side only and never
+// appears on the wire.
+enum TLanceIndexJobResultCode {
+ PRE_INVOCATION_STALE_ADMISSION = 1,
+ PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT = 2,
+ PRE_INVOCATION_CREDENTIAL_EXPIRED = 3,
+ PRE_INVOCATION_RESOURCE_REJECTED = 4,
+ NATIVE_OK = 5,
+ NATIVE_COMMIT_CONFLICT = 6,
+ NATIVE_NOT_FOUND = 7,
+ NATIVE_INVALID_ARGUMENT = 8,
+ NATIVE_NOT_SUPPORTED = 9,
+ NATIVE_INDEX = 10,
+ NATIVE_IO = 11,
+ NATIVE_INTERNAL = 12
+}
+
+enum TLanceIndexCompletionReason {
+ NONE = 1,
+ IF_CONDITION_NOOP = 2
+}
+
+// Possible-live termination proof carried by the result envelope. BE_PROCESS_EPOCH_GONE
+// is derived FE-side from heartbeat epochs and never appears on the wire.
+enum TLanceIndexTerminationProof {
+ NONE = 1,
+ CHILD_REAPED = 2
+}
+
+// Typed result envelope of one Lance index mutation invocation, reported by the BE
+// supervisor to the master FE. The envelope carries only what is needed to classify
+// the single invocation: the matching invocation identity and BE process epoch, the
+// typed result code, a bounded sanitized message, and the matching child-reap proof
+// when available. Only a complete identity-matched envelope proves COMMITTED or
+// NOT_COMMITTED; EOF, signal, timeout, OOM, BE loss, malformed/partial protocol, or
+// identity mismatch after acceptance yields UNKNOWN on the FE side. Stale or
+// identity-mismatched reports are logged and dropped.
+struct TLanceIndexJobReport {
+ 1: required i64 job_id
+ 2: required i64 dispatch_revision
+ 3: required string invocation_id
+ 4: required i64 be_process_epoch
+ 5: required TLanceIndexJobResultCode result_code
+ 6: optional TLanceIndexCompletionReason completion_reason
+ 7: optional string sanitized_message
+ 8: optional bool external_metadata_advanced
+ 9: optional TLanceIndexTerminationProof termination_proof
+}
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_index_admission.groovy b/regression-test/suites/external_table_p0/lance/test_lance_index_admission.groovy
index 40358f4b5f27de..22ce5977a66890 100644
--- a/regression-test/suites/external_table_p0/lance/test_lance_index_admission.groovy
+++ b/regression-test/suites/external_table_p0/lance/test_lance_index_admission.groovy
@@ -27,12 +27,13 @@ suite("test_lance_index_admission", "p0,external,nonConcurrent") {
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
String lanceRestPort = context.config.otherConfigs.get("lance_rest_port")
- // Admitted jobs are durable and stay PENDING forever in this delivery slice: dispatch,
- // FORCE_RELEASE and job GC only land in later slices, so their fences and quota charges
- // can never be released here. Every index name (and the filesystem catalog itself,
- // because fence/quota keys include the persisted catalog id) carries this per-run suffix
- // so that rerunning the suite on a shared pipeline cluster can never collide with a
- // previous run's leftovers.
+ // Admitted jobs are durable and stay PENDING in this delivery slice: no worker
+ // exists to execute a dispatched job and the dispatcher is pinned silent below,
+ // while FORCE_RELEASE and job GC only land in later slices, so their fences and
+ // quota charges can never be released here. Every index name (and the filesystem
+ // catalog itself, because fence/quota keys include the persisted catalog id) carries
+ // this per-run suffix so that rerunning the suite on a shared pipeline cluster can
+ // never collide with a previous run's leftovers.
String runSuffix = "${System.currentTimeMillis()}"
String filesystemCatalog = "test_lance_index_admission_${runSuffix}"
String restCatalog = "test_lance_index_admission_rest"
@@ -55,7 +56,7 @@ suite("test_lance_index_admission", "p0,external,nonConcurrent") {
sql """DROP CATALOG IF EXISTS `${restCatalog}`"""
try_sql "DROP USER '${user}'@'%'"
- // Both settings are masterOnly. Read them on the master even if the suite's
+ // All three settings are masterOnly. Read them on the master even if the suite's
// ordinary JDBC connection points at a follower. SHOW uses the experimental
// display name for the gate, while ADMIN SET accepts its unprefixed alias.
def gateRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'experimental_enable_lance_index_mutation'"""
@@ -64,6 +65,12 @@ suite("test_lance_index_admission", "p0,external,nonConcurrent") {
assertEquals(1, quotaRows.size())
String originalGate = gateRows[0][1].toString()
String originalQuota = quotaRows[0][1].toString()
+ def dispatchIntervalRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'lance_index_job_dispatch_interval_second'"""
+ assertEquals(1, dispatchIntervalRows.size())
+ String originalDispatchInterval = dispatchIntervalRows[0][1].toString()
+ // The interval pin below outlives the daemon's in-flight wait on this shipped
+ // default, so the wait is sized correctly.
+ assertEquals("10", originalDispatchInterval)
// The main scenario admits two jobs on one table, independently of the
// cluster's original quota. The dedicated quota case temporarily lowers it.
String suiteQuota = Math.max(2L, originalQuota.toLong()).toString()
@@ -71,6 +78,15 @@ suite("test_lance_index_admission", "p0,external,nonConcurrent") {
try {
master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_max_unresolved_per_table" = "${suiteQuota}")"""
+ // Pin the dispatcher's polling interval to one hour so no dispatch round can
+ // fire between admission and the PENDING assertions below: this slice's
+ // backends answer submit_lance_index_job with a clean not-implemented error,
+ // which converges a dispatched job to NOT_COMMITTED and would break this
+ // suite's PENDING premise. A cycle already sleeping on the shipped interval
+ // can still wake once more within that interval; outliving it guarantees the
+ // pin is in full effect before the first job is admitted.
+ master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_dispatch_interval_second" = "3600")"""
+ sleep((originalDispatchInterval.toLong() + 1) * 1000L)
// Open the mutation gate for this suite only. masterOnly configs set through
// ADMIN SET land on the master node locally, which is where admission reads them;
// the finally block below restores the gate no matter where the suite fails (T4).
@@ -249,6 +265,7 @@ suite("test_lance_index_admission", "p0,external,nonConcurrent") {
[
{ master_sql """ADMIN SET FRONTEND CONFIG ("enable_lance_index_mutation" = "${originalGate}")""" },
{ master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_max_unresolved_per_table" = "${originalQuota}")""" },
+ { master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_dispatch_interval_second" = "${originalDispatchInterval}")""" },
{ sql "DROP USER IF EXISTS '${user}'@'%'" },
{ sql """DROP CATALOG IF EXISTS `${restCatalog}`""" }
].each { cleanup ->
diff --git a/regression-test/suites/external_table_p0/lance/test_lance_index_dispatch.groovy b/regression-test/suites/external_table_p0/lance/test_lance_index_dispatch.groovy
new file mode 100644
index 00000000000000..39223285495fe5
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lance/test_lance_index_dispatch.groovy
@@ -0,0 +1,243 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// PR3D regression scope: no backend worker exists in this delivery slice to consume
+// a dispatched Lance index job request (the isolated worker lands in a later slice),
+// so a cluster cannot exercise positive dispatch, the result callback, or the
+// terminal-job refresh driver end to end, and every case in this suite is negative
+// or static: the six new dispatcher configs are smoked through SHOW/SET (validator
+// rejections plus a boolean two-state round trip), and, with the dispatcher's
+// polling interval pinned to one hour, an admitted job's user-visible row is proven
+// frozen for the suite's window (the daemon exists but is disabled by configuration
+// from dispatching). The mutation gate is opened only to admit that one job; both
+// the gate and the interval are restored in the finally block. Deliberately NOT
+// covered here (all wait for the worker slice / fake-worker UT): dispatch and
+// callback e2e, genuine UNKNOWN creation, and the G2/G4 evidence.
+
+suite("test_lance_index_dispatch", "p0,external,nonConcurrent") {
+ // The Lance fixture is preinstalled in the MinIO container of the Iceberg
+ // external environment, so this suite deliberately shares its switch.
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Lance index dispatch test because the Iceberg MinIO environment is disabled.")
+ return
+ }
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ // Admitted jobs are durable and stay PENDING in this delivery slice: no worker
+ // exists to execute a dispatched job and the dispatcher is pinned silent below,
+ // while FORCE_RELEASE and job GC only land in later slices, so their fences and
+ // quota charges can never be released here.
+ // Every index name (and the filesystem catalog itself, because fence/quota keys
+ // include the persisted catalog id) carries this per-run suffix so that rerunning
+ // the suite on a shared pipeline cluster can never collide with a previous run's
+ // leftovers.
+ String runSuffix = "${System.currentTimeMillis()}"
+ String filesystemCatalog = "test_lance_index_dispatch_${runSuffix}"
+ String tableName = "vs_ivf_pq_f32"
+ String dispatchIndexName = "idx_dispatch_${runSuffix}"
+
+ // All seven settings are masterOnly. Read them on the master even if the suite's
+ // ordinary JDBC connection points at a follower. SHOW uses the experimental
+ // display name for the gate, while ADMIN SET accepts its unprefixed alias.
+ def gateRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'experimental_enable_lance_index_mutation'"""
+ def intervalRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'lance_index_job_dispatch_interval_second'"""
+ def deadlineRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'lance_index_job_execute_deadline_second'"""
+ def roundRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'lance_index_job_max_dispatch_per_round'"""
+ def inflightRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'lance_index_job_max_inflight_per_backend'"""
+ def retryRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'lance_index_job_refresh_retry_second'"""
+ def localFileRows = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'enable_lance_index_local_file_mutation'"""
+ assertEquals(1, gateRows.size())
+ assertEquals(1, intervalRows.size())
+ assertEquals(1, deadlineRows.size())
+ assertEquals(1, roundRows.size())
+ assertEquals(1, inflightRows.size())
+ assertEquals(1, retryRows.size())
+ assertEquals(1, localFileRows.size())
+ String originalGate = gateRows[0][1].toString()
+ String originalInterval = intervalRows[0][1].toString()
+ String originalDeadline = deadlineRows[0][1].toString()
+ String originalRound = roundRows[0][1].toString()
+ String originalInflight = inflightRows[0][1].toString()
+ String originalRetry = retryRows[0][1].toString()
+ String originalLocalFile = localFileRows[0][1].toString()
+ // The six dispatcher configs ship with these documented defaults; asserting them
+ // here fails loudly if a shared cluster has drifted instead of silently
+ // restoring a non-default value afterwards. The derived wait below also relies
+ // on the interval really being the shipped 10 seconds when the pin is applied.
+ assertEquals("10", originalInterval)
+ assertEquals("3600", originalDeadline)
+ assertEquals("16", originalRound)
+ assertEquals("2", originalInflight)
+ assertEquals("300", originalRetry)
+ assertEquals("false", originalLocalFile)
+ long dispatchIntervalSecond = originalInterval.toLong()
+ Throwable suiteFailure = null
+
+ // test { ... exception } always runs on the suite's default connection; the
+ // masterOnly config rejections below must be asserted on the master connection.
+ def expectMasterSqlException = { String stmt, String substring ->
+ String caught = null
+ try {
+ master_sql(stmt)
+ } catch (Throwable t) {
+ caught = t.toString()
+ }
+ assertTrue(caught != null && caught.contains(substring))
+ }
+
+ try {
+ // Dispatcher config smoke: the five numeric items are guarded by the
+ // positive-int/positive-long callback, whose rejection message carries the
+ // field name and the offending value. Zero and negative values are rejected
+ // before assignment, so none of these sets can take effect.
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_dispatch_interval_second" = "0")""",
+ "must be a positive int")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_dispatch_interval_second" = "-1")""",
+ "must be a positive int")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_execute_deadline_second" = "0")""",
+ "must be a positive long")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_execute_deadline_second" = "-1")""",
+ "must be a positive long")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_max_dispatch_per_round" = "0")""",
+ "must be a positive int")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_max_dispatch_per_round" = "-1")""",
+ "must be a positive int")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_max_inflight_per_backend" = "0")""",
+ "must be a positive int")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_max_inflight_per_backend" = "-1")""",
+ "must be a positive int")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_refresh_retry_second" = "0")""",
+ "must be a positive int")
+ expectMasterSqlException("""ADMIN SET FRONTEND CONFIG ("lance_index_job_refresh_retry_second" = "-1")""",
+ "must be a positive int")
+
+ // A positive value is accepted and visible immediately on the master.
+ master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_max_dispatch_per_round" = "17")"""
+ def roundRowsAfterSet = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'lance_index_job_max_dispatch_per_round'"""
+ assertEquals("17", roundRowsAfterSet[0][1].toString())
+ master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_max_dispatch_per_round" = "${originalRound}")"""
+
+ // The file:// mutation assertion is a plain boolean with no validator
+ // callback: both of its states are settable and immediately visible.
+ master_sql """ADMIN SET FRONTEND CONFIG ("enable_lance_index_local_file_mutation" = "true")"""
+ def localFileRowsTrue = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'enable_lance_index_local_file_mutation'"""
+ assertEquals("true", localFileRowsTrue[0][1].toString())
+ master_sql """ADMIN SET FRONTEND CONFIG ("enable_lance_index_local_file_mutation" = "false")"""
+ def localFileRowsFalse = master_sql """ADMIN SHOW FRONTEND CONFIG LIKE 'enable_lance_index_local_file_mutation'"""
+ assertEquals("false", localFileRowsFalse[0][1].toString())
+ // The original value (asserted to be the shipped "false" above) is restored
+ // again defensively by the finally block.
+
+ // The dispatcher daemon polls every lance_index_job_dispatch_interval_second
+ // regardless of the mutation gate (durable jobs must be driven even with
+ // admission closed), and in this slice a round really reaches the backends:
+ // their handler answers submit_lance_index_job with a clean not-implemented
+ // error, which converges the job to NOT_COMMITTED
+ // (PRE_INVOCATION_RESOURCE_REJECTED). An unpinned round would therefore
+ // legitimately advance the admitted job, so the state-stability case below
+ // pins the interval to one hour first; a cycle already sleeping on the
+ // shipped interval can still wake once more within that interval, and
+ // outliving it here guarantees zero rounds for the rest of the suite.
+ master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_dispatch_interval_second" = "3600")"""
+ sleep((dispatchIntervalSecond + 1) * 1000L)
+
+ master_sql """ADMIN SET FRONTEND CONFIG ("enable_lance_index_mutation" = "true")"""
+
+ sql """
+ CREATE CATALOG `${filesystemCatalog}` PROPERTIES (
+ "type" = "lance",
+ "lance.catalog.type" = "filesystem",
+ "warehouse" = "s3://warehouse/lance",
+ "s3.endpoint" = "http://${externalEnvIp}:${minioPort}",
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.region" = "us-east-1",
+ "use_path_style" = "true"
+ )
+ """
+
+ // CREATE INDEX is admitted and returns a single-column JobId result set with
+ // one row; the job is visible as PENDING right after admission.
+ def createRows = sql """CREATE INDEX `${dispatchIndexName}` ON `${filesystemCatalog}`.`doris`.`${tableName}` (embedding) USING ANN
+ PROPERTIES("index_type"="IVF_PQ", "metric"="l2", "num_partitions"="256", "num_sub_vectors"="16")"""
+ assertEquals(1, createRows.size())
+ assertEquals(1, createRows[0].size())
+ String createJobId = createRows[0][0].toString()
+
+ def jobsBeforeWait = sql_return_maparray """SHOW LANCE INDEX JOBS FROM `${filesystemCatalog}`.`doris`
+ WHERE TableName = "${tableName}" """
+ def jobRowBeforeWait = jobsBeforeWait.find { it.IndexName == dispatchIndexName }
+ assertTrue(jobRowBeforeWait != null)
+ assertEquals(createJobId, jobRowBeforeWait.JobId.toString())
+ assertEquals("PENDING", jobRowBeforeWait.State.toString())
+ // A never-dispatched job holds no possible-live worker slot and has no
+ // refresh obligation yet; both must stay that way across the wait below.
+ assertEquals("NO", jobRowBeforeWait.PossibleLive.toString())
+
+ // A short static wait inside the pinned window: with the daemon disabled by
+ // configuration no round can dispatch the job, so the visible row must not
+ // move.
+ sleep(5000)
+
+ def jobsAfterWait = sql_return_maparray """SHOW LANCE INDEX JOBS FROM `${filesystemCatalog}`.`doris`
+ WHERE TableName = "${tableName}" """
+ def jobRowAfterWait = jobsAfterWait.find { it.IndexName == dispatchIndexName }
+ assertTrue(jobRowAfterWait != null)
+ assertEquals("PENDING", jobRowAfterWait.State.toString())
+ assertEquals("NO", jobRowAfterWait.PossibleLive.toString())
+ // No lifecycle column moved between the two reads: with the dispatcher
+ // pinned silent no round can dispatch the job, and without a worker nothing
+ // else can advance it.
+ ["JobId", "CatalogName", "DbName", "TableName", "IndexName", "Operation",
+ "State", "RefreshState", "PossibleLive"].each { column ->
+ assertEquals(jobRowBeforeWait[column].toString(), jobRowAfterWait[column].toString())
+ }
+ } catch (Throwable failure) {
+ suiteFailure = failure
+ throw failure
+ } finally {
+ // Attempt every cleanup, but never report success after a failed restore.
+ // Preserve the scenario failure and attach cleanup failures to it.
+ Throwable cleanupFailure = suiteFailure
+ [
+ { master_sql """ADMIN SET FRONTEND CONFIG ("enable_lance_index_mutation" = "${originalGate}")""" },
+ { master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_dispatch_interval_second" = "${originalInterval}")""" },
+ { master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_execute_deadline_second" = "${originalDeadline}")""" },
+ { master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_max_dispatch_per_round" = "${originalRound}")""" },
+ { master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_max_inflight_per_backend" = "${originalInflight}")""" },
+ { master_sql """ADMIN SET FRONTEND CONFIG ("lance_index_job_refresh_retry_second" = "${originalRetry}")""" },
+ { master_sql """ADMIN SET FRONTEND CONFIG ("enable_lance_index_local_file_mutation" = "${originalLocalFile}")""" }
+ ].each { cleanup ->
+ try {
+ cleanup()
+ } catch (Throwable failure) {
+ if (cleanupFailure == null) {
+ cleanupFailure = failure
+ } else {
+ cleanupFailure.addSuppressed(failure)
+ }
+ }
+ }
+ if (suiteFailure == null && cleanupFailure != null) {
+ throw cleanupFailure
+ }
+ // The filesystem catalog stays behind: the admitted job remains unresolved
+ // and guards DROP CATALOG until FORCE_RELEASE lands in a later slice.
+ }
+}