From 0fbef865d599418bfb216d61cd0624b061ef9e1c Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Tue, 28 Apr 2026 16:28:54 +0800 Subject: [PATCH 01/20] [server] Respect the option of 'table.log.ttl' for local segments --- .../apache/fluss/server/log/LogTablet.java | 24 ++++-- .../log/remote/TieredLocalSegmentTtlTest.java | 84 +++++++++++++++++++ 2 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 150979d4c8..942e596078 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -107,6 +107,7 @@ public final class LogTablet { private volatile int tieredLogLocalSegments; private final Clock clock; private final boolean isChangeLog; + private final long logTtlMs; @GuardedBy("lock") private volatile LogOffsetMetadata highWatermarkMetadata; @@ -156,6 +157,7 @@ private LogTablet( (int) conf.get(ConfigOptions.WRITER_ID_EXPIRATION_CHECK_INTERVAL).toMillis(); this.writerStateManager = writerStateManager; this.highWatermarkMetadata = new LogOffsetMetadata(0L); + this.logTtlMs = conf.get(ConfigOptions.TABLE_LOG_TTL).toMillis(); this.scheduler = scheduler; // scheduler the writer expiration interval check. @@ -1269,7 +1271,7 @@ private void deleteOldSegments(long endOffset, SegmentDeletionReason reason) } /** Returns the segments that can be deleted by checking log end offset. */ - private List deletableSegments(long endOffset) { + private List deletableSegments(long endOffset) throws IOException { if (localLog.getSegments().isEmpty()) { return Collections.emptyList(); } @@ -1278,17 +1280,27 @@ private List deletableSegments(long endOffset) { // readers is in progress. List deletableSegments = new ArrayList<>(); List logSegments = localLog.getSegments().values(); - // ignore the segments configured to be retained - for (int i = 0; i < logSegments.size() - tieredLogLocalSegments; i++) { - if (logSegments.get(i + 1).getBaseOffset() <= endOffset) { - deletableSegments.add(logSegments.get(i)); - } else { + int tierProtectedStartIndex = logSegments.size() - tieredLogLocalSegments; + long now = clock.milliseconds(); + + for (int i = 0; i < logSegments.size() - 1; i++) { + if (logSegments.get(i + 1).getBaseOffset() > endOffset) { break; } + if (i < tierProtectedStartIndex || isSegmentExpired(now, logSegments.get(i))) { + deletableSegments.add(logSegments.get(i)); + } } return deletableSegments; } + private boolean isSegmentExpired(long now, LogSegment segment) throws IOException { + if (logTtlMs <= 0L) { + return false; + } + return now - segment.maxTimestampSoFar() > logTtlMs; + } + private void deleteSegments(List deletableSegments, SegmentDeletionReason reason) throws IOException { localLog.checkIfMemoryMappedBufferClosed(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java new file mode 100644 index 0000000000..3c409569dd --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log.remote; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.server.log.LogTablet; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; + +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH_PK; +import static org.apache.fluss.server.zk.data.LeaderAndIsr.INITIAL_LEADER_EPOCH; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies that among the last {@code table.log.tiered.local-segments} local segments, the inactive + * ones can still be removed once {@link ConfigOptions#TABLE_LOG_TTL} has passed (while the default + * long TTL would keep them until offset-based cleanup allows it). + */ +final class TieredLocalSegmentTtlTest extends RemoteLogTestBase { + + @Override + public Configuration getServerConf() { + Configuration conf = super.getServerConf(); + conf.set(ConfigOptions.TABLE_LOG_TTL, Duration.ofMillis(50)); + return conf; + } + + @BeforeEach + public void setup() throws Exception { + super.setup(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throws Exception { + TableBucket tb = + partitionTable + ? new TableBucket(DATA1_TABLE_ID, 0L, 0) + : new TableBucket(DATA1_TABLE_ID, 0); + + makeKvTableAsLeader(tb, DATA1_TABLE_PATH_PK, INITIAL_LEADER_EPOCH, partitionTable); + LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); + + addMultiSegmentsToLogTablet(logTablet, 5); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + logTablet.updateRemoteLogEndOffset(40L); + assertThat(logTablet.getSegments()).hasSize(5); + + logTablet.updateMinRetainOffset(33L); + assertThat(logTablet.getSegments()).hasSize(2); + + // Below the inactive tier-retained segment's TTL: cleanup runs but segment stays. + logTablet.updateMinRetainOffset(34L); + assertThat(logTablet.getSegments()).hasSize(2); + + manualClock.advanceTime(Duration.ofMillis(200)); + + logTablet.updateMinRetainOffset(35L); + assertThat(logTablet.getSegments()).hasSize(1); + } +} From 34ee08128a486a87b65e29d014390597f717e506 Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Wed, 29 Apr 2026 11:50:46 +0800 Subject: [PATCH 02/20] optimize --- .../apache/fluss/server/log/LogTablet.java | 26 ++++++ .../server/log/remote/LogTieringTask.java | 1 + .../server/log/remote/RemoteLogTTLTest.java | 67 +++++++++++++-- .../log/remote/TieredLocalSegmentTtlTest.java | 84 ------------------- 4 files changed, 85 insertions(+), 93 deletions(-) delete mode 100644 fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 942e596078..78d700b917 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -1014,6 +1014,32 @@ public void roll(Optional expectedNextOffset) throws Exception { } } + /** + * Rolls the active segment if it is non-empty and all of its records have passed the log TTL. + * + *

This keeps an empty active segment available for future appends while making the expired + * segment eligible for remote tiering and local cleanup. + */ + public void rollActiveSegmentIfExpired() throws Exception { + if (logTtlMs <= 0L) { + return; + } + + synchronized (lock) { + LogSegment activeSegment = localLog.getSegments().activeSegment(); + if (activeSegment.getSizeInBytes() == 0 + || !isSegmentExpired(clock.milliseconds(), activeSegment)) { + return; + } + + LOG.info( + "Rolling expired active log segment {} for bucket {}.", + activeSegment, + getTableBucket()); + roll(Optional.empty()); + } + } + /** Truncate this log so that it ends with the greatest offset < targetOffset. */ boolean truncateTo(long targetOffset) throws LogStorageException { if (targetOffset < 0) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java index c9cb32215e..3ea77ca6c5 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java @@ -128,6 +128,7 @@ private void runOnce() throws InterruptedException { TableMetricGroup metricGroup = replica.tableMetrics(); maybeUpdateCopiedOffset(logTablet); + logTablet.rollActiveSegmentIfExpired(); // Get these candidate log segments to copy and these expired remote log segments to // clean up. List candidateToCopySegments = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java index d23667431f..216c59d883 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java @@ -25,6 +25,7 @@ import org.apache.fluss.server.log.LogTablet; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -33,7 +34,9 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; +import static org.apache.fluss.record.TestData.DATA1; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsWithWriterId; import static org.assertj.core.api.Assertions.assertThat; /** Test for remote log ttl in {@link RemoteLogManager}. */ @@ -72,11 +75,13 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { // advance time past TTL (7 days) manualClock.advanceTime(Duration.ofDays(7).plusHours(1)); - // since data lake is enabled and no data has been tiered to data lake, - // the expired segments should not be deleted. + // Since data lake is enabled and no data has been tiered to data lake, + // the expired segments should not be deleted. The expired active segment is rolled + // and uploaded in this task run. remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); - assertThat(remoteLog.allRemoteLogSegments()).hasSize(4); + assertThat(remoteLog.allRemoteLogSegments()).hasSize(5); assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(0L); + assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(50L); // set lake log end offset to 20, meaning only the first 2 segments // ([0,10) and [10,20)) have been tiered to lake @@ -86,10 +91,10 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); // only segments with remoteLogEndOffset <= 20 should be deleted (first 2 segments) - // remaining segments: [20,30) and [30,40) - assertThat(remoteLog.allRemoteLogSegments()).hasSize(2); + // remaining segments: [20,30), [30,40) and [40,50) + assertThat(remoteLog.allRemoteLogSegments()).hasSize(3); assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(20L); - assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(40L); + assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(50L); // verify remaining segments have the expected offsets assertThat(remoteLog.allRemoteLogSegments()) .allSatisfy( @@ -99,7 +104,14 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { // now advance lake log end offset to include all remaining segments logTablet.updateLakeLogEndOffset(40L); - // trigger again, remaining expired segments should now be deleted + // trigger again, segments whose end offset is <= 40 should now be deleted + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + assertThat(remoteLog.allRemoteLogSegments()).hasSize(1); + assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(40L); + assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(50L); + + logTablet.updateLakeLogEndOffset(50L); + // trigger again, all remaining expired segments should now be deleted remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); assertThat(remoteLog.allRemoteLogSegments()).isEmpty(); assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(Long.MAX_VALUE); @@ -107,8 +119,8 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { // Fetch records from remote. // mock to update remote log end offset and remote log start offset as // NotifyRemoteLogOffsetsRequest do. - logTablet.updateRemoteLogStartOffset(40L); - logTablet.updateRemoteLogEndOffset(40L); + logTablet.updateRemoteLogStartOffset(50L); + logTablet.updateRemoteLogEndOffset(50L); CompletableFuture> future = new CompletableFuture<>(); replicaManager.fetchLogRecords( @@ -122,4 +134,41 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { assertThat(resultForBucket.getErrorCode()) .isEqualTo(Errors.LOG_OFFSET_OUT_OF_RANGE_EXCEPTION.code()); } + + @Test + void testExpiredActiveSegmentRolledUploadedAndLocallyDeleted() throws Exception { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tb, false); + LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); + + addMultiSegmentsToLogTablet(logTablet, 1); + assertThat(logTablet.getSegments()).hasSize(1); + assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); + + manualClock.advanceTime(Duration.ofDays(7).plusHours(1)); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogTablet remoteLog = remoteLogManager.remoteLogTablet(tb); + assertThat(remoteLog.allRemoteLogSegments()).hasSize(1); + assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(0L); + assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(10L); + assertThat(logTablet.getSegments()).hasSize(1); + assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); + assertThat(logTablet.activeLogSegment().getSizeInBytes()).isZero(); + + logTablet.appendAsLeader( + genMemoryLogRecordsWithWriterId( + Collections.singletonList(DATA1.get(0)), + manualClock.milliseconds(), + 0, + 10L)); + + assertThat(logTablet.localLogEndOffset()).isEqualTo(11L); + assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); + assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); + + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + assertThat(remoteLog.allRemoteLogSegments()).isEmpty(); + assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(Long.MAX_VALUE); + } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java deleted file mode 100644 index 3c409569dd..0000000000 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.log.remote; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.server.log.LogTablet; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import java.time.Duration; - -import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; -import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH_PK; -import static org.apache.fluss.server.zk.data.LeaderAndIsr.INITIAL_LEADER_EPOCH; -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Verifies that among the last {@code table.log.tiered.local-segments} local segments, the inactive - * ones can still be removed once {@link ConfigOptions#TABLE_LOG_TTL} has passed (while the default - * long TTL would keep them until offset-based cleanup allows it). - */ -final class TieredLocalSegmentTtlTest extends RemoteLogTestBase { - - @Override - public Configuration getServerConf() { - Configuration conf = super.getServerConf(); - conf.set(ConfigOptions.TABLE_LOG_TTL, Duration.ofMillis(50)); - return conf; - } - - @BeforeEach - public void setup() throws Exception { - super.setup(); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throws Exception { - TableBucket tb = - partitionTable - ? new TableBucket(DATA1_TABLE_ID, 0L, 0) - : new TableBucket(DATA1_TABLE_ID, 0); - - makeKvTableAsLeader(tb, DATA1_TABLE_PATH_PK, INITIAL_LEADER_EPOCH, partitionTable); - LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); - - addMultiSegmentsToLogTablet(logTablet, 5); - remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); - - logTablet.updateRemoteLogEndOffset(40L); - assertThat(logTablet.getSegments()).hasSize(5); - - logTablet.updateMinRetainOffset(33L); - assertThat(logTablet.getSegments()).hasSize(2); - - // Below the inactive tier-retained segment's TTL: cleanup runs but segment stays. - logTablet.updateMinRetainOffset(34L); - assertThat(logTablet.getSegments()).hasSize(2); - - manualClock.advanceTime(Duration.ofMillis(200)); - - logTablet.updateMinRetainOffset(35L); - assertThat(logTablet.getSegments()).hasSize(1); - } -} From 75f9b2d6846c078cae4d333e9ced82737ddf9f4b Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Fri, 15 May 2026 15:41:23 +0800 Subject: [PATCH 03/20] ttlms from table config --- .../apache/fluss/server/log/LogManager.java | 23 ++++++++++- .../apache/fluss/server/log/LogTablet.java | 39 ++++++++++++++++++- .../apache/fluss/server/replica/Replica.java | 1 + 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index 8188719958..01d7abe9b2 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -17,10 +17,9 @@ package org.apache.fluss.server.log; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussException; +import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.LogStorageException; import org.apache.fluss.exception.PartitionNotExistException; @@ -280,6 +279,7 @@ private void waitForLoadLogsInDir(LogRecoveryTask recoveryTask) throws Throwable * @param tableBucket the table bucket * @param logFormat the log format * @param tieredLogLocalSegments the number of segments to retain in local for tiered log + * @param logTtlMs the log TTL in milliseconds from table configuration * @param isChangelog whether the log is a changelog of primary key table */ public LogTablet getOrCreateLog( @@ -288,6 +288,7 @@ public LogTablet getOrCreateLog( TableBucket tableBucket, LogFormat logFormat, int tieredLogLocalSegments, + long logTtlMs, boolean isChangelog) throws Exception { return inLock( @@ -310,6 +311,7 @@ public LogTablet getOrCreateLog( scheduler, logFormat, tieredLogLocalSegments, + logTtlMs, isChangelog, clock, true); @@ -324,6 +326,22 @@ public LogTablet getOrCreateLog( }); } + /** + * using the default {@link ConfigOptions#TABLE_LOG_TTL} when no table properties are + * configured. + */ + public LogTablet getOrCreateLog( + PhysicalTablePath tablePath, + TableBucket tableBucket, + LogFormat logFormat, + int tieredLogLocalSegments, + boolean isChangelog) + throws Exception { + long logTtlMs = new TableConfig(new Configuration()).getLogTTLMs(); + return getOrCreateLog( + tablePath, tableBucket, logFormat, tieredLogLocalSegments, logTtlMs, isChangelog); + } + public Optional getLog(TableBucket tableBucket) { return Optional.ofNullable(currentLogs.get(tableBucket)); } @@ -434,6 +452,7 @@ private LogTablet loadLog( scheduler, tableInfo.getTableConfig().getLogFormat(), tableInfo.getTableConfig().getTieredLogLocalSegments(), + tableInfo.getTableConfig().getLogTTLMs(), tableInfo.hasPrimaryKey(), clock, isCleanShutdown); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 78d700b917..3b7c858241 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -20,6 +20,7 @@ import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.CorruptRecordException; import org.apache.fluss.exception.DuplicateSequenceException; import org.apache.fluss.exception.FlussRuntimeException; @@ -146,6 +147,7 @@ private LogTablet( WriterStateManager writerStateManager, LogFormat logFormat, int tieredLogLocalSegments, + long logTtlMs, boolean isChangelog, Clock clock) { this.dataDir = dataDir; @@ -157,7 +159,7 @@ private LogTablet( (int) conf.get(ConfigOptions.WRITER_ID_EXPIRATION_CHECK_INTERVAL).toMillis(); this.writerStateManager = writerStateManager; this.highWatermarkMetadata = new LogOffsetMetadata(0L); - this.logTtlMs = conf.get(ConfigOptions.TABLE_LOG_TTL).toMillis(); + this.logTtlMs = logTtlMs; this.scheduler = scheduler; // scheduler the writer expiration interval check. @@ -345,6 +347,7 @@ public static LogTablet create( Scheduler scheduler, LogFormat logFormat, int tieredLogLocalSegments, + long logTtlMs, boolean isChangelog, Clock clock, boolean isCleanShutdown) @@ -393,10 +396,44 @@ public static LogTablet create( writerStateManager, logFormat, tieredLogLocalSegments, + logTtlMs, isChangelog, clock); } + /** + * Same as {@link #create(PhysicalTablePath, File, Configuration, TabletServerMetricGroup, long, + * Scheduler, LogFormat, int, long, boolean, Clock, boolean)} using the default {@link + * ConfigOptions#TABLE_LOG_TTL} when no table properties are configured. + */ + public static LogTablet create( + PhysicalTablePath tablePath, + File tabletDir, + Configuration conf, + TabletServerMetricGroup serverMetricGroup, + long recoveryPoint, + Scheduler scheduler, + LogFormat logFormat, + int tieredLogLocalSegments, + boolean isChangelog, + Clock clock, + boolean isCleanShutdown) + throws Exception { + return create( + tablePath, + tabletDir, + conf, + serverMetricGroup, + recoveryPoint, + scheduler, + logFormat, + tieredLogLocalSegments, + new TableConfig(new Configuration()).getLogTTLMs(), + isChangelog, + clock, + isCleanShutdown); + } + /** Register metrics for this log tablet in the metric group. */ public void registerMetrics(BucketMetricGroup bucketMetricGroup) { MetricGroup metricGroup = bucketMetricGroup.addGroup("log"); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 1b0407e813..f3d1ff76af 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -2162,6 +2162,7 @@ private LogTablet createLog( tableBucket, tableConfig.getLogFormat(), tableConfig.getTieredLogLocalSegments(), + tableConfig.getLogTTLMs(), isKvTable()); // update high watermark. Optional watermarkOpt = lazyHighWatermarkCheckpoint.fetch(tableBucket); From 5de2570ce135c1bee2633cb43f1c7806ae6812aa Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Fri, 15 May 2026 15:58:42 +0800 Subject: [PATCH 04/20] fix --- .../apache/fluss/server/log/LogManager.java | 18 ++++++++++++------ .../org/apache/fluss/server/log/LogTablet.java | 8 +++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index 01d7abe9b2..4335fec4fa 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -17,9 +17,11 @@ package org.apache.fluss.server.log; +import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.exception.FlussException; +import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.FlussException; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.exception.LogStorageException; import org.apache.fluss.exception.PartitionNotExistException; @@ -326,11 +328,9 @@ public LogTablet getOrCreateLog( }); } - /** - * using the default {@link ConfigOptions#TABLE_LOG_TTL} when no table properties are - * configured. - */ + @VisibleForTesting public LogTablet getOrCreateLog( + File dataDir, PhysicalTablePath tablePath, TableBucket tableBucket, LogFormat logFormat, @@ -339,7 +339,13 @@ public LogTablet getOrCreateLog( throws Exception { long logTtlMs = new TableConfig(new Configuration()).getLogTTLMs(); return getOrCreateLog( - tablePath, tableBucket, logFormat, tieredLogLocalSegments, logTtlMs, isChangelog); + dataDir, + tablePath, + tableBucket, + logFormat, + tieredLogLocalSegments, + logTtlMs, + isChangelog); } public Optional getLog(TableBucket tableBucket) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 3b7c858241..0488b02a3f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -401,12 +401,9 @@ public static LogTablet create( clock); } - /** - * Same as {@link #create(PhysicalTablePath, File, Configuration, TabletServerMetricGroup, long, - * Scheduler, LogFormat, int, long, boolean, Clock, boolean)} using the default {@link - * ConfigOptions#TABLE_LOG_TTL} when no table properties are configured. - */ + @VisibleForTesting public static LogTablet create( + File dataDir, PhysicalTablePath tablePath, File tabletDir, Configuration conf, @@ -420,6 +417,7 @@ public static LogTablet create( boolean isCleanShutdown) throws Exception { return create( + dataDir, tablePath, tabletDir, conf, From d03ad5e8edd4168efd855104899edfbd7b17de00 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 8 Jul 2026 19:43:52 +0800 Subject: [PATCH 05/20] local segment roll time --- .../fluss/client/admin/FlussAdminITCase.java | 44 ++++++++-- .../apache/fluss/config/ConfigOptions.java | 8 ++ .../apache/fluss/config/FlussConfigUtils.java | 1 + .../org/apache/fluss/config/TableConfig.java | 12 +++ .../apache/fluss/config/TableConfigTest.java | 27 ++++++ .../apache/fluss/server/log/LogManager.java | 10 ++- .../apache/fluss/server/log/LogTablet.java | 32 +++++-- .../apache/fluss/server/replica/Replica.java | 16 ++++ .../fluss/server/replica/ReplicaManager.java | 14 ++- .../utils/TableDescriptorValidation.java | 26 ++++++ .../server/log/remote/RemoteLogITCase.java | 88 ++++++++++++++++--- .../server/log/remote/RemoteLogTTLTest.java | 51 +++++++++++ .../server/replica/ReplicaManagerTest.java | 59 +++++++++++++ .../fluss/server/replica/ReplicaTest.java | 23 ++++- 14 files changed, 381 insertions(+), 30 deletions(-) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index 150b84e567..cda01f4299 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -857,29 +857,55 @@ void testCreateTableWithInvalidProperty() { TableDescriptor.builder() .schema(DEFAULT_SCHEMA) .comment("test table") - .property(ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.key(), "0") + .property(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), "0s") .build(); // should throw exception assertThatThrownBy(() -> admin.createTable(tablePath, t3, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) - .hasMessage("'table.log.tiered.local-segments' must be greater than 0."); + .hasMessage("'table.log.segment.active-roll-time' must be greater than 0."); TableDescriptor t4 = + TableDescriptor.builder() + .schema(DEFAULT_SCHEMA) + .comment("test table") + .property(ConfigOptions.TABLE_LOG_TTL.key(), "1h") + .property(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), "2h") + .build(); + // should throw exception + assertThatThrownBy(() -> admin.createTable(tablePath, t4, false).get()) + .cause() + .isInstanceOf(InvalidConfigException.class) + .hasMessage( + "'table.log.segment.active-roll-time' must be less than or equal to 'table.log.ttl'."); + + TableDescriptor t5 = + TableDescriptor.builder() + .schema(DEFAULT_SCHEMA) + .comment("test table") + .property(ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.key(), "0") + .build(); + // should throw exception + assertThatThrownBy(() -> admin.createTable(tablePath, t5, false).get()) + .cause() + .isInstanceOf(InvalidConfigException.class) + .hasMessage("'table.log.tiered.local-segments' must be greater than 0."); + + TableDescriptor t6 = TableDescriptor.builder() .schema(DEFAULT_SCHEMA) // no pk .comment("test table") .property(ConfigOptions.TABLE_MERGE_ENGINE.key(), "versioned") .build(); // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t4, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t6, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessage( "'%s' must be set for versioned merge engine.", ConfigOptions.TABLE_MERGE_ENGINE_VERSION_COLUMN.key()); - TableDescriptor t5 = + TableDescriptor t7 = TableDescriptor.builder() .schema(DEFAULT_SCHEMA) // no pk .comment("test table") @@ -889,13 +915,13 @@ void testCreateTableWithInvalidProperty() { "non-existed") .build(); // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t5, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t7, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessage( "The version column 'non-existed' for versioned merge engine doesn't exist in schema."); - TableDescriptor t6 = + TableDescriptor t8 = TableDescriptor.builder() .schema(DEFAULT_SCHEMA) // no pk .comment("test table") @@ -903,14 +929,14 @@ void testCreateTableWithInvalidProperty() { .property(ConfigOptions.TABLE_MERGE_ENGINE_VERSION_COLUMN.key(), "name") .build(); // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t6, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t8, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessage( "The version column 'name' for versioned merge engine must be one type of " + "[INT, BIGINT, TIMESTAMP, TIMESTAMP_LTZ], but got STRING."); - TableDescriptor t7 = + TableDescriptor t9 = TableDescriptor.builder() .schema( Schema.newBuilder() @@ -921,7 +947,7 @@ void testCreateTableWithInvalidProperty() { .kvFormat(KvFormat.COMPACTED) .logFormat(LogFormat.INDEXED) .build(); - assertThatThrownBy(() -> admin.createTable(tablePath, t7, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t9, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessageContaining( diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 7ceee8fb5d..692539f719 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -1765,6 +1765,14 @@ public class ConfigOptions { + "we will retain a log before we will delete old segments to free up " + "space. If set to -1, the log will not be deleted."); + public static final ConfigOption TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME = + key("table.log.segment.active-roll-time") + .durationType() + .noDefaultValue() + .withDescription( + "The maximum time a non-empty active log segment may stay open before it is rolled. " + + "If not configured, the value falls back to table.log.ttl."); + public static final ConfigOption TABLE_TIERED_LOG_LOCAL_SEGMENTS = key("table.log.tiered.local-segments") .intType() diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 4f978a0b30..e526fff5ed 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -52,6 +52,7 @@ public class FlussConfigUtils { ConfigOptions.TABLE_DATALAKE_AUTO_COMPACTION.key(), ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.key(), ConfigOptions.TABLE_AUTO_PARTITION_ENABLED.key(), + ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION.key(), ConfigOptions.TABLE_AUTO_PARTITION_NUM_PRECREATE.key(), ConfigOptions.TABLE_STATISTICS_COLUMNS.key(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index fbf8c77264..c533c0b10b 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -90,6 +90,18 @@ public long getLogTTLMs() { return config.get(ConfigOptions.TABLE_LOG_TTL).toMillis(); } + /** Gets the optional active segment roll time of the table. */ + public Optional getActiveSegmentRollTime() { + return config.getOptional(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME); + } + + /** Gets the effective active segment roll time of the table in milliseconds. */ + public long getEffectiveActiveSegmentRollTimeMs() { + return getActiveSegmentRollTime() + .orElse(config.get(ConfigOptions.TABLE_LOG_TTL)) + .toMillis(); + } + /** Gets the local segments to retain for tiered log of the table. */ public int getTieredLogLocalSegments() { return config.get(ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS); diff --git a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java index 5d18fcd1c9..67954adc00 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java @@ -21,6 +21,8 @@ import org.junit.jupiter.api.Test; +import java.time.Duration; + import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link TableConfig}. */ @@ -44,4 +46,29 @@ void testDeleteBehavior() { TableConfig tableConfig3 = new TableConfig(conf); assertThat(tableConfig3.getDeleteBehavior()).hasValue(DeleteBehavior.IGNORE); } + + @Test + void testActiveSegmentRollTimeFallsBackToLogTtl() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.TABLE_LOG_TTL, Duration.ofHours(6)); + + TableConfig tableConfig = new TableConfig(conf); + + assertThat(tableConfig.getActiveSegmentRollTime()).isEmpty(); + assertThat(tableConfig.getEffectiveActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofHours(6).toMillis()); + } + + @Test + void testActiveSegmentRollTimeOverridesLogTtl() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.TABLE_LOG_TTL, Duration.ofHours(6)); + conf.set(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME, Duration.ofMinutes(30)); + + TableConfig tableConfig = new TableConfig(conf); + + assertThat(tableConfig.getActiveSegmentRollTime()).hasValue(Duration.ofMinutes(30)); + assertThat(tableConfig.getEffectiveActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofMinutes(30).toMillis()); + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index 4335fec4fa..939f6d72d6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -282,6 +282,8 @@ private void waitForLoadLogsInDir(LogRecoveryTask recoveryTask) throws Throwable * @param logFormat the log format * @param tieredLogLocalSegments the number of segments to retain in local for tiered log * @param logTtlMs the log TTL in milliseconds from table configuration + * @param activeSegmentRollTimeMs the active segment roll time in milliseconds from table + * configuration * @param isChangelog whether the log is a changelog of primary key table */ public LogTablet getOrCreateLog( @@ -291,6 +293,7 @@ public LogTablet getOrCreateLog( LogFormat logFormat, int tieredLogLocalSegments, long logTtlMs, + long activeSegmentRollTimeMs, boolean isChangelog) throws Exception { return inLock( @@ -314,6 +317,7 @@ public LogTablet getOrCreateLog( logFormat, tieredLogLocalSegments, logTtlMs, + activeSegmentRollTimeMs, isChangelog, clock, true); @@ -337,14 +341,15 @@ public LogTablet getOrCreateLog( int tieredLogLocalSegments, boolean isChangelog) throws Exception { - long logTtlMs = new TableConfig(new Configuration()).getLogTTLMs(); + TableConfig tableConfig = new TableConfig(new Configuration()); return getOrCreateLog( dataDir, tablePath, tableBucket, logFormat, tieredLogLocalSegments, - logTtlMs, + tableConfig.getLogTTLMs(), + tableConfig.getEffectiveActiveSegmentRollTimeMs(), isChangelog); } @@ -459,6 +464,7 @@ private LogTablet loadLog( tableInfo.getTableConfig().getLogFormat(), tableInfo.getTableConfig().getTieredLogLocalSegments(), tableInfo.getTableConfig().getLogTTLMs(), + tableInfo.getTableConfig().getEffectiveActiveSegmentRollTimeMs(), tableInfo.hasPrimaryKey(), clock, isCleanShutdown); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 0488b02a3f..3d2462ceab 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -109,6 +109,7 @@ public final class LogTablet { private final Clock clock; private final boolean isChangeLog; private final long logTtlMs; + private volatile long activeSegmentRollTimeMs; @GuardedBy("lock") private volatile LogOffsetMetadata highWatermarkMetadata; @@ -148,6 +149,7 @@ private LogTablet( LogFormat logFormat, int tieredLogLocalSegments, long logTtlMs, + long activeSegmentRollTimeMs, boolean isChangelog, Clock clock) { this.dataDir = dataDir; @@ -160,6 +162,7 @@ private LogTablet( this.writerStateManager = writerStateManager; this.highWatermarkMetadata = new LogOffsetMetadata(0L); this.logTtlMs = logTtlMs; + this.activeSegmentRollTimeMs = activeSegmentRollTimeMs; this.scheduler = scheduler; // scheduler the writer expiration interval check. @@ -348,6 +351,7 @@ public static LogTablet create( LogFormat logFormat, int tieredLogLocalSegments, long logTtlMs, + long activeSegmentRollTimeMs, boolean isChangelog, Clock clock, boolean isCleanShutdown) @@ -397,6 +401,7 @@ public static LogTablet create( logFormat, tieredLogLocalSegments, logTtlMs, + activeSegmentRollTimeMs, isChangelog, clock); } @@ -416,6 +421,7 @@ public static LogTablet create( Clock clock, boolean isCleanShutdown) throws Exception { + TableConfig tableConfig = new TableConfig(new Configuration()); return create( dataDir, tablePath, @@ -426,7 +432,8 @@ public static LogTablet create( scheduler, logFormat, tieredLogLocalSegments, - new TableConfig(new Configuration()).getLogTTLMs(), + tableConfig.getLogTTLMs(), + tableConfig.getEffectiveActiveSegmentRollTimeMs(), isChangelog, clock, isCleanShutdown); @@ -653,6 +660,14 @@ public int getTieredLogLocalSegments() { return tieredLogLocalSegments; } + public void updateActiveSegmentRollTimeMs(long activeSegmentRollTimeMs) { + this.activeSegmentRollTimeMs = activeSegmentRollTimeMs; + } + + public long getActiveSegmentRollTimeMs() { + return activeSegmentRollTimeMs; + } + public void updateLakeTableSnapshotId(long snapshotId) { if (snapshotId > this.lakeTableSnapshotId) { this.lakeTableSnapshotId = snapshotId; @@ -1056,14 +1071,15 @@ public void roll(Optional expectedNextOffset) throws Exception { * segment eligible for remote tiering and local cleanup. */ public void rollActiveSegmentIfExpired() throws Exception { - if (logTtlMs <= 0L) { + if (activeSegmentRollTimeMs <= 0L) { return; } synchronized (lock) { LogSegment activeSegment = localLog.getSegments().activeSegment(); if (activeSegment.getSizeInBytes() == 0 - || !isSegmentExpired(clock.milliseconds(), activeSegment)) { + || !isSegmentExpired( + clock.milliseconds(), activeSegment, activeSegmentRollTimeMs)) { return; } @@ -1348,18 +1364,20 @@ private List deletableSegments(long endOffset) throws IOException { if (logSegments.get(i + 1).getBaseOffset() > endOffset) { break; } - if (i < tierProtectedStartIndex || isSegmentExpired(now, logSegments.get(i))) { + if (i < tierProtectedStartIndex + || isSegmentExpired(now, logSegments.get(i), logTtlMs)) { deletableSegments.add(logSegments.get(i)); } } return deletableSegments; } - private boolean isSegmentExpired(long now, LogSegment segment) throws IOException { - if (logTtlMs <= 0L) { + private boolean isSegmentExpired(long now, LogSegment segment, long expirationTimeMs) + throws IOException { + if (expirationTimeMs <= 0L) { return false; } - return now - segment.maxTimestampSoFar() > logTtlMs; + return now - segment.maxTimestampSoFar() > expirationTimeMs; } private void deleteSegments(List deletableSegments, SegmentDeletionReason reason) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index f3d1ff76af..f8d965a4b3 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -683,6 +683,21 @@ public void updateTieredLogLocalSegments(int tieredLogLocalSegments) { tieredLogLocalSegments); } + public void updateActiveSegmentRollTimeMs(long activeSegmentRollTimeMs) { + long oldValue = logTablet.getActiveSegmentRollTimeMs(); + if (oldValue == activeSegmentRollTimeMs) { + return; + } + + logTablet.updateActiveSegmentRollTimeMs(activeSegmentRollTimeMs); + + LOG.info( + "Replica for {} activeSegmentRollTimeMs changed from {} to {}", + tableBucket, + oldValue, + activeSegmentRollTimeMs); + } + private void createKv() { try { // create a closeable registry for the closable related to kv @@ -2163,6 +2178,7 @@ private LogTablet createLog( tableConfig.getLogFormat(), tableConfig.getTieredLogLocalSegments(), tableConfig.getLogTTLMs(), + tableConfig.getEffectiveActiveSegmentRollTimeMs(), isKvTable()); // update high watermark. Optional watermarkOpt = lazyHighWatermarkCheckpoint.fetch(tableBucket); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index e38537e279..ab4ade8346 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -565,6 +565,7 @@ public void maybeUpdateMetadataCache(int coordinatorEpoch, ClusterMetadata clust private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { Map tableIdToLakeFlag = new HashMap<>(); Map tableIdToTieredLogLocalSegments = new HashMap<>(); + Map tableIdToActiveSegmentRollTimeMs = new HashMap<>(); for (TableMetadata tableMetadata : clusterMetadata.getTableMetadataList()) { TableInfo tableInfo = tableMetadata.getTableInfo(); @@ -579,9 +580,15 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { // Collect tiered log local segments configuration int tieredLogLocalSegments = tableInfo.getTableConfig().getTieredLogLocalSegments(); tableIdToTieredLogLocalSegments.put(tableId, tieredLogLocalSegments); + + long activeSegmentRollTimeMs = + tableInfo.getTableConfig().getEffectiveActiveSegmentRollTimeMs(); + tableIdToActiveSegmentRollTimeMs.put(tableId, activeSegmentRollTimeMs); } - if (tableIdToLakeFlag.isEmpty() && tableIdToTieredLogLocalSegments.isEmpty()) { + if (tableIdToLakeFlag.isEmpty() + && tableIdToTieredLogLocalSegments.isEmpty() + && tableIdToActiveSegmentRollTimeMs.isEmpty()) { return; } @@ -601,6 +608,11 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { replica.updateTieredLogLocalSegments( tableIdToTieredLogLocalSegments.get(tableId)); } + + if (tableIdToActiveSegmentRollTimeMs.containsKey(tableId)) { + replica.updateActiveSegmentRollTimeMs( + tableIdToActiveSegmentRollTimeMs.get(tableId)); + } } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 8a6e9fa2c9..30209da7e0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -126,6 +126,7 @@ public static void validateTableDescriptor( checkMergeEngine(tableConf, hasPrimaryKey, schema); checkDeleteBehavior(tableConf, hasPrimaryKey); checkTieredLog(tableConf); + checkActiveSegmentRollTime(tableConf); checkPartition(tableConf, tableDescriptor.getPartitionKeys(), schema.getRowType()); checkSystemColumns(schema.getRowType()); validateStatisticsConfig(tableDescriptor); @@ -459,6 +460,31 @@ private static void checkTieredLog(Configuration tableConf) { } } + private static void checkActiveSegmentRollTime(Configuration tableConf) { + Optional activeRollTime = + tableConf.getOptional(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME); + if (!activeRollTime.isPresent()) { + return; + } + + long activeRollTimeMs = activeRollTime.get().toMillis(); + if (activeRollTimeMs <= 0) { + throw new InvalidConfigException( + String.format( + "'%s' must be greater than 0.", + ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key())); + } + + long logTtlMs = tableConf.get(ConfigOptions.TABLE_LOG_TTL).toMillis(); + if (logTtlMs > 0 && activeRollTimeMs > logTtlMs) { + throw new InvalidConfigException( + String.format( + "'%s' must be less than or equal to '%s'.", + ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), + ConfigOptions.TABLE_LOG_TTL.key())); + } + } + private static void checkPartition( Configuration tableConf, List partitionKeys, RowType rowType) { boolean isPartitioned = !partitionKeys.isEmpty(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java index 872682ad26..adca827fde 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java @@ -224,6 +224,74 @@ void testFollowerFetchAlreadyMoveToRemoteLog(boolean withWriterId) throws Except FLUSS_CLUSTER_EXTENSION.waitUntilReplicaExpandToIsr(tb, follower); } + @Test + void testActiveSegmentRollTimeWithAlterTable() throws Exception { + TablePath tablePath = TablePath.of("fluss", "test_active_segment_roll_time_with_alter"); + TableDescriptor tableDescriptor = + TableDescriptor.builder() + .schema(DATA1_SCHEMA) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_TTL, Duration.ofDays(7)) + .build(); + + long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, tableDescriptor); + TableBucket tb = new TableBucket(tableId, 0); + FLUSS_CLUSTER_EXTENSION.waitUntilAllReplicaReady(tb); + + int leaderId = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(tb); + TabletServerGateway leaderGateway = + FLUSS_CLUSTER_EXTENSION.newTabletServerClientForNode(leaderId); + assertProduceLogResponse( + leaderGateway + .produceLog( + newProduceLogRequest( + tb.getTableId(), 0, 1, genMemoryLogRecordsByObject(DATA1))) + .get(), + 0, + 0L); + + TabletServer tabletServer = FLUSS_CLUSTER_EXTENSION.getTabletServerById(leaderId); + RemoteLogManager remoteLogManager = tabletServer.getReplicaManager().getRemoteLogManager(); + RemoteLogTablet remoteLogTablet = remoteLogManager.remoteLogTablet(tb); + Replica leaderReplica = FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(tb); + LogTablet logTablet = leaderReplica.getLogTablet(); + + assertThat(remoteLogTablet.allRemoteLogSegments()).isEmpty(); + assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); + assertThat(logTablet.getActiveSegmentRollTimeMs()).isEqualTo(Duration.ofDays(7).toMillis()); + + MANUAL_CLOCK.advanceTime(Duration.ofHours(2)); + retry( + Duration.ofSeconds(5), + () -> assertThat(remoteLogTablet.allRemoteLogSegments()).isEmpty()); + + CoordinatorGateway coordinatorGateway = FLUSS_CLUSTER_EXTENSION.newCoordinatorClient(); + coordinatorGateway + .alterTable( + newAlterTableRequest( + tablePath, + Collections.singletonMap( + ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), + "1h"), + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + + retry( + Duration.ofMinutes(1), + () -> + assertThat(logTablet.getActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofHours(1).toMillis())); + retry( + Duration.ofMinutes(2), + () -> { + assertThat(remoteLogTablet.allRemoteLogSegments()).hasSize(1); + assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); + assertThat(logTablet.activeLogSegment().getSizeInBytes()).isZero(); + }); + } + @Test void testRemoteLogTTLWithDynamicLakeToggle() throws Exception { TablePath tablePath = TablePath.of("fluss", "test_remote_log_ttl_dynamic_lake"); @@ -315,31 +383,31 @@ void testRemoteLogTTLWithDynamicLakeToggle() throws Exception { long partialLakeOffset = sortedSegments.get(midIndex).remoteLogEndOffset(); logTablet.updateLakeLogEndOffset(partialLakeOffset); - final int expectedRemainingSegments = sortedSegments.size() - midIndex - 1; - // The new remoteLogStartOffset should be the start offset of the first remaining segment - final long expectedNewStartOffset = sortedSegments.get(midIndex + 1).remoteLogStartOffset(); - // Wait for partial cleanup - only segments that have been tiered should be deleted retry( Duration.ofMinutes(2), () -> { // Some segments should be deleted (those with endOffset <= partialLakeOffset) int currentSegmentCount = remoteLogTablet.allRemoteLogSegments().size(); - assertThat(currentSegmentCount).isEqualTo(expectedRemainingSegments); - // Remote log start offset should be updated to the first remaining segment's - // start - assertThat(remoteLogTablet.getRemoteLogStartOffset()) - .isEqualTo(expectedNewStartOffset); + assertThat(currentSegmentCount) + .isGreaterThan(0) + .isLessThan(sortedSegments.size()); // Remaining segments should have remoteLogEndOffset > partialLakeOffset assertThat(remoteLogTablet.allRemoteLogSegments()) .allSatisfy( segment -> assertThat(segment.remoteLogEndOffset()) .isGreaterThan(partialLakeOffset)); + assertThat(remoteLogTablet.getRemoteLogStartOffset()) + .isEqualTo( + remoteLogTablet.allRemoteLogSegments().stream() + .mapToLong(RemoteLogSegment::remoteLogStartOffset) + .min() + .orElseThrow(IllegalStateException::new)); }); // Step 2: Fully update lake log end offset to trigger complete cleanup - logTablet.updateLakeLogEndOffset(stageBRemoteLogEndOffset); + logTablet.updateLakeLogEndOffset(remoteLogTablet.getRemoteLogEndOffset().orElse(-1L)); // Wait for complete cleanup - all segments should be deleted retry( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java index 216c59d883..45c430a3c5 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java @@ -17,12 +17,17 @@ package org.apache.fluss.server.log.remote; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.server.entity.FetchReqInfo; +import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.log.FetchParams; import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -31,11 +36,16 @@ import java.time.Duration; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.apache.fluss.record.TestData.DATA1; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH; +import static org.apache.fluss.server.zk.data.LeaderAndIsr.INITIAL_BUCKET_EPOCH; +import static org.apache.fluss.server.zk.data.LeaderAndIsr.INITIAL_LEADER_EPOCH; import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsWithWriterId; import static org.assertj.core.api.Assertions.assertThat; @@ -135,6 +145,47 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { .isEqualTo(Errors.LOG_OFFSET_OUT_OF_RANGE_EXCEPTION.code()); } + @Test + void testActiveSegmentRollTimeOverride() throws Exception { + long tableId = 10001L; + TablePath tablePath = TablePath.of("fluss", "test_active_segment_roll_time_override"); + Map properties = new HashMap<>(); + properties.put(ConfigOptions.TABLE_LOG_TTL.key(), "7d"); + properties.put(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), "1h"); + registerTableInZkClient( + tablePath, DATA1_SCHEMA, tableId, Collections.emptyList(), properties); + + TableBucket tb = new TableBucket(tableId, 0); + makeLeaderAndFollower( + Collections.singletonList( + new NotifyLeaderAndIsrData( + PhysicalTablePath.of(tablePath), + tb, + Collections.singletonList(TABLET_SERVER_ID), + new LeaderAndIsr( + TABLET_SERVER_ID, + INITIAL_LEADER_EPOCH, + Collections.singletonList(TABLET_SERVER_ID), + Collections.emptyList(), + INITIAL_COORDINATOR_EPOCH, + INITIAL_BUCKET_EPOCH)))); + LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); + + addMultiSegmentsToLogTablet(logTablet, 1); + assertThat(logTablet.getSegments()).hasSize(1); + assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); + + manualClock.advanceTime(Duration.ofHours(2)); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + RemoteLogTablet remoteLog = remoteLogManager.remoteLogTablet(tb); + assertThat(remoteLog.allRemoteLogSegments()).hasSize(1); + assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(0L); + assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(10L); + assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); + assertThat(logTablet.activeLogSegment().getSizeInBytes()).isZero(); + } + @Test void testExpiredActiveSegmentRolledUploadedAndLocallyDeleted() throws Exception { TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index b738c5a859..94861b7314 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -33,6 +33,7 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaGetter; import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.ChangeType; @@ -99,6 +100,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -2170,6 +2172,63 @@ void testUpdateMetadata() throws Exception { + "The latest known coordinator epoch is 2"); } + @Test + void testUpdateActiveSegmentRollTimeFromMetadataCache() throws Exception { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tb.getBucket()); + Replica replica = replicaManager.getReplicaOrException(tb); + assertThat(replica.getLogTablet().getActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofDays(7).toMillis()); + + TableDescriptor updatedTableDescriptor = + TableDescriptor.builder() + .schema(DATA1_SCHEMA) + .distributedBy(3) + .property( + ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME, + Duration.ofHours(1)) + .build(); + TableInfo tableInfo = + TableInfo.of( + DATA1_TABLE_PATH, + DATA1_TABLE_ID, + 1, + updatedTableDescriptor, + DEFAULT_REMOTE_DATA_DIR, + System.currentTimeMillis(), + System.currentTimeMillis()); + TableMetadata tableMetadata = new TableMetadata(tableInfo, Collections.emptyList()); + Set tsServerInfoList = + new HashSet<>( + Arrays.asList( + new ServerInfo( + TABLET_SERVER_ID, + "rack1", + Endpoint.fromListenersString("CLIENT://localhost:90"), + ServerType.TABLET_SERVER), + new ServerInfo( + 2, + "rack2", + Endpoint.fromListenersString("CLIENT://localhost:91"), + ServerType.TABLET_SERVER), + new ServerInfo( + 3, + "rack3", + Endpoint.fromListenersString("CLIENT://localhost:92"), + ServerType.TABLET_SERVER))); + + replicaManager.maybeUpdateMetadataCache( + 0, + buildClusterMetadata( + null, + tsServerInfoList, + Collections.singletonList(tableMetadata), + Collections.emptyList())); + + assertThat(replica.getLogTablet().getActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofHours(1).toMillis()); + } + @Test void testGetReplicaOrException() { // 1. Test online replica diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index d37e91dc6d..d3ae2ca38d 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -825,6 +825,28 @@ void testUpdateIsDataLakeEnabled() throws Exception { assertThat(logReplica.getLogTablet().isDataLakeEnabled()).isFalse(); } + @Test + void testUpdateActiveSegmentRollTimeMs() throws Exception { + Replica logReplica = + makeLogReplica(DATA1_PHYSICAL_TABLE_PATH, new TableBucket(DATA1_TABLE_ID, 1)); + makeLogReplicaAsLeader(logReplica); + + assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofDays(7).toMillis()); + + logReplica.updateActiveSegmentRollTimeMs(Duration.ofHours(1).toMillis()); + assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofHours(1).toMillis()); + + logReplica.updateActiveSegmentRollTimeMs(Duration.ofHours(1).toMillis()); + assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofHours(1).toMillis()); + + logReplica.updateActiveSegmentRollTimeMs(Duration.ofMinutes(30).toMillis()); + assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) + .isEqualTo(Duration.ofMinutes(30).toMillis()); + } + @SuppressWarnings("unchecked") private long getLakeTieringGaugeValue(Replica replica, String metricName) { MetricGroup lakeTieringMetricGroup = replica.bucketMetrics().addGroup("lakeTiering"); @@ -832,7 +854,6 @@ private long getLakeTieringGaugeValue(Replica replica, String metricName) { (Gauge) ((AbstractMetricGroup) lakeTieringMetricGroup).getMetrics().get(metricName); return gauge.getValue(); - } private void makeLogReplicaAsLeader(Replica replica) throws Exception { makeLeaderReplica( From 2cf2030638677c984df3a8f62832f62ddb4d9210 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 9 Jul 2026 11:20:12 +0800 Subject: [PATCH 06/20] fix test --- .../fluss/client/admin/FlussAdminITCase.java | 9 ++--- .../fluss/flink/utils/FlinkConversions.java | 14 +++++--- .../flink/utils/FlinkConversionsTest.java | 33 +++++++++++++++++++ .../utils/TableDescriptorValidation.java | 9 ----- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index cda01f4299..df95988fda 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -125,6 +125,7 @@ import static org.apache.fluss.testutils.InternalRowAssert.assertThatRow; import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link FlussAdmin}. */ @@ -872,12 +873,8 @@ void testCreateTableWithInvalidProperty() { .property(ConfigOptions.TABLE_LOG_TTL.key(), "1h") .property(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), "2h") .build(); - // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t4, false).get()) - .cause() - .isInstanceOf(InvalidConfigException.class) - .hasMessage( - "'table.log.segment.active-roll-time' must be less than or equal to 'table.log.ttl'."); + assertThatCode(() -> admin.createTable(tablePath, t4, false).get()) + .doesNotThrowAnyException(); TableDescriptor t5 = TableDescriptor.builder() diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java index 01f9a9817e..f4086f5be8 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java @@ -350,11 +350,15 @@ public static org.apache.flink.configuration.ConfigOption toFlinkOption( option = builder.doubleType().defaultValue((Double) flussOption.defaultValue()); } else if (clazz.equals(Duration.class)) { // use string type in Flink option instead to make convert back easier - option = - builder.stringType() - .defaultValue( - TimeUtils.formatWithHighestUnit( - (Duration) flussOption.defaultValue())); + if (flussOption.hasDefaultValue()) { + option = + builder.stringType() + .defaultValue( + TimeUtils.formatWithHighestUnit( + (Duration) flussOption.defaultValue())); + } else { + option = builder.stringType().noDefaultValue(); + } } else if (clazz.equals(Password.class)) { String defaultValue = ((Password) flussOption.defaultValue()).value(); option = builder.stringType().defaultValue(defaultValue); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index efc386eb64..f202e22197 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -44,6 +44,7 @@ import org.apache.flink.table.refresh.RefreshHandler; import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -55,6 +56,7 @@ import static org.apache.flink.table.api.DataTypes.VARCHAR; import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_KEY; import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_NUMBER; +import static org.apache.fluss.flink.FlinkConnectorOptions.TABLE_OPTIONS; import static org.apache.fluss.flink.utils.CatalogTableTestUtils.addOptions; import static org.apache.fluss.flink.utils.CatalogTableTestUtils.checkEqualsIgnoreSchema; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; @@ -385,6 +387,37 @@ void testOptionConversions() { .description())); } + @Test + void testOptionConversionForDurationWithoutDefaultValue() { + org.apache.fluss.config.ConfigOption noDefaultDurationOption = + org.apache.fluss.config.ConfigBuilder.key("test.duration") + .durationType() + .noDefaultValue() + .withDescription("test duration"); + + ConfigOption flinkOption = FlinkConversions.toFlinkOption(noDefaultDurationOption); + + assertThat(flinkOption) + .isEqualTo( + org.apache.flink.configuration.ConfigOptions.key("test.duration") + .stringType() + .noDefaultValue() + .withDescription("test duration")); + } + + @Test + void testTableOptionsContainsDurationOptionWithoutDefaultValue() { + assertThat(TABLE_OPTIONS) + .contains( + org.apache.flink.configuration.ConfigOptions.key( + ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key()) + .stringType() + .noDefaultValue() + .withDescription( + ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME + .description())); + } + @Test void testFlinkMaterializedTableConversions() { ResolvedSchema schema = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 30209da7e0..7aa2f1aea4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -474,15 +474,6 @@ private static void checkActiveSegmentRollTime(Configuration tableConf) { "'%s' must be greater than 0.", ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key())); } - - long logTtlMs = tableConf.get(ConfigOptions.TABLE_LOG_TTL).toMillis(); - if (logTtlMs > 0 && activeRollTimeMs > logTtlMs) { - throw new InvalidConfigException( - String.format( - "'%s' must be less than or equal to '%s'.", - ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), - ConfigOptions.TABLE_LOG_TTL.key())); - } } private static void checkPartition( From f8fbbbd946a7b446c49af6e6075bc4a38c69eb60 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 9 Jul 2026 14:58:41 +0800 Subject: [PATCH 07/20] fix test --- .../source/FlinkTableSourceBatchITCase.java | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java index 51d05fa036..f7f5c688b3 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java @@ -59,8 +59,8 @@ abstract class FlinkTableSourceBatchITCase extends FlinkTestBase { static final String CATALOG_NAME = "testcatalog"; - static final String DEFAULT_DB = "defaultdb"; protected StreamTableEnvironment tEnv; + private String databaseName; @BeforeEach void before() { @@ -75,15 +75,15 @@ void before() { tEnv.executeSql("use catalog " + CATALOG_NAME); tEnv.getConfig().set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4); - // create database - tEnv.executeSql("create database " + DEFAULT_DB); - tEnv.useDatabase(DEFAULT_DB); + databaseName = "defaultdb_" + RandomUtils.nextInt(); + tEnv.executeSql("create database " + databaseName); + tEnv.useDatabase(databaseName); } @AfterEach void after() { tEnv.useDatabase(BUILTIN_DATABASE); - tEnv.executeSql(String.format("drop database %s cascade", DEFAULT_DB)); + tEnv.executeSql(String.format("drop database %s cascade", databaseName)); } @Test @@ -94,10 +94,10 @@ void testScanSingleRowFilter() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[and(=(id, 1), =(name, _UTF-16LE'name1':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name])", - tableName)); + databaseName, tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, address1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -111,10 +111,10 @@ void testScanSingleRowFilter2() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[and(=(id, 1), =(name, _UTF-16LE'name1':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name])", - tableName)); + databaseName, tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, address1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -128,10 +128,10 @@ void testScanSingleRowFilter3() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[=(id, 1)], " + "project=[id, name]]], fields=[id, name])", - tableName)); + databaseName, tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -140,7 +140,7 @@ void testScanSingleRowFilter3() throws Exception { @Test void testScanSingleRowFilterOnPartitionedTable() throws Exception { String tableName = prepareSourceTable(new String[] {"id", "dt"}, "dt"); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), tablePath); Iterator partitionIterator = @@ -152,10 +152,10 @@ void testScanSingleRowFilterOnPartitionedTable() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[and(=(id, 1), =(dt, _UTF-16LE'%s':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name, dt])\n", - tableName, partition1)); + databaseName, tableName, partition1)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = @@ -194,7 +194,7 @@ void testFilterOnLookupSource() throws Exception { + " 'table.auto-partition.time-unit' = 'year')", dimTableName)); - TablePath srcTablePath = TablePath.of(DEFAULT_DB, srcTableName); + TablePath srcTablePath = TablePath.of(databaseName, srcTableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), srcTablePath); // just pick first partition to insert data @@ -212,7 +212,7 @@ void testFilterOnLookupSource() throws Exception { upsertWriter.flush(); } - TablePath dimTablePath = TablePath.of(DEFAULT_DB, dimTableName); + TablePath dimTablePath = TablePath.of(databaseName, dimTableName); // prepare dim table data try (Table dimTable = conn.getTable(dimTablePath)) { UpsertWriter upsertWriter = dimTable.newUpsert().createWriter(); @@ -263,7 +263,7 @@ void testLakeTableQueryOnLakeDisabledTable() throws Exception { .isInstanceOf(UnsupportedOperationException.class) .hasMessage( String.format( - "Table %s.%s is not datalake enabled.", DEFAULT_DB, tableName)); + "Table %s.%s is not datalake enabled.", databaseName, tableName)); } @Test @@ -461,7 +461,9 @@ void testCountPushDownWithWALMode() throws Exception { assertThatThrownBy(() -> tEnv.executeSql(query)) .hasRootCauseInstanceOf(InvalidTableException.class) .hasMessageContaining( - "Row count is disabled for this table 'defaultdb.test_count_table_with_wal'."); + String.format( + "Row count is disabled for this table '%s.test_count_table_with_wal'.", + databaseName)); } @ParameterizedTest @@ -546,7 +548,7 @@ private String prepareSourceTable(String[] keys, String partitionedKey) throws E tableName, String.join(",", keys), partitionedKey)); } - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); String partition1 = null; if (partitionedKey != null) { Map partitionNameById = @@ -587,7 +589,7 @@ private String prepareLogTable() throws Exception { + ")", tableName)); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); // prepare table data with NULL values in address column try (Table table = conn.getTable(tablePath)) { @@ -619,7 +621,7 @@ protected String preparePartitionedLogTable() throws Exception { + " 'table.auto-partition.time-unit' = 'year')", tableName)); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), tablePath); Collection partitions = partitionNameById.values(); @@ -657,7 +659,7 @@ private String prepareLogTableWithComplexTypes() throws Exception { + ")", tableName)); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); // prepare table data with complex types try (Table table = conn.getTable(tablePath)) { From c019cbf349b9a050be806d02e1179a0ff673834c Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Thu, 9 Jul 2026 16:31:08 +0800 Subject: [PATCH 08/20] optimize --- .../apache/fluss/server/log/LogTablet.java | 39 ++++++++++++++----- .../server/log/remote/LogTieringTask.java | 21 +++++++--- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 3d2462ceab..ef22e2d23f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -631,11 +631,15 @@ public void updateRemoteLogSize(long remoteLogSize) { } public void updateRemoteLogEndOffset(long remoteLogEndOffset) { + updateRemoteLogEndOffset(remoteLogEndOffset, clock.milliseconds()); + } + + public void updateRemoteLogEndOffset(long remoteLogEndOffset, long currentTimeMs) { if (remoteLogEndOffset > this.remoteLogEndOffset) { this.remoteLogEndOffset = remoteLogEndOffset; // try to delete these segments already exist in remote storage. - deleteSegmentsAlreadyExistsInRemote(); + deleteSegmentsAlreadyExistsInRemote(currentTimeMs); } } @@ -750,7 +754,11 @@ public void loadWriterSnapshot(long lastOffset) throws IOException { } public void deleteSegmentsAlreadyExistsInRemote() { - deleteSegments(remoteLogEndOffset); + deleteSegmentsAlreadyExistsInRemote(clock.milliseconds()); + } + + public void deleteSegmentsAlreadyExistsInRemote(long currentTimeMs) { + deleteSegments(remoteLogEndOffset, currentTimeMs); } /** @@ -767,7 +775,7 @@ public LogOffsetSnapshot fetchOffsetSnapshot() throws IOException { highWatermark); } - private void deleteSegments(long cleanUpToOffset) { + private void deleteSegments(long cleanUpToOffset, long currentTimeMs) { // cache to local variables long localLogStartOffset = localLog.getLocalLogStartOffset(); if (cleanUpToOffset < localLogStartOffset) { @@ -793,7 +801,8 @@ private void deleteSegments(long cleanUpToOffset) { try { // shouldn't clean up segments that will be used by kv recovery. long cleanupToOffset = Math.min(minRetainOffset, cleanUpToOffset); - deleteOldSegments(cleanupToOffset, SegmentDeletionReason.LOG_MOVE_TO_REMOTE); + deleteOldSegments( + cleanupToOffset, SegmentDeletionReason.LOG_MOVE_TO_REMOTE, currentTimeMs); } catch (IOException e) { LOG.error( "Failed to delete the local log segments to cleanUpToOffset {} for table-bucket {}.", @@ -1065,12 +1074,17 @@ public void roll(Optional expectedNextOffset) throws Exception { } /** - * Rolls the active segment if it is non-empty and all of its records have passed the log TTL. + * Rolls the active segment if it is non-empty and all of its records have passed the configured + * segment rolling interval. * *

This keeps an empty active segment available for future appends while making the expired * segment eligible for remote tiering and local cleanup. */ public void rollActiveSegmentIfExpired() throws Exception { + rollActiveSegmentIfExpired(clock.milliseconds()); + } + + public void rollActiveSegmentIfExpired(long currentTimeMs) throws Exception { if (activeSegmentRollTimeMs <= 0L) { return; } @@ -1079,7 +1093,7 @@ public void rollActiveSegmentIfExpired() throws Exception { LogSegment activeSegment = localLog.getSegments().activeSegment(); if (activeSegment.getSizeInBytes() == 0 || !isSegmentExpired( - clock.milliseconds(), activeSegment, activeSegmentRollTimeMs)) { + currentTimeMs, activeSegment, activeSegmentRollTimeMs)) { return; } @@ -1339,8 +1353,13 @@ snapshot, getTableBucket(), getLogDir().getParent()), private void deleteOldSegments(long endOffset, SegmentDeletionReason reason) throws IOException { + deleteOldSegments(endOffset, reason, clock.milliseconds()); + } + + private void deleteOldSegments(long endOffset, SegmentDeletionReason reason, long currentTimeMs) + throws IOException { synchronized (lock) { - List deletableSegments = deletableSegments(endOffset); + List deletableSegments = deletableSegments(endOffset, currentTimeMs); if (!deletableSegments.isEmpty()) { deleteSegments(deletableSegments, reason); } @@ -1348,7 +1367,8 @@ private void deleteOldSegments(long endOffset, SegmentDeletionReason reason) } /** Returns the segments that can be deleted by checking log end offset. */ - private List deletableSegments(long endOffset) throws IOException { + private List deletableSegments(long endOffset, long currentTimeMs) + throws IOException { if (localLog.getSegments().isEmpty()) { return Collections.emptyList(); } @@ -1358,14 +1378,13 @@ private List deletableSegments(long endOffset) throws IOException { List deletableSegments = new ArrayList<>(); List logSegments = localLog.getSegments().values(); int tierProtectedStartIndex = logSegments.size() - tieredLogLocalSegments; - long now = clock.milliseconds(); for (int i = 0; i < logSegments.size() - 1; i++) { if (logSegments.get(i + 1).getBaseOffset() > endOffset) { break; } if (i < tierProtectedStartIndex - || isSegmentExpired(now, logSegments.get(i), logTtlMs)) { + || isSegmentExpired(currentTimeMs, logSegments.get(i), logTtlMs)) { deletableSegments.add(logSegments.get(i)); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java index 3ea77ca6c5..d642487ba7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java @@ -93,7 +93,7 @@ public void run() { try { // Try to copy these candidate copy log segments to remote storage and try to clean // up these expired remote log segments from remote. - runOnce(); + runOnce(clock.milliseconds()); } catch (InterruptedException ex) { if (!isCancelled()) { LOG.warn( @@ -117,7 +117,7 @@ public void run() { } } - private void runOnce() throws InterruptedException { + private void runOnce(long currentTimeMs) throws InterruptedException { if (isCancelled()) { LOG.info("Returning from LogTieringTask runOnes as the task state is changed"); return; @@ -128,7 +128,7 @@ private void runOnce() throws InterruptedException { TableMetricGroup metricGroup = replica.tableMetrics(); maybeUpdateCopiedOffset(logTablet); - logTablet.rollActiveSegmentIfExpired(); + logTablet.rollActiveSegmentIfExpired(currentTimeMs); // Get these candidate log segments to copy and these expired remote log segments to // clean up. List candidateToCopySegments = @@ -136,7 +136,7 @@ private void runOnce() throws InterruptedException { // Only delete segments that have been tiered to lake to ensure data safety List expiredRemoteLogSegments = remoteLog.expiredRemoteLogSegments( - clock.milliseconds(), + currentTimeMs, logTablet.isDataLakeEnabled() ? logTablet.getLakeLogEndOffset() : null); // 1. For these candidateToCopySegments, we will first copy segment files to @@ -151,7 +151,7 @@ private void runOnce() throws InterruptedException { if (!copiedSegments.isEmpty() || !expiredRemoteLogSegments.isEmpty()) { boolean success = tryToCommitRemoteLogManifest( - remoteLog, expiredRemoteLogSegments, copiedSegments); + remoteLog, expiredRemoteLogSegments, copiedSegments, currentTimeMs); if (success) { if (!expiredRemoteLogSegments.isEmpty()) { @@ -328,6 +328,15 @@ public boolean tryToCommitRemoteLogManifest( RemoteLogTablet remoteLogTablet, List expiredSegments, List newAddedSegments) { + return tryToCommitRemoteLogManifest( + remoteLogTablet, expiredSegments, newAddedSegments, clock.milliseconds()); + } + + public boolean tryToCommitRemoteLogManifest( + RemoteLogTablet remoteLogTablet, + List expiredSegments, + List newAddedSegments, + long currentTimeMs) { // 1. apply the build snapshot method. RemoteLogManifest newRemoteLogManifest = @@ -384,7 +393,7 @@ public boolean tryToCommitRemoteLogManifest( LogTablet logTablet = replica.getLogTablet(); logTablet.updateRemoteLogStartOffset(newRemoteLogStartOffset); // make the local log cleaner clean log segments that are committed to remote. - logTablet.updateRemoteLogEndOffset(newRemoteLogEndOffset); + logTablet.updateRemoteLogEndOffset(newRemoteLogEndOffset, currentTimeMs); logTablet.updateRemoteLogSize(newRemoteLogSize); return true; } From d56e70f222531db2b6140e46c74355fd9caafcb4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 13 Jul 2026 16:08:06 +0800 Subject: [PATCH 09/20] local segment ttl --- .../fluss/client/admin/FlussAdminITCase.java | 41 ++---- .../apache/fluss/config/ConfigOptions.java | 8 -- .../apache/fluss/config/FlussConfigUtils.java | 1 - .../org/apache/fluss/config/TableConfig.java | 12 -- .../apache/fluss/config/TableConfigTest.java | 27 ---- .../fluss/flink/utils/FlinkConversions.java | 14 +-- .../flink/utils/FlinkConversionsTest.java | 33 ----- .../apache/fluss/server/log/LogManager.java | 6 - .../apache/fluss/server/log/LogTablet.java | 76 ++--------- .../server/log/remote/LogTieringTask.java | 20 +-- .../apache/fluss/server/replica/Replica.java | 16 --- .../fluss/server/replica/ReplicaManager.java | 14 +-- .../utils/TableDescriptorValidation.java | 17 --- .../server/log/remote/RemoteLogITCase.java | 88 ++----------- .../server/log/remote/RemoteLogTTLTest.java | 118 ++---------------- .../log/remote/TieredLocalSegmentTtlTest.java | 73 +++++++++++ .../server/replica/ReplicaManagerTest.java | 59 --------- .../fluss/server/replica/ReplicaTest.java | 3 +- 18 files changed, 122 insertions(+), 504 deletions(-) create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index df95988fda..150b84e567 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -125,7 +125,6 @@ import static org.apache.fluss.testutils.InternalRowAssert.assertThatRow; import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link FlussAdmin}. */ @@ -855,54 +854,32 @@ void testCreateTableWithInvalidProperty() { + "Reason: Could not parse value 'unknown' for key 'table.log.ttl'."); TableDescriptor t3 = - TableDescriptor.builder() - .schema(DEFAULT_SCHEMA) - .comment("test table") - .property(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), "0s") - .build(); - // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t3, false).get()) - .cause() - .isInstanceOf(InvalidConfigException.class) - .hasMessage("'table.log.segment.active-roll-time' must be greater than 0."); - - TableDescriptor t4 = - TableDescriptor.builder() - .schema(DEFAULT_SCHEMA) - .comment("test table") - .property(ConfigOptions.TABLE_LOG_TTL.key(), "1h") - .property(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), "2h") - .build(); - assertThatCode(() -> admin.createTable(tablePath, t4, false).get()) - .doesNotThrowAnyException(); - - TableDescriptor t5 = TableDescriptor.builder() .schema(DEFAULT_SCHEMA) .comment("test table") .property(ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.key(), "0") .build(); // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t5, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t3, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessage("'table.log.tiered.local-segments' must be greater than 0."); - TableDescriptor t6 = + TableDescriptor t4 = TableDescriptor.builder() .schema(DEFAULT_SCHEMA) // no pk .comment("test table") .property(ConfigOptions.TABLE_MERGE_ENGINE.key(), "versioned") .build(); // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t6, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t4, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessage( "'%s' must be set for versioned merge engine.", ConfigOptions.TABLE_MERGE_ENGINE_VERSION_COLUMN.key()); - TableDescriptor t7 = + TableDescriptor t5 = TableDescriptor.builder() .schema(DEFAULT_SCHEMA) // no pk .comment("test table") @@ -912,13 +889,13 @@ void testCreateTableWithInvalidProperty() { "non-existed") .build(); // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t7, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t5, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessage( "The version column 'non-existed' for versioned merge engine doesn't exist in schema."); - TableDescriptor t8 = + TableDescriptor t6 = TableDescriptor.builder() .schema(DEFAULT_SCHEMA) // no pk .comment("test table") @@ -926,14 +903,14 @@ void testCreateTableWithInvalidProperty() { .property(ConfigOptions.TABLE_MERGE_ENGINE_VERSION_COLUMN.key(), "name") .build(); // should throw exception - assertThatThrownBy(() -> admin.createTable(tablePath, t8, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t6, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessage( "The version column 'name' for versioned merge engine must be one type of " + "[INT, BIGINT, TIMESTAMP, TIMESTAMP_LTZ], but got STRING."); - TableDescriptor t9 = + TableDescriptor t7 = TableDescriptor.builder() .schema( Schema.newBuilder() @@ -944,7 +921,7 @@ void testCreateTableWithInvalidProperty() { .kvFormat(KvFormat.COMPACTED) .logFormat(LogFormat.INDEXED) .build(); - assertThatThrownBy(() -> admin.createTable(tablePath, t9, false).get()) + assertThatThrownBy(() -> admin.createTable(tablePath, t7, false).get()) .cause() .isInstanceOf(InvalidConfigException.class) .hasMessageContaining( diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 692539f719..7ceee8fb5d 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -1765,14 +1765,6 @@ public class ConfigOptions { + "we will retain a log before we will delete old segments to free up " + "space. If set to -1, the log will not be deleted."); - public static final ConfigOption TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME = - key("table.log.segment.active-roll-time") - .durationType() - .noDefaultValue() - .withDescription( - "The maximum time a non-empty active log segment may stay open before it is rolled. " - + "If not configured, the value falls back to table.log.ttl."); - public static final ConfigOption TABLE_TIERED_LOG_LOCAL_SEGMENTS = key("table.log.tiered.local-segments") .intType() diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index e526fff5ed..4f978a0b30 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -52,7 +52,6 @@ public class FlussConfigUtils { ConfigOptions.TABLE_DATALAKE_AUTO_COMPACTION.key(), ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS.key(), ConfigOptions.TABLE_AUTO_PARTITION_ENABLED.key(), - ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), ConfigOptions.TABLE_AUTO_PARTITION_NUM_RETENTION.key(), ConfigOptions.TABLE_AUTO_PARTITION_NUM_PRECREATE.key(), ConfigOptions.TABLE_STATISTICS_COLUMNS.key(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index c533c0b10b..fbf8c77264 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -90,18 +90,6 @@ public long getLogTTLMs() { return config.get(ConfigOptions.TABLE_LOG_TTL).toMillis(); } - /** Gets the optional active segment roll time of the table. */ - public Optional getActiveSegmentRollTime() { - return config.getOptional(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME); - } - - /** Gets the effective active segment roll time of the table in milliseconds. */ - public long getEffectiveActiveSegmentRollTimeMs() { - return getActiveSegmentRollTime() - .orElse(config.get(ConfigOptions.TABLE_LOG_TTL)) - .toMillis(); - } - /** Gets the local segments to retain for tiered log of the table. */ public int getTieredLogLocalSegments() { return config.get(ConfigOptions.TABLE_TIERED_LOG_LOCAL_SEGMENTS); diff --git a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java index 67954adc00..5d18fcd1c9 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/TableConfigTest.java @@ -21,8 +21,6 @@ import org.junit.jupiter.api.Test; -import java.time.Duration; - import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link TableConfig}. */ @@ -46,29 +44,4 @@ void testDeleteBehavior() { TableConfig tableConfig3 = new TableConfig(conf); assertThat(tableConfig3.getDeleteBehavior()).hasValue(DeleteBehavior.IGNORE); } - - @Test - void testActiveSegmentRollTimeFallsBackToLogTtl() { - Configuration conf = new Configuration(); - conf.set(ConfigOptions.TABLE_LOG_TTL, Duration.ofHours(6)); - - TableConfig tableConfig = new TableConfig(conf); - - assertThat(tableConfig.getActiveSegmentRollTime()).isEmpty(); - assertThat(tableConfig.getEffectiveActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofHours(6).toMillis()); - } - - @Test - void testActiveSegmentRollTimeOverridesLogTtl() { - Configuration conf = new Configuration(); - conf.set(ConfigOptions.TABLE_LOG_TTL, Duration.ofHours(6)); - conf.set(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME, Duration.ofMinutes(30)); - - TableConfig tableConfig = new TableConfig(conf); - - assertThat(tableConfig.getActiveSegmentRollTime()).hasValue(Duration.ofMinutes(30)); - assertThat(tableConfig.getEffectiveActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofMinutes(30).toMillis()); - } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java index f4086f5be8..01f9a9817e 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java @@ -350,15 +350,11 @@ public static org.apache.flink.configuration.ConfigOption toFlinkOption( option = builder.doubleType().defaultValue((Double) flussOption.defaultValue()); } else if (clazz.equals(Duration.class)) { // use string type in Flink option instead to make convert back easier - if (flussOption.hasDefaultValue()) { - option = - builder.stringType() - .defaultValue( - TimeUtils.formatWithHighestUnit( - (Duration) flussOption.defaultValue())); - } else { - option = builder.stringType().noDefaultValue(); - } + option = + builder.stringType() + .defaultValue( + TimeUtils.formatWithHighestUnit( + (Duration) flussOption.defaultValue())); } else if (clazz.equals(Password.class)) { String defaultValue = ((Password) flussOption.defaultValue()).value(); option = builder.stringType().defaultValue(defaultValue); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index f202e22197..efc386eb64 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -44,7 +44,6 @@ import org.apache.flink.table.refresh.RefreshHandler; import org.junit.jupiter.api.Test; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -56,7 +55,6 @@ import static org.apache.flink.table.api.DataTypes.VARCHAR; import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_KEY; import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_NUMBER; -import static org.apache.fluss.flink.FlinkConnectorOptions.TABLE_OPTIONS; import static org.apache.fluss.flink.utils.CatalogTableTestUtils.addOptions; import static org.apache.fluss.flink.utils.CatalogTableTestUtils.checkEqualsIgnoreSchema; import static org.apache.fluss.record.TestData.DEFAULT_REMOTE_DATA_DIR; @@ -387,37 +385,6 @@ void testOptionConversions() { .description())); } - @Test - void testOptionConversionForDurationWithoutDefaultValue() { - org.apache.fluss.config.ConfigOption noDefaultDurationOption = - org.apache.fluss.config.ConfigBuilder.key("test.duration") - .durationType() - .noDefaultValue() - .withDescription("test duration"); - - ConfigOption flinkOption = FlinkConversions.toFlinkOption(noDefaultDurationOption); - - assertThat(flinkOption) - .isEqualTo( - org.apache.flink.configuration.ConfigOptions.key("test.duration") - .stringType() - .noDefaultValue() - .withDescription("test duration")); - } - - @Test - void testTableOptionsContainsDurationOptionWithoutDefaultValue() { - assertThat(TABLE_OPTIONS) - .contains( - org.apache.flink.configuration.ConfigOptions.key( - ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key()) - .stringType() - .noDefaultValue() - .withDescription( - ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME - .description())); - } - @Test void testFlinkMaterializedTableConversions() { ResolvedSchema schema = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index 939f6d72d6..9172243edb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -282,8 +282,6 @@ private void waitForLoadLogsInDir(LogRecoveryTask recoveryTask) throws Throwable * @param logFormat the log format * @param tieredLogLocalSegments the number of segments to retain in local for tiered log * @param logTtlMs the log TTL in milliseconds from table configuration - * @param activeSegmentRollTimeMs the active segment roll time in milliseconds from table - * configuration * @param isChangelog whether the log is a changelog of primary key table */ public LogTablet getOrCreateLog( @@ -293,7 +291,6 @@ public LogTablet getOrCreateLog( LogFormat logFormat, int tieredLogLocalSegments, long logTtlMs, - long activeSegmentRollTimeMs, boolean isChangelog) throws Exception { return inLock( @@ -317,7 +314,6 @@ public LogTablet getOrCreateLog( logFormat, tieredLogLocalSegments, logTtlMs, - activeSegmentRollTimeMs, isChangelog, clock, true); @@ -349,7 +345,6 @@ public LogTablet getOrCreateLog( logFormat, tieredLogLocalSegments, tableConfig.getLogTTLMs(), - tableConfig.getEffectiveActiveSegmentRollTimeMs(), isChangelog); } @@ -464,7 +459,6 @@ private LogTablet loadLog( tableInfo.getTableConfig().getLogFormat(), tableInfo.getTableConfig().getTieredLogLocalSegments(), tableInfo.getTableConfig().getLogTTLMs(), - tableInfo.getTableConfig().getEffectiveActiveSegmentRollTimeMs(), tableInfo.hasPrimaryKey(), clock, isCleanShutdown); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index ef22e2d23f..24cc7b7974 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -109,7 +109,6 @@ public final class LogTablet { private final Clock clock; private final boolean isChangeLog; private final long logTtlMs; - private volatile long activeSegmentRollTimeMs; @GuardedBy("lock") private volatile LogOffsetMetadata highWatermarkMetadata; @@ -149,7 +148,6 @@ private LogTablet( LogFormat logFormat, int tieredLogLocalSegments, long logTtlMs, - long activeSegmentRollTimeMs, boolean isChangelog, Clock clock) { this.dataDir = dataDir; @@ -162,7 +160,6 @@ private LogTablet( this.writerStateManager = writerStateManager; this.highWatermarkMetadata = new LogOffsetMetadata(0L); this.logTtlMs = logTtlMs; - this.activeSegmentRollTimeMs = activeSegmentRollTimeMs; this.scheduler = scheduler; // scheduler the writer expiration interval check. @@ -351,7 +348,6 @@ public static LogTablet create( LogFormat logFormat, int tieredLogLocalSegments, long logTtlMs, - long activeSegmentRollTimeMs, boolean isChangelog, Clock clock, boolean isCleanShutdown) @@ -401,7 +397,6 @@ public static LogTablet create( logFormat, tieredLogLocalSegments, logTtlMs, - activeSegmentRollTimeMs, isChangelog, clock); } @@ -433,7 +428,6 @@ public static LogTablet create( logFormat, tieredLogLocalSegments, tableConfig.getLogTTLMs(), - tableConfig.getEffectiveActiveSegmentRollTimeMs(), isChangelog, clock, isCleanShutdown); @@ -631,15 +625,11 @@ public void updateRemoteLogSize(long remoteLogSize) { } public void updateRemoteLogEndOffset(long remoteLogEndOffset) { - updateRemoteLogEndOffset(remoteLogEndOffset, clock.milliseconds()); - } - - public void updateRemoteLogEndOffset(long remoteLogEndOffset, long currentTimeMs) { if (remoteLogEndOffset > this.remoteLogEndOffset) { this.remoteLogEndOffset = remoteLogEndOffset; // try to delete these segments already exist in remote storage. - deleteSegmentsAlreadyExistsInRemote(currentTimeMs); + deleteSegmentsAlreadyExistsInRemote(); } } @@ -664,14 +654,6 @@ public int getTieredLogLocalSegments() { return tieredLogLocalSegments; } - public void updateActiveSegmentRollTimeMs(long activeSegmentRollTimeMs) { - this.activeSegmentRollTimeMs = activeSegmentRollTimeMs; - } - - public long getActiveSegmentRollTimeMs() { - return activeSegmentRollTimeMs; - } - public void updateLakeTableSnapshotId(long snapshotId) { if (snapshotId > this.lakeTableSnapshotId) { this.lakeTableSnapshotId = snapshotId; @@ -754,11 +736,7 @@ public void loadWriterSnapshot(long lastOffset) throws IOException { } public void deleteSegmentsAlreadyExistsInRemote() { - deleteSegmentsAlreadyExistsInRemote(clock.milliseconds()); - } - - public void deleteSegmentsAlreadyExistsInRemote(long currentTimeMs) { - deleteSegments(remoteLogEndOffset, currentTimeMs); + deleteSegments(remoteLogEndOffset); } /** @@ -775,7 +753,7 @@ public LogOffsetSnapshot fetchOffsetSnapshot() throws IOException { highWatermark); } - private void deleteSegments(long cleanUpToOffset, long currentTimeMs) { + private void deleteSegments(long cleanUpToOffset) { // cache to local variables long localLogStartOffset = localLog.getLocalLogStartOffset(); if (cleanUpToOffset < localLogStartOffset) { @@ -801,8 +779,7 @@ private void deleteSegments(long cleanUpToOffset, long currentTimeMs) { try { // shouldn't clean up segments that will be used by kv recovery. long cleanupToOffset = Math.min(minRetainOffset, cleanUpToOffset); - deleteOldSegments( - cleanupToOffset, SegmentDeletionReason.LOG_MOVE_TO_REMOTE, currentTimeMs); + deleteOldSegments(cleanupToOffset, SegmentDeletionReason.LOG_MOVE_TO_REMOTE); } catch (IOException e) { LOG.error( "Failed to delete the local log segments to cleanUpToOffset {} for table-bucket {}.", @@ -1073,38 +1050,6 @@ public void roll(Optional expectedNextOffset) throws Exception { } } - /** - * Rolls the active segment if it is non-empty and all of its records have passed the configured - * segment rolling interval. - * - *

This keeps an empty active segment available for future appends while making the expired - * segment eligible for remote tiering and local cleanup. - */ - public void rollActiveSegmentIfExpired() throws Exception { - rollActiveSegmentIfExpired(clock.milliseconds()); - } - - public void rollActiveSegmentIfExpired(long currentTimeMs) throws Exception { - if (activeSegmentRollTimeMs <= 0L) { - return; - } - - synchronized (lock) { - LogSegment activeSegment = localLog.getSegments().activeSegment(); - if (activeSegment.getSizeInBytes() == 0 - || !isSegmentExpired( - currentTimeMs, activeSegment, activeSegmentRollTimeMs)) { - return; - } - - LOG.info( - "Rolling expired active log segment {} for bucket {}.", - activeSegment, - getTableBucket()); - roll(Optional.empty()); - } - } - /** Truncate this log so that it ends with the greatest offset < targetOffset. */ boolean truncateTo(long targetOffset) throws LogStorageException { if (targetOffset < 0) { @@ -1353,13 +1298,8 @@ snapshot, getTableBucket(), getLogDir().getParent()), private void deleteOldSegments(long endOffset, SegmentDeletionReason reason) throws IOException { - deleteOldSegments(endOffset, reason, clock.milliseconds()); - } - - private void deleteOldSegments(long endOffset, SegmentDeletionReason reason, long currentTimeMs) - throws IOException { synchronized (lock) { - List deletableSegments = deletableSegments(endOffset, currentTimeMs); + List deletableSegments = deletableSegments(endOffset); if (!deletableSegments.isEmpty()) { deleteSegments(deletableSegments, reason); } @@ -1367,8 +1307,7 @@ private void deleteOldSegments(long endOffset, SegmentDeletionReason reason, lon } /** Returns the segments that can be deleted by checking log end offset. */ - private List deletableSegments(long endOffset, long currentTimeMs) - throws IOException { + private List deletableSegments(long endOffset) throws IOException { if (localLog.getSegments().isEmpty()) { return Collections.emptyList(); } @@ -1378,13 +1317,14 @@ private List deletableSegments(long endOffset, long currentTimeMs) List deletableSegments = new ArrayList<>(); List logSegments = localLog.getSegments().values(); int tierProtectedStartIndex = logSegments.size() - tieredLogLocalSegments; + long now = clock.milliseconds(); for (int i = 0; i < logSegments.size() - 1; i++) { if (logSegments.get(i + 1).getBaseOffset() > endOffset) { break; } if (i < tierProtectedStartIndex - || isSegmentExpired(currentTimeMs, logSegments.get(i), logTtlMs)) { + || isSegmentExpired(now, logSegments.get(i), logTtlMs)) { deletableSegments.add(logSegments.get(i)); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java index d642487ba7..c9cb32215e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java @@ -93,7 +93,7 @@ public void run() { try { // Try to copy these candidate copy log segments to remote storage and try to clean // up these expired remote log segments from remote. - runOnce(clock.milliseconds()); + runOnce(); } catch (InterruptedException ex) { if (!isCancelled()) { LOG.warn( @@ -117,7 +117,7 @@ public void run() { } } - private void runOnce(long currentTimeMs) throws InterruptedException { + private void runOnce() throws InterruptedException { if (isCancelled()) { LOG.info("Returning from LogTieringTask runOnes as the task state is changed"); return; @@ -128,7 +128,6 @@ private void runOnce(long currentTimeMs) throws InterruptedException { TableMetricGroup metricGroup = replica.tableMetrics(); maybeUpdateCopiedOffset(logTablet); - logTablet.rollActiveSegmentIfExpired(currentTimeMs); // Get these candidate log segments to copy and these expired remote log segments to // clean up. List candidateToCopySegments = @@ -136,7 +135,7 @@ private void runOnce(long currentTimeMs) throws InterruptedException { // Only delete segments that have been tiered to lake to ensure data safety List expiredRemoteLogSegments = remoteLog.expiredRemoteLogSegments( - currentTimeMs, + clock.milliseconds(), logTablet.isDataLakeEnabled() ? logTablet.getLakeLogEndOffset() : null); // 1. For these candidateToCopySegments, we will first copy segment files to @@ -151,7 +150,7 @@ private void runOnce(long currentTimeMs) throws InterruptedException { if (!copiedSegments.isEmpty() || !expiredRemoteLogSegments.isEmpty()) { boolean success = tryToCommitRemoteLogManifest( - remoteLog, expiredRemoteLogSegments, copiedSegments, currentTimeMs); + remoteLog, expiredRemoteLogSegments, copiedSegments); if (success) { if (!expiredRemoteLogSegments.isEmpty()) { @@ -328,15 +327,6 @@ public boolean tryToCommitRemoteLogManifest( RemoteLogTablet remoteLogTablet, List expiredSegments, List newAddedSegments) { - return tryToCommitRemoteLogManifest( - remoteLogTablet, expiredSegments, newAddedSegments, clock.milliseconds()); - } - - public boolean tryToCommitRemoteLogManifest( - RemoteLogTablet remoteLogTablet, - List expiredSegments, - List newAddedSegments, - long currentTimeMs) { // 1. apply the build snapshot method. RemoteLogManifest newRemoteLogManifest = @@ -393,7 +383,7 @@ public boolean tryToCommitRemoteLogManifest( LogTablet logTablet = replica.getLogTablet(); logTablet.updateRemoteLogStartOffset(newRemoteLogStartOffset); // make the local log cleaner clean log segments that are committed to remote. - logTablet.updateRemoteLogEndOffset(newRemoteLogEndOffset, currentTimeMs); + logTablet.updateRemoteLogEndOffset(newRemoteLogEndOffset); logTablet.updateRemoteLogSize(newRemoteLogSize); return true; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index f8d965a4b3..f3d1ff76af 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -683,21 +683,6 @@ public void updateTieredLogLocalSegments(int tieredLogLocalSegments) { tieredLogLocalSegments); } - public void updateActiveSegmentRollTimeMs(long activeSegmentRollTimeMs) { - long oldValue = logTablet.getActiveSegmentRollTimeMs(); - if (oldValue == activeSegmentRollTimeMs) { - return; - } - - logTablet.updateActiveSegmentRollTimeMs(activeSegmentRollTimeMs); - - LOG.info( - "Replica for {} activeSegmentRollTimeMs changed from {} to {}", - tableBucket, - oldValue, - activeSegmentRollTimeMs); - } - private void createKv() { try { // create a closeable registry for the closable related to kv @@ -2178,7 +2163,6 @@ private LogTablet createLog( tableConfig.getLogFormat(), tableConfig.getTieredLogLocalSegments(), tableConfig.getLogTTLMs(), - tableConfig.getEffectiveActiveSegmentRollTimeMs(), isKvTable()); // update high watermark. Optional watermarkOpt = lazyHighWatermarkCheckpoint.fetch(tableBucket); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index ab4ade8346..e38537e279 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -565,7 +565,6 @@ public void maybeUpdateMetadataCache(int coordinatorEpoch, ClusterMetadata clust private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { Map tableIdToLakeFlag = new HashMap<>(); Map tableIdToTieredLogLocalSegments = new HashMap<>(); - Map tableIdToActiveSegmentRollTimeMs = new HashMap<>(); for (TableMetadata tableMetadata : clusterMetadata.getTableMetadataList()) { TableInfo tableInfo = tableMetadata.getTableInfo(); @@ -580,15 +579,9 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { // Collect tiered log local segments configuration int tieredLogLocalSegments = tableInfo.getTableConfig().getTieredLogLocalSegments(); tableIdToTieredLogLocalSegments.put(tableId, tieredLogLocalSegments); - - long activeSegmentRollTimeMs = - tableInfo.getTableConfig().getEffectiveActiveSegmentRollTimeMs(); - tableIdToActiveSegmentRollTimeMs.put(tableId, activeSegmentRollTimeMs); } - if (tableIdToLakeFlag.isEmpty() - && tableIdToTieredLogLocalSegments.isEmpty() - && tableIdToActiveSegmentRollTimeMs.isEmpty()) { + if (tableIdToLakeFlag.isEmpty() && tableIdToTieredLogLocalSegments.isEmpty()) { return; } @@ -608,11 +601,6 @@ private void updateReplicaTableConfig(ClusterMetadata clusterMetadata) { replica.updateTieredLogLocalSegments( tableIdToTieredLogLocalSegments.get(tableId)); } - - if (tableIdToActiveSegmentRollTimeMs.containsKey(tableId)) { - replica.updateActiveSegmentRollTimeMs( - tableIdToActiveSegmentRollTimeMs.get(tableId)); - } } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index 7aa2f1aea4..8a6e9fa2c9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -126,7 +126,6 @@ public static void validateTableDescriptor( checkMergeEngine(tableConf, hasPrimaryKey, schema); checkDeleteBehavior(tableConf, hasPrimaryKey); checkTieredLog(tableConf); - checkActiveSegmentRollTime(tableConf); checkPartition(tableConf, tableDescriptor.getPartitionKeys(), schema.getRowType()); checkSystemColumns(schema.getRowType()); validateStatisticsConfig(tableDescriptor); @@ -460,22 +459,6 @@ private static void checkTieredLog(Configuration tableConf) { } } - private static void checkActiveSegmentRollTime(Configuration tableConf) { - Optional activeRollTime = - tableConf.getOptional(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME); - if (!activeRollTime.isPresent()) { - return; - } - - long activeRollTimeMs = activeRollTime.get().toMillis(); - if (activeRollTimeMs <= 0) { - throw new InvalidConfigException( - String.format( - "'%s' must be greater than 0.", - ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key())); - } - } - private static void checkPartition( Configuration tableConf, List partitionKeys, RowType rowType) { boolean isPartitioned = !partitionKeys.isEmpty(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java index adca827fde..872682ad26 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java @@ -224,74 +224,6 @@ void testFollowerFetchAlreadyMoveToRemoteLog(boolean withWriterId) throws Except FLUSS_CLUSTER_EXTENSION.waitUntilReplicaExpandToIsr(tb, follower); } - @Test - void testActiveSegmentRollTimeWithAlterTable() throws Exception { - TablePath tablePath = TablePath.of("fluss", "test_active_segment_roll_time_with_alter"); - TableDescriptor tableDescriptor = - TableDescriptor.builder() - .schema(DATA1_SCHEMA) - .distributedBy(1) - .property(ConfigOptions.TABLE_LOG_TTL, Duration.ofDays(7)) - .build(); - - long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, tableDescriptor); - TableBucket tb = new TableBucket(tableId, 0); - FLUSS_CLUSTER_EXTENSION.waitUntilAllReplicaReady(tb); - - int leaderId = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(tb); - TabletServerGateway leaderGateway = - FLUSS_CLUSTER_EXTENSION.newTabletServerClientForNode(leaderId); - assertProduceLogResponse( - leaderGateway - .produceLog( - newProduceLogRequest( - tb.getTableId(), 0, 1, genMemoryLogRecordsByObject(DATA1))) - .get(), - 0, - 0L); - - TabletServer tabletServer = FLUSS_CLUSTER_EXTENSION.getTabletServerById(leaderId); - RemoteLogManager remoteLogManager = tabletServer.getReplicaManager().getRemoteLogManager(); - RemoteLogTablet remoteLogTablet = remoteLogManager.remoteLogTablet(tb); - Replica leaderReplica = FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(tb); - LogTablet logTablet = leaderReplica.getLogTablet(); - - assertThat(remoteLogTablet.allRemoteLogSegments()).isEmpty(); - assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); - assertThat(logTablet.getActiveSegmentRollTimeMs()).isEqualTo(Duration.ofDays(7).toMillis()); - - MANUAL_CLOCK.advanceTime(Duration.ofHours(2)); - retry( - Duration.ofSeconds(5), - () -> assertThat(remoteLogTablet.allRemoteLogSegments()).isEmpty()); - - CoordinatorGateway coordinatorGateway = FLUSS_CLUSTER_EXTENSION.newCoordinatorClient(); - coordinatorGateway - .alterTable( - newAlterTableRequest( - tablePath, - Collections.singletonMap( - ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), - "1h"), - Collections.emptyList(), - Collections.emptyList(), - false)) - .get(); - - retry( - Duration.ofMinutes(1), - () -> - assertThat(logTablet.getActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofHours(1).toMillis())); - retry( - Duration.ofMinutes(2), - () -> { - assertThat(remoteLogTablet.allRemoteLogSegments()).hasSize(1); - assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); - assertThat(logTablet.activeLogSegment().getSizeInBytes()).isZero(); - }); - } - @Test void testRemoteLogTTLWithDynamicLakeToggle() throws Exception { TablePath tablePath = TablePath.of("fluss", "test_remote_log_ttl_dynamic_lake"); @@ -383,31 +315,31 @@ void testRemoteLogTTLWithDynamicLakeToggle() throws Exception { long partialLakeOffset = sortedSegments.get(midIndex).remoteLogEndOffset(); logTablet.updateLakeLogEndOffset(partialLakeOffset); + final int expectedRemainingSegments = sortedSegments.size() - midIndex - 1; + // The new remoteLogStartOffset should be the start offset of the first remaining segment + final long expectedNewStartOffset = sortedSegments.get(midIndex + 1).remoteLogStartOffset(); + // Wait for partial cleanup - only segments that have been tiered should be deleted retry( Duration.ofMinutes(2), () -> { // Some segments should be deleted (those with endOffset <= partialLakeOffset) int currentSegmentCount = remoteLogTablet.allRemoteLogSegments().size(); - assertThat(currentSegmentCount) - .isGreaterThan(0) - .isLessThan(sortedSegments.size()); + assertThat(currentSegmentCount).isEqualTo(expectedRemainingSegments); + // Remote log start offset should be updated to the first remaining segment's + // start + assertThat(remoteLogTablet.getRemoteLogStartOffset()) + .isEqualTo(expectedNewStartOffset); // Remaining segments should have remoteLogEndOffset > partialLakeOffset assertThat(remoteLogTablet.allRemoteLogSegments()) .allSatisfy( segment -> assertThat(segment.remoteLogEndOffset()) .isGreaterThan(partialLakeOffset)); - assertThat(remoteLogTablet.getRemoteLogStartOffset()) - .isEqualTo( - remoteLogTablet.allRemoteLogSegments().stream() - .mapToLong(RemoteLogSegment::remoteLogStartOffset) - .min() - .orElseThrow(IllegalStateException::new)); }); // Step 2: Fully update lake log end offset to trigger complete cleanup - logTablet.updateLakeLogEndOffset(remoteLogTablet.getRemoteLogEndOffset().orElse(-1L)); + logTablet.updateLakeLogEndOffset(stageBRemoteLogEndOffset); // Wait for complete cleanup - all segments should be deleted retry( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java index 45c430a3c5..d23667431f 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTTLTest.java @@ -17,36 +17,23 @@ package org.apache.fluss.server.log.remote; -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.server.entity.FetchReqInfo; -import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.log.FetchParams; import org.apache.fluss.server.log.LogTablet; -import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.time.Duration; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; -import static org.apache.fluss.record.TestData.DATA1; -import static org.apache.fluss.record.TestData.DATA1_SCHEMA; import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; -import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH; -import static org.apache.fluss.server.zk.data.LeaderAndIsr.INITIAL_BUCKET_EPOCH; -import static org.apache.fluss.server.zk.data.LeaderAndIsr.INITIAL_LEADER_EPOCH; -import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsWithWriterId; import static org.assertj.core.api.Assertions.assertThat; /** Test for remote log ttl in {@link RemoteLogManager}. */ @@ -85,13 +72,11 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { // advance time past TTL (7 days) manualClock.advanceTime(Duration.ofDays(7).plusHours(1)); - // Since data lake is enabled and no data has been tiered to data lake, - // the expired segments should not be deleted. The expired active segment is rolled - // and uploaded in this task run. + // since data lake is enabled and no data has been tiered to data lake, + // the expired segments should not be deleted. remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); - assertThat(remoteLog.allRemoteLogSegments()).hasSize(5); + assertThat(remoteLog.allRemoteLogSegments()).hasSize(4); assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(0L); - assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(50L); // set lake log end offset to 20, meaning only the first 2 segments // ([0,10) and [10,20)) have been tiered to lake @@ -101,10 +86,10 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); // only segments with remoteLogEndOffset <= 20 should be deleted (first 2 segments) - // remaining segments: [20,30), [30,40) and [40,50) - assertThat(remoteLog.allRemoteLogSegments()).hasSize(3); + // remaining segments: [20,30) and [30,40) + assertThat(remoteLog.allRemoteLogSegments()).hasSize(2); assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(20L); - assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(50L); + assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(40L); // verify remaining segments have the expected offsets assertThat(remoteLog.allRemoteLogSegments()) .allSatisfy( @@ -114,14 +99,7 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { // now advance lake log end offset to include all remaining segments logTablet.updateLakeLogEndOffset(40L); - // trigger again, segments whose end offset is <= 40 should now be deleted - remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); - assertThat(remoteLog.allRemoteLogSegments()).hasSize(1); - assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(40L); - assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(50L); - - logTablet.updateLakeLogEndOffset(50L); - // trigger again, all remaining expired segments should now be deleted + // trigger again, remaining expired segments should now be deleted remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); assertThat(remoteLog.allRemoteLogSegments()).isEmpty(); assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(Long.MAX_VALUE); @@ -129,8 +107,8 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { // Fetch records from remote. // mock to update remote log end offset and remote log start offset as // NotifyRemoteLogOffsetsRequest do. - logTablet.updateRemoteLogStartOffset(50L); - logTablet.updateRemoteLogEndOffset(50L); + logTablet.updateRemoteLogStartOffset(40L); + logTablet.updateRemoteLogEndOffset(40L); CompletableFuture> future = new CompletableFuture<>(); replicaManager.fetchLogRecords( @@ -144,82 +122,4 @@ void testRemoteLogTTL(boolean partitionTable) throws Exception { assertThat(resultForBucket.getErrorCode()) .isEqualTo(Errors.LOG_OFFSET_OUT_OF_RANGE_EXCEPTION.code()); } - - @Test - void testActiveSegmentRollTimeOverride() throws Exception { - long tableId = 10001L; - TablePath tablePath = TablePath.of("fluss", "test_active_segment_roll_time_override"); - Map properties = new HashMap<>(); - properties.put(ConfigOptions.TABLE_LOG_TTL.key(), "7d"); - properties.put(ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME.key(), "1h"); - registerTableInZkClient( - tablePath, DATA1_SCHEMA, tableId, Collections.emptyList(), properties); - - TableBucket tb = new TableBucket(tableId, 0); - makeLeaderAndFollower( - Collections.singletonList( - new NotifyLeaderAndIsrData( - PhysicalTablePath.of(tablePath), - tb, - Collections.singletonList(TABLET_SERVER_ID), - new LeaderAndIsr( - TABLET_SERVER_ID, - INITIAL_LEADER_EPOCH, - Collections.singletonList(TABLET_SERVER_ID), - Collections.emptyList(), - INITIAL_COORDINATOR_EPOCH, - INITIAL_BUCKET_EPOCH)))); - LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); - - addMultiSegmentsToLogTablet(logTablet, 1); - assertThat(logTablet.getSegments()).hasSize(1); - assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); - - manualClock.advanceTime(Duration.ofHours(2)); - remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); - - RemoteLogTablet remoteLog = remoteLogManager.remoteLogTablet(tb); - assertThat(remoteLog.allRemoteLogSegments()).hasSize(1); - assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(0L); - assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(10L); - assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); - assertThat(logTablet.activeLogSegment().getSizeInBytes()).isZero(); - } - - @Test - void testExpiredActiveSegmentRolledUploadedAndLocallyDeleted() throws Exception { - TableBucket tb = new TableBucket(DATA1_TABLE_ID, 0); - makeLogTableAsLeader(tb, false); - LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); - - addMultiSegmentsToLogTablet(logTablet, 1); - assertThat(logTablet.getSegments()).hasSize(1); - assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); - - manualClock.advanceTime(Duration.ofDays(7).plusHours(1)); - remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); - - RemoteLogTablet remoteLog = remoteLogManager.remoteLogTablet(tb); - assertThat(remoteLog.allRemoteLogSegments()).hasSize(1); - assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(0L); - assertThat(remoteLog.getRemoteLogEndOffset()).hasValue(10L); - assertThat(logTablet.getSegments()).hasSize(1); - assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); - assertThat(logTablet.activeLogSegment().getSizeInBytes()).isZero(); - - logTablet.appendAsLeader( - genMemoryLogRecordsWithWriterId( - Collections.singletonList(DATA1.get(0)), - manualClock.milliseconds(), - 0, - 10L)); - - assertThat(logTablet.localLogEndOffset()).isEqualTo(11L); - assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(10L); - assertThat(logTablet.activeLogSegment().getSizeInBytes()).isGreaterThan(0); - - remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); - assertThat(remoteLog.allRemoteLogSegments()).isEmpty(); - assertThat(remoteLog.getRemoteLogStartOffset()).isEqualTo(Long.MAX_VALUE); - } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java new file mode 100644 index 0000000000..61fcea0edc --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log.remote; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.server.log.LogTablet; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.time.Duration; +import java.util.Collections; + +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests TTL-based cleanup of inactive local segments retained by tiered storage. */ +final class TieredLocalSegmentTtlTest extends RemoteLogTestBase { + + @BeforeEach + public void setup() throws Exception { + super.setup(); + registerTableInZkClient( + DATA1_TABLE_PATH, + DATA1_SCHEMA, + DATA1_TABLE_ID, + Collections.emptyList(), + Collections.singletonMap(ConfigOptions.TABLE_LOG_TTL.key(), "50ms")); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throws Exception { + TableBucket tb = + partitionTable + ? new TableBucket(DATA1_TABLE_ID, 0L, 0) + : new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tb, partitionTable); + LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); + + addMultiSegmentsToLogTablet(logTablet, 5); + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + + // The configured number of local segments is retained before their TTL expires. + assertThat(logTablet.getSegments()).hasSize(2); + + manualClock.advanceTime(Duration.ofMillis(200)); + logTablet.deleteSegmentsAlreadyExistsInRemote(); + + // The inactive segment is expired and deleted, while the active segment is retained. + assertThat(logTablet.getSegments()).hasSize(1); + assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(40L); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index 94861b7314..b738c5a859 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -33,7 +33,6 @@ import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.SchemaGetter; import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.ChangeType; @@ -100,7 +99,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -2172,63 +2170,6 @@ void testUpdateMetadata() throws Exception { + "The latest known coordinator epoch is 2"); } - @Test - void testUpdateActiveSegmentRollTimeFromMetadataCache() throws Exception { - TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1); - makeLogTableAsLeader(tb.getBucket()); - Replica replica = replicaManager.getReplicaOrException(tb); - assertThat(replica.getLogTablet().getActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofDays(7).toMillis()); - - TableDescriptor updatedTableDescriptor = - TableDescriptor.builder() - .schema(DATA1_SCHEMA) - .distributedBy(3) - .property( - ConfigOptions.TABLE_LOG_SEGMENT_ACTIVE_ROLL_TIME, - Duration.ofHours(1)) - .build(); - TableInfo tableInfo = - TableInfo.of( - DATA1_TABLE_PATH, - DATA1_TABLE_ID, - 1, - updatedTableDescriptor, - DEFAULT_REMOTE_DATA_DIR, - System.currentTimeMillis(), - System.currentTimeMillis()); - TableMetadata tableMetadata = new TableMetadata(tableInfo, Collections.emptyList()); - Set tsServerInfoList = - new HashSet<>( - Arrays.asList( - new ServerInfo( - TABLET_SERVER_ID, - "rack1", - Endpoint.fromListenersString("CLIENT://localhost:90"), - ServerType.TABLET_SERVER), - new ServerInfo( - 2, - "rack2", - Endpoint.fromListenersString("CLIENT://localhost:91"), - ServerType.TABLET_SERVER), - new ServerInfo( - 3, - "rack3", - Endpoint.fromListenersString("CLIENT://localhost:92"), - ServerType.TABLET_SERVER))); - - replicaManager.maybeUpdateMetadataCache( - 0, - buildClusterMetadata( - null, - tsServerInfoList, - Collections.singletonList(tableMetadata), - Collections.emptyList())); - - assertThat(replica.getLogTablet().getActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofHours(1).toMillis()); - } - @Test void testGetReplicaOrException() { // 1. Test online replica diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index d3ae2ca38d..4a0f532bbc 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -854,7 +854,8 @@ private long getLakeTieringGaugeValue(Replica replica, String metricName) { (Gauge) ((AbstractMetricGroup) lakeTieringMetricGroup).getMetrics().get(metricName); return gauge.getValue(); - + } + private void makeLogReplicaAsLeader(Replica replica) throws Exception { makeLeaderReplica( replica, From 29c4d6ecfcc5b3f6c94ab9c44f878c6dcb3998aa Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 13 Jul 2026 17:46:51 +0800 Subject: [PATCH 10/20] fix test --- .../java/org/apache/fluss/server/log/LogTablet.java | 8 +++++--- .../server/log/remote/TieredLocalSegmentTtlTest.java | 12 +++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 24cc7b7974..e69d023878 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -627,10 +627,12 @@ public void updateRemoteLogSize(long remoteLogSize) { public void updateRemoteLogEndOffset(long remoteLogEndOffset) { if (remoteLogEndOffset > this.remoteLogEndOffset) { this.remoteLogEndOffset = remoteLogEndOffset; - - // try to delete these segments already exist in remote storage. - deleteSegmentsAlreadyExistsInRemote(); } + + // Try to delete segments that have already existed in remote storage on every retention + // pass. A remote TTL cleanup may leave the manifest end offset unchanged or decrease it, + // but locally retained inactive segments may have expired in the meantime. + deleteSegmentsAlreadyExistsInRemote(); } public void updateMinRetainOffset(long minRetainOffset) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java index 61fcea0edc..67d2ae8d9e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java @@ -44,7 +44,7 @@ public void setup() throws Exception { DATA1_SCHEMA, DATA1_TABLE_ID, Collections.emptyList(), - Collections.singletonMap(ConfigOptions.TABLE_LOG_TTL.key(), "50ms")); + Collections.singletonMap(ConfigOptions.TABLE_LOG_TTL.key(), "1h")); } @ParameterizedTest @@ -59,15 +59,21 @@ void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throw addMultiSegmentsToLogTablet(logTablet, 5); remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + RemoteLogTablet remoteLog = remoteLogManager.remoteLogTablet(tb); // The configured number of local segments is retained before their TTL expires. assertThat(logTablet.getSegments()).hasSize(2); + assertThat(logTablet.localLogStartOffset()).isEqualTo(30L); + assertThat(remoteLog.allRemoteLogSegments()).hasSize(4); - manualClock.advanceTime(Duration.ofMillis(200)); - logTablet.deleteSegmentsAlreadyExistsInRemote(); + manualClock.advanceTime(Duration.ofHours(2)); + // Trigger a TTL-only retention pass without appending or uploading another segment. + remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); // The inactive segment is expired and deleted, while the active segment is retained. + assertThat(remoteLog.allRemoteLogSegments()).isEmpty(); assertThat(logTablet.getSegments()).hasSize(1); + assertThat(logTablet.localLogStartOffset()).isEqualTo(40L); assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(40L); } } From f1434d060999481beb213ba33a976d15da558ddb Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 14 Jul 2026 15:49:18 +0800 Subject: [PATCH 11/20] fix review --- .../fluss/flink/source/FlinkTableSourceBatchITCase.java | 8 ++++++++ ...SegmentTtlTest.java => TieredLocalSegmentTTLTest.java} | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) rename fluss-server/src/test/java/org/apache/fluss/server/log/remote/{TieredLocalSegmentTtlTest.java => TieredLocalSegmentTTLTest.java} (98%) diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java index f7f5c688b3..3257640615 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java @@ -61,9 +61,12 @@ abstract class FlinkTableSourceBatchITCase extends FlinkTestBase { static final String CATALOG_NAME = "testcatalog"; protected StreamTableEnvironment tEnv; private String databaseName; + private boolean databaseCreated; @BeforeEach void before() { + databaseName = null; + databaseCreated = false; StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); // create table environment tEnv = StreamTableEnvironment.create(execEnv, EnvironmentSettings.inBatchMode()); @@ -77,13 +80,18 @@ void before() { tEnv.getConfig().set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4); databaseName = "defaultdb_" + RandomUtils.nextInt(); tEnv.executeSql("create database " + databaseName); + databaseCreated = true; tEnv.useDatabase(databaseName); } @AfterEach void after() { + if (tEnv == null || !databaseCreated) { + return; + } tEnv.useDatabase(BUILTIN_DATABASE); tEnv.executeSql(String.format("drop database %s cascade", databaseName)); + databaseCreated = false; } @Test diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java similarity index 98% rename from fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java rename to fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java index 67d2ae8d9e..fff1650fd7 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTtlTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java @@ -34,7 +34,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** Tests TTL-based cleanup of inactive local segments retained by tiered storage. */ -final class TieredLocalSegmentTtlTest extends RemoteLogTestBase { +final class TieredLocalSegmentTTLTest extends RemoteLogTestBase { @BeforeEach public void setup() throws Exception { From a918898c6d041253cd44b9a1c4c947295c6be920 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 15 Jul 2026 15:32:54 +0800 Subject: [PATCH 12/20] fix review --- .../apache/fluss/config/ConfigOptions.java | 9 ++++ .../apache/fluss/server/log/LogManager.java | 52 +++++++++++++++++++ .../apache/fluss/server/log/LogTablet.java | 5 -- .../fluss/server/log/LogManagerTest.java | 13 ++++- .../log/remote/TieredLocalSegmentTTLTest.java | 7 +-- .../fluss/server/replica/ReplicaTest.java | 2 +- 6 files changed, 78 insertions(+), 10 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 7ceee8fb5d..05d1941ca5 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -947,6 +947,15 @@ public class ConfigOptions { + "flush which acts as the log recovery point. The default " + "setting is 60 seconds."); + public static final ConfigOption LOG_RETENTION_CHECK_INTERVAL = + key("log.retention-check-interval") + .durationType() + .defaultValue(Duration.ofMinutes(5)) + .withDescription( + "The frequency with which the log manager checks whether tiered local " + + "log segments are eligible for retention cleanup. If the value " + + "is set to 0, the periodic cleanup is disabled."); + public static final ConfigOption LOG_REPLICA_HIGH_WATERMARK_CHECKPOINT_INTERVAL = key("log.replica.high-watermark.checkpoint-interval") .durationType() diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index 9172243edb..cfd8d7ab81 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -109,6 +109,7 @@ public final class LogManager extends TabletManagerBase { private volatile Map recoveryPointCheckpoints; private volatile ScheduledFuture recoveryPointCheckpointTask; + private volatile ScheduledFuture localLogRetentionTask; private boolean loadLogsCompletedFlag = false; private LogManager( @@ -162,6 +163,19 @@ public void startup() { this::checkpointRecoveryOffsets, checkpointIntervalMs, checkpointIntervalMs); + + long retentionIntervalMs = conf.get(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL).toMillis(); + if (retentionIntervalMs > 0L) { + LOG.info( + "Starting tiered local log retention with a period of {} ms.", + retentionIntervalMs); + localLogRetentionTask = + scheduler.schedule( + "fluss-tiered-local-log-retention", + this::cleanupTieredLocalLogSegments, + retentionIntervalMs, + retentionIntervalMs); + } } private void initializeCheckpointMaps() throws IOException { @@ -486,6 +500,10 @@ private LogTablet loadLog( /** Close all the logs. */ public void shutdown() { LOG.info("Shutting down LogManager."); + if (localLogRetentionTask != null) { + localLogRetentionTask.cancel(false); + localLogRetentionTask = null; + } if (recoveryPointCheckpointTask != null) { recoveryPointCheckpointTask.cancel(false); recoveryPointCheckpointTask = null; @@ -535,6 +553,40 @@ private void closeLogTablet(LogTablet logTablet) { } } + /** Runs one retention pass for tiered local log segments of all loaded tablets. */ + public void cleanupTieredLocalLogSegments() { + for (LogTablet logTablet : currentLogs.values()) { + try { + logTablet.deleteSegmentsAlreadyExistsInRemote(); + } catch (Exception e) { + LOG.error( + "Failed to clean up tiered local log segments for table bucket {}.", + logTablet.getTableBucket(), + e); + } + } + } + + private LogShutdownTask shutdownLogsInDir(File dataDir, List logs) { + String dataDirAbsolutePath = dataDir.getAbsolutePath(); + LOG.info("Shutting down {} logs in dir {}", logs.size(), dataDirAbsolutePath); + + List> jobsForTabletDir = new ArrayList<>(); + ExecutorService pool = createThreadPool("log-tablet-closing-" + dataDirAbsolutePath); + for (LogTablet logTablet : logs) { + Runnable runnable = + () -> { + try { + logTablet.flush(true); + logTablet.close(); + } catch (IOException e) { + throw new FlussRuntimeException(e); + } + }; + jobsForTabletDir.add(pool.submit(runnable)); + } + } + private void waitForShutdownLogsInDir( File dataDir, List logs, CompletableFuture closingFuture) { boolean closeSucceeded; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index e69d023878..1cfe1f4d45 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -628,11 +628,6 @@ public void updateRemoteLogEndOffset(long remoteLogEndOffset) { if (remoteLogEndOffset > this.remoteLogEndOffset) { this.remoteLogEndOffset = remoteLogEndOffset; } - - // Try to delete segments that have already existed in remote storage on every retention - // pass. A remote TTL cleanup may leave the manifest end offset unchanged or decrease it, - // but locally retained inactive segments may have expired in the meantime. - deleteSegmentsAlreadyExistsInRemote(); } public void updateMinRetainOffset(long minRetainOffset) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java index 81d20249b7..3df6a357e9 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java @@ -386,7 +386,7 @@ void testCheckpointRecoveryPointsAreWrittenPerDirectory() throws Exception { } @Test - void testPeriodicRecoveryPointCheckpoint() throws Exception { + void testPeriodicLogManagerTasks() throws Exception { logManager.shutdown(); logManager = null; localDiskManager.close(); @@ -409,6 +409,12 @@ void testPeriodicRecoveryPointCheckpoint() throws Exception { assertThat(checkpointTask.getDelayMs()).isEqualTo(60_000L); assertThat(checkpointTask.getPeriodMs()).isEqualTo(60_000L); + RecordingScheduledTask retentionTask = + scheduler.getTask("fluss-tiered-local-log-retention"); + assertThat((Object) retentionTask).isNotNull(); + assertThat(retentionTask.getDelayMs()).isEqualTo(300_000L); + assertThat(retentionTask.getPeriodMs()).isEqualTo(300_000L); + initTableBuckets(null); LogTablet log1 = getOrCreateLog(tablePath1, null, tableBucket1); log1.appendAsLeader(genMemoryLogRecordsByObject(DATA1)); @@ -421,6 +427,11 @@ void testPeriodicRecoveryPointCheckpoint() throws Exception { new File(tempDir, LogManager.RECOVERY_POINT_CHECKPOINT_FILE)) .read(); assertThat(checkpoints).containsEntry(tableBucket1, log1.getRecoveryPoint()); + + logManager.shutdown(); + logManager = null; + assertThat(checkpointTask.isCancelled()).isTrue(); + assertThat(retentionTask.isCancelled()).isTrue(); } @Test diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java index fff1650fd7..fa45cadd34 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java @@ -60,6 +60,7 @@ void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throw addMultiSegmentsToLogTablet(logTablet, 5); remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); RemoteLogTablet remoteLog = remoteLogManager.remoteLogTablet(tb); + logManager.cleanupTieredLocalLogSegments(); // The configured number of local segments is retained before their TTL expires. assertThat(logTablet.getSegments()).hasSize(2); @@ -67,11 +68,11 @@ void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throw assertThat(remoteLog.allRemoteLogSegments()).hasSize(4); manualClock.advanceTime(Duration.ofHours(2)); - // Trigger a TTL-only retention pass without appending or uploading another segment. - remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); + // Run local retention without updating the remote manifest or uploading another segment. + logManager.cleanupTieredLocalLogSegments(); // The inactive segment is expired and deleted, while the active segment is retained. - assertThat(remoteLog.allRemoteLogSegments()).isEmpty(); + assertThat(remoteLog.allRemoteLogSegments()).hasSize(4); assertThat(logTablet.getSegments()).hasSize(1); assertThat(logTablet.localLogStartOffset()).isEqualTo(40L); assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(40L); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index 4a0f532bbc..bb3f921ec0 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -855,7 +855,7 @@ private long getLakeTieringGaugeValue(Replica replica, String metricName) { ((AbstractMetricGroup) lakeTieringMetricGroup).getMetrics().get(metricName); return gauge.getValue(); } - + private void makeLogReplicaAsLeader(Replica replica) throws Exception { makeLeaderReplica( replica, From 69349f4859029d20f53c2dca9f84e90aa247800e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 15 Jul 2026 18:16:41 +0800 Subject: [PATCH 13/20] fix review --- .../src/main/java/org/apache/fluss/server/log/LogTablet.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 1cfe1f4d45..3d494a54d7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -618,6 +618,9 @@ public void updateRemoteLogStartOffset(long remoteLogStartOffset) { if (prev == Long.MAX_VALUE || remoteLogStartOffset > prev) { this.remoteLogStartOffset = remoteLogStartOffset; } + + // try to delete these segments already exist in remote storage. + deleteSegmentsAlreadyExistsInRemote(); } public void updateRemoteLogSize(long remoteLogSize) { From db33a8aa5bf5de92541d3db8ce635bc78c1727cb Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 15 Jul 2026 19:39:14 +0800 Subject: [PATCH 14/20] fix review --- .../source/FlinkTableSourceBatchITCase.java | 54 ++++++++----------- .../apache/fluss/server/log/LogTablet.java | 8 +-- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java index 3257640615..51d05fa036 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java @@ -59,14 +59,11 @@ abstract class FlinkTableSourceBatchITCase extends FlinkTestBase { static final String CATALOG_NAME = "testcatalog"; + static final String DEFAULT_DB = "defaultdb"; protected StreamTableEnvironment tEnv; - private String databaseName; - private boolean databaseCreated; @BeforeEach void before() { - databaseName = null; - databaseCreated = false; StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); // create table environment tEnv = StreamTableEnvironment.create(execEnv, EnvironmentSettings.inBatchMode()); @@ -78,20 +75,15 @@ void before() { tEnv.executeSql("use catalog " + CATALOG_NAME); tEnv.getConfig().set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4); - databaseName = "defaultdb_" + RandomUtils.nextInt(); - tEnv.executeSql("create database " + databaseName); - databaseCreated = true; - tEnv.useDatabase(databaseName); + // create database + tEnv.executeSql("create database " + DEFAULT_DB); + tEnv.useDatabase(DEFAULT_DB); } @AfterEach void after() { - if (tEnv == null || !databaseCreated) { - return; - } tEnv.useDatabase(BUILTIN_DATABASE); - tEnv.executeSql(String.format("drop database %s cascade", databaseName)); - databaseCreated = false; + tEnv.executeSql(String.format("drop database %s cascade", DEFAULT_DB)); } @Test @@ -102,10 +94,10 @@ void testScanSingleRowFilter() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, %s, %s, " + "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "filter=[and(=(id, 1), =(name, _UTF-16LE'name1':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name])", - databaseName, tableName)); + tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, address1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -119,10 +111,10 @@ void testScanSingleRowFilter2() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, %s, %s, " + "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "filter=[and(=(id, 1), =(name, _UTF-16LE'name1':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name])", - databaseName, tableName)); + tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, address1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -136,10 +128,10 @@ void testScanSingleRowFilter3() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, %s, %s, " + "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "filter=[=(id, 1)], " + "project=[id, name]]], fields=[id, name])", - databaseName, tableName)); + tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -148,7 +140,7 @@ void testScanSingleRowFilter3() throws Exception { @Test void testScanSingleRowFilterOnPartitionedTable() throws Exception { String tableName = prepareSourceTable(new String[] {"id", "dt"}, "dt"); - TablePath tablePath = TablePath.of(databaseName, tableName); + TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), tablePath); Iterator partitionIterator = @@ -160,10 +152,10 @@ void testScanSingleRowFilterOnPartitionedTable() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, %s, %s, " + "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "filter=[and(=(id, 1), =(dt, _UTF-16LE'%s':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name, dt])\n", - databaseName, tableName, partition1)); + tableName, partition1)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = @@ -202,7 +194,7 @@ void testFilterOnLookupSource() throws Exception { + " 'table.auto-partition.time-unit' = 'year')", dimTableName)); - TablePath srcTablePath = TablePath.of(databaseName, srcTableName); + TablePath srcTablePath = TablePath.of(DEFAULT_DB, srcTableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), srcTablePath); // just pick first partition to insert data @@ -220,7 +212,7 @@ void testFilterOnLookupSource() throws Exception { upsertWriter.flush(); } - TablePath dimTablePath = TablePath.of(databaseName, dimTableName); + TablePath dimTablePath = TablePath.of(DEFAULT_DB, dimTableName); // prepare dim table data try (Table dimTable = conn.getTable(dimTablePath)) { UpsertWriter upsertWriter = dimTable.newUpsert().createWriter(); @@ -271,7 +263,7 @@ void testLakeTableQueryOnLakeDisabledTable() throws Exception { .isInstanceOf(UnsupportedOperationException.class) .hasMessage( String.format( - "Table %s.%s is not datalake enabled.", databaseName, tableName)); + "Table %s.%s is not datalake enabled.", DEFAULT_DB, tableName)); } @Test @@ -469,9 +461,7 @@ void testCountPushDownWithWALMode() throws Exception { assertThatThrownBy(() -> tEnv.executeSql(query)) .hasRootCauseInstanceOf(InvalidTableException.class) .hasMessageContaining( - String.format( - "Row count is disabled for this table '%s.test_count_table_with_wal'.", - databaseName)); + "Row count is disabled for this table 'defaultdb.test_count_table_with_wal'."); } @ParameterizedTest @@ -556,7 +546,7 @@ private String prepareSourceTable(String[] keys, String partitionedKey) throws E tableName, String.join(",", keys), partitionedKey)); } - TablePath tablePath = TablePath.of(databaseName, tableName); + TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); String partition1 = null; if (partitionedKey != null) { Map partitionNameById = @@ -597,7 +587,7 @@ private String prepareLogTable() throws Exception { + ")", tableName)); - TablePath tablePath = TablePath.of(databaseName, tableName); + TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); // prepare table data with NULL values in address column try (Table table = conn.getTable(tablePath)) { @@ -629,7 +619,7 @@ protected String preparePartitionedLogTable() throws Exception { + " 'table.auto-partition.time-unit' = 'year')", tableName)); - TablePath tablePath = TablePath.of(databaseName, tableName); + TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), tablePath); Collection partitions = partitionNameById.values(); @@ -667,7 +657,7 @@ private String prepareLogTableWithComplexTypes() throws Exception { + ")", tableName)); - TablePath tablePath = TablePath.of(databaseName, tableName); + TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); // prepare table data with complex types try (Table table = conn.getTable(tablePath)) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 3d494a54d7..127194e7a3 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -618,9 +618,6 @@ public void updateRemoteLogStartOffset(long remoteLogStartOffset) { if (prev == Long.MAX_VALUE || remoteLogStartOffset > prev) { this.remoteLogStartOffset = remoteLogStartOffset; } - - // try to delete these segments already exist in remote storage. - deleteSegmentsAlreadyExistsInRemote(); } public void updateRemoteLogSize(long remoteLogSize) { @@ -631,6 +628,9 @@ public void updateRemoteLogEndOffset(long remoteLogEndOffset) { if (remoteLogEndOffset > this.remoteLogEndOffset) { this.remoteLogEndOffset = remoteLogEndOffset; } + + // try to delete these segments already exist in remote storage. + deleteSegmentsAlreadyExistsInRemote(); } public void updateMinRetainOffset(long minRetainOffset) { @@ -1326,6 +1326,8 @@ private List deletableSegments(long endOffset) throws IOException { if (i < tierProtectedStartIndex || isSegmentExpired(now, logSegments.get(i), logTtlMs)) { deletableSegments.add(logSegments.get(i)); + } else { + break; } } return deletableSegments; From 45ea73266d687eb264cf23e29797641afd0fcff3 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 15 Jul 2026 20:16:24 +0800 Subject: [PATCH 15/20] fix review --- .../scanner/log/RemoteLogDownloaderTest.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java index 8984f8fc67..4c2bcdaa12 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteLogDownloaderTest.java @@ -312,7 +312,8 @@ class BlockingFileDownloader extends RemoteFileDownloader { protected long downloadFile(Path targetFilePath, FsPath remoteFilePath) throws IOException { int count = enteredCount.incrementAndGet(); - if (count > 1) { + boolean shouldBlock = count > 1; + if (shouldBlock) { // Block the 2nd and 3rd downloads to simulate in-flight state. inFlightStarted.countDown(); try { @@ -321,9 +322,13 @@ protected long downloadFile(Path targetFilePath, FsPath remoteFilePath) throw new IOException("Interrupted while blocking", e); } } - long downloadBytes = super.downloadFile(targetFilePath, remoteFilePath); - inFlightFinished.countDown(); - return downloadBytes; + try { + return super.downloadFile(targetFilePath, remoteFilePath); + } finally { + if (shouldBlock) { + inFlightFinished.countDown(); + } + } } } @@ -347,8 +352,10 @@ protected long downloadFile(Path targetFilePath, FsPath remoteFilePath) // Wait for the first download to complete (already-downloaded file on disk) and 2 // in-flight downloads to enter the blocked state. // At this point: 1 file downloaded, 2 blocked in-flight, 2 pending. - retry(Duration.ofMinutes(1), () -> assertThat(futures.get(0).isDone()).isTrue()); assertThat(inFlightStarted.await(30, TimeUnit.SECONDS)).isTrue(); + retry( + Duration.ofMinutes(1), + () -> assertThat(futures.subList(0, 3)).anyMatch(future -> future.isDone())); Path localLogDir = downloader.getLocalLogDir(); assertThat(localLogDir.toFile().exists()).isTrue(); @@ -362,7 +369,9 @@ protected long downloadFile(Path targetFilePath, FsPath remoteFilePath) // Wait for 2 in-flight downloads finished. blockLatch.countDown(); assertThat(inFlightFinished.await(30, TimeUnit.SECONDS)).isTrue(); - retry(Duration.ofMinutes(1), () -> assertThat(futures.get(1).isDone()).isTrue()); + retry( + Duration.ofMinutes(1), + () -> assertThat(futures.subList(0, 3)).allMatch(future -> future.isDone())); // Verify that ultimately the local directory does not exist. retry(Duration.ofMinutes(1), () -> assertThat(localLogDir.toFile().exists()).isFalse()); From baca18a5711bd6a6936c84d9bf5fe1b6876425f3 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 20 Jul 2026 15:03:24 +0800 Subject: [PATCH 16/20] ttl delete independently --- .../apache/fluss/config/ConfigOptions.java | 8 +- .../apache/fluss/config/FlussConfigUtils.java | 1 + .../fluss/config/FlussConfigUtilsTest.java | 24 ++++++ .../org/apache/fluss/server/log/LocalLog.java | 9 +++ .../apache/fluss/server/log/LogManager.java | 47 +++--------- .../apache/fluss/server/log/LogTablet.java | 74 +++++++++++++++---- .../log/remote/TieredLocalSegmentTTLTest.java | 48 +++++++++++- .../fluss/server/replica/ReplicaTest.java | 22 ------ 8 files changed, 155 insertions(+), 78 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 05d1941ca5..40d7e0bae2 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -948,13 +948,13 @@ public class ConfigOptions { + "setting is 60 seconds."); public static final ConfigOption LOG_RETENTION_CHECK_INTERVAL = - key("log.retention-check-interval") + key("log.retention.check-interval") .durationType() .defaultValue(Duration.ofMinutes(5)) .withDescription( - "The frequency with which the log manager checks whether tiered local " - + "log segments are eligible for retention cleanup. If the value " - + "is set to 0, the periodic cleanup is disabled."); + "The frequency with which the log manager checks whether local log " + + "segments are eligible for TTL cleanup. The value must be " + + "greater than 0."); public static final ConfigOption LOG_REPLICA_HIGH_WATERMARK_CHECKPOINT_INTERVAL = key("log.replica.high-watermark.checkpoint-interval") diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 4f978a0b30..71bd9702eb 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -220,6 +220,7 @@ protected static void validateServerConfigs(Configuration conf) { validMinValue(conf, ConfigOptions.KV_MAX_RETAINED_SNAPSHOTS, 1); validMinValue(conf, ConfigOptions.SERVER_IO_POOL_SIZE, 1); validMinValue(conf, ConfigOptions.BACKGROUND_THREADS, 1); + validMinDuration(conf, ConfigOptions.LOG_RETENTION_CHECK_INTERVAL, 1); if (conf.get(ConfigOptions.LOG_SEGMENT_FILE_SIZE).getBytes() > Integer.MAX_VALUE) { throw new IllegalConfigurationException( diff --git a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java index f5e4b20ca4..2b9e815ee7 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/FlussConfigUtilsTest.java @@ -191,6 +191,30 @@ void testValidateTabletConfigs() { .hasMessageContaining("it must be greater than or equal 0"); } + @Test + void testValidateLogRetentionCheckInterval() { + assertThat(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL.key()) + .isEqualTo("log.retention.check-interval"); + + Configuration conf = new Configuration(); + conf.set(ConfigOptions.REMOTE_DATA_DIR, "s3://bucket/path"); + + conf.set(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL, Duration.ZERO); + assertThatThrownBy(() -> validateCoordinatorConfigs(conf)) + .isInstanceOf(IllegalConfigurationException.class) + .hasMessageContaining(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL.key()) + .hasMessageContaining("must be greater than or equal 1 ms"); + + conf.set(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL, Duration.ofMillis(-1)); + assertThatThrownBy(() -> validateCoordinatorConfigs(conf)) + .isInstanceOf(IllegalConfigurationException.class) + .hasMessageContaining(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL.key()) + .hasMessageContaining("must be greater than or equal 1 ms"); + + conf.set(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL, Duration.ofMillis(1)); + validateCoordinatorConfigs(conf); + } + @Test void testValidateClientConfigs() { // valid defaults should pass diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java index bb67ea7b82..851763913a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java @@ -683,6 +683,7 @@ public interface SegmentDeletionReason { SegmentDeletionReason LOG_ROLL = new LogRoll(); SegmentDeletionReason LOG_DELETION = new LogDeletion(); SegmentDeletionReason LOG_MOVE_TO_REMOTE = new LogMoveToRemote(); + SegmentDeletionReason LOG_RETENTION = new LogRetention(); void logReason(List toDelete); } @@ -718,4 +719,12 @@ public void logReason(List toDelete) { LOG.info("Deleting segments as the log has been moved to remote: " + toDelete); } } + + /** Delete due to log retention. */ + private static class LogRetention implements SegmentDeletionReason { + @Override + public void logReason(List toDelete) { + LOG.info("Deleting segments due to log retention: " + toDelete); + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index cfd8d7ab81..f18411cbfa 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -165,17 +165,14 @@ public void startup() { checkpointIntervalMs); long retentionIntervalMs = conf.get(ConfigOptions.LOG_RETENTION_CHECK_INTERVAL).toMillis(); - if (retentionIntervalMs > 0L) { - LOG.info( - "Starting tiered local log retention with a period of {} ms.", - retentionIntervalMs); - localLogRetentionTask = - scheduler.schedule( - "fluss-tiered-local-log-retention", - this::cleanupTieredLocalLogSegments, - retentionIntervalMs, - retentionIntervalMs); - } + LOG.info( + "Starting tiered local log retention with a period of {} ms.", retentionIntervalMs); + localLogRetentionTask = + scheduler.schedule( + "fluss-tiered-local-log-retention", + this::cleanupExpiredLocalLogSegments, + retentionIntervalMs, + retentionIntervalMs); } private void initializeCheckpointMaps() throws IOException { @@ -553,40 +550,20 @@ private void closeLogTablet(LogTablet logTablet) { } } - /** Runs one retention pass for tiered local log segments of all loaded tablets. */ - public void cleanupTieredLocalLogSegments() { + /** Runs one TTL retention pass for local log segments of all loaded tablets. */ + public void cleanupExpiredLocalLogSegments() { for (LogTablet logTablet : currentLogs.values()) { try { - logTablet.deleteSegmentsAlreadyExistsInRemote(); + logTablet.deleteExpiredSegments(); } catch (Exception e) { LOG.error( - "Failed to clean up tiered local log segments for table bucket {}.", + "Failed to clean up expired local log segments for table bucket {}.", logTablet.getTableBucket(), e); } } } - private LogShutdownTask shutdownLogsInDir(File dataDir, List logs) { - String dataDirAbsolutePath = dataDir.getAbsolutePath(); - LOG.info("Shutting down {} logs in dir {}", logs.size(), dataDirAbsolutePath); - - List> jobsForTabletDir = new ArrayList<>(); - ExecutorService pool = createThreadPool("log-tablet-closing-" + dataDirAbsolutePath); - for (LogTablet logTablet : logs) { - Runnable runnable = - () -> { - try { - logTablet.flush(true); - logTablet.close(); - } catch (IOException e) { - throw new FlussRuntimeException(e); - } - }; - jobsForTabletDir.add(pool.submit(runnable)); - } - } - private void waitForShutdownLogsInDir( File dataDir, List logs, CompletableFuture closingFuture) { boolean closeSucceeded; diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 127194e7a3..13963f4177 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -627,10 +627,10 @@ public void updateRemoteLogSize(long remoteLogSize) { public void updateRemoteLogEndOffset(long remoteLogEndOffset) { if (remoteLogEndOffset > this.remoteLogEndOffset) { this.remoteLogEndOffset = remoteLogEndOffset; - } - // try to delete these segments already exist in remote storage. - deleteSegmentsAlreadyExistsInRemote(); + // try to delete these segments already exist in remote storage. + deleteSegmentsAlreadyExistsInRemote(); + } } public void updateMinRetainOffset(long minRetainOffset) { @@ -736,7 +736,23 @@ public void loadWriterSnapshot(long lastOffset) throws IOException { } public void deleteSegmentsAlreadyExistsInRemote() { - deleteSegments(remoteLogEndOffset); + deleteSegments( + remoteLogEndOffset, + SegmentDeletionReason.LOG_MOVE_TO_REMOTE, + this::deletableRemoteSegments); + } + + /** Deletes inactive local segments that have expired according to the table log TTL. */ + public void deleteExpiredSegments() { + // A missing remote end offset can mean either that no segment has been uploaded or that + // all remote segments have expired. In both cases, table.log.ttl remains authoritative for + // local retention, while the high watermark and minRetainOffset still protect data that + // cannot be deleted yet. + long cleanupToOffset = remoteLogEndOffset == -1L ? getHighWatermark() : remoteLogEndOffset; + deleteSegments( + cleanupToOffset, + SegmentDeletionReason.LOG_RETENTION, + this::deletableExpiredSegments); } /** @@ -753,7 +769,10 @@ public LogOffsetSnapshot fetchOffsetSnapshot() throws IOException { highWatermark); } - private void deleteSegments(long cleanUpToOffset) { + private void deleteSegments( + long cleanUpToOffset, + SegmentDeletionReason reason, + DeletableSegmentsFinder deletableSegmentsFinder) { // cache to local variables long localLogStartOffset = localLog.getLocalLogStartOffset(); if (cleanUpToOffset < localLogStartOffset) { @@ -779,7 +798,7 @@ private void deleteSegments(long cleanUpToOffset) { try { // shouldn't clean up segments that will be used by kv recovery. long cleanupToOffset = Math.min(minRetainOffset, cleanUpToOffset); - deleteOldSegments(cleanupToOffset, SegmentDeletionReason.LOG_MOVE_TO_REMOTE); + deleteOldSegments(cleanupToOffset, reason, deletableSegmentsFinder); } catch (IOException e) { LOG.error( "Failed to delete the local log segments to cleanUpToOffset {} for table-bucket {}.", @@ -1296,18 +1315,21 @@ snapshot, getTableBucket(), getLogDir().getParent()), } } - private void deleteOldSegments(long endOffset, SegmentDeletionReason reason) + private void deleteOldSegments( + long endOffset, + SegmentDeletionReason reason, + DeletableSegmentsFinder deletableSegmentsFinder) throws IOException { synchronized (lock) { - List deletableSegments = deletableSegments(endOffset); + List deletableSegments = deletableSegmentsFinder.find(endOffset); if (!deletableSegments.isEmpty()) { deleteSegments(deletableSegments, reason); } } } - /** Returns the segments that can be deleted by checking log end offset. */ - private List deletableSegments(long endOffset) throws IOException { + /** Returns uploaded segments that exceed the configured local segment retention count. */ + private List deletableRemoteSegments(long endOffset) { if (localLog.getSegments().isEmpty()) { return Collections.emptyList(); } @@ -1317,22 +1339,44 @@ private List deletableSegments(long endOffset) throws IOException { List deletableSegments = new ArrayList<>(); List logSegments = localLog.getSegments().values(); int tierProtectedStartIndex = logSegments.size() - tieredLogLocalSegments; - long now = clock.milliseconds(); for (int i = 0; i < logSegments.size() - 1; i++) { if (logSegments.get(i + 1).getBaseOffset() > endOffset) { break; } - if (i < tierProtectedStartIndex - || isSegmentExpired(now, logSegments.get(i), logTtlMs)) { - deletableSegments.add(logSegments.get(i)); - } else { + if (i >= tierProtectedStartIndex) { + break; + } + deletableSegments.add(logSegments.get(i)); + } + return deletableSegments; + } + + /** Returns the contiguous prefix of inactive segments that has expired. */ + private List deletableExpiredSegments(long endOffset) throws IOException { + if (localLog.getSegments().isEmpty()) { + return Collections.emptyList(); + } + + List deletableSegments = new ArrayList<>(); + List logSegments = localLog.getSegments().values(); + long now = clock.milliseconds(); + + for (int i = 0; i < logSegments.size() - 1; i++) { + if (logSegments.get(i + 1).getBaseOffset() > endOffset + || !isSegmentExpired(now, logSegments.get(i), logTtlMs)) { break; } + deletableSegments.add(logSegments.get(i)); } return deletableSegments; } + @FunctionalInterface + private interface DeletableSegmentsFinder { + List find(long endOffset) throws IOException; + } + private boolean isSegmentExpired(long now, LogSegment segment, long expirationTimeMs) throws IOException { if (expirationTimeMs <= 0L) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java index fa45cadd34..1ea904ef9a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/TieredLocalSegmentTTLTest.java @@ -60,7 +60,7 @@ void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throw addMultiSegmentsToLogTablet(logTablet, 5); remoteLogTaskScheduler.triggerPeriodicScheduledTasks(); RemoteLogTablet remoteLog = remoteLogManager.remoteLogTablet(tb); - logManager.cleanupTieredLocalLogSegments(); + logManager.cleanupExpiredLocalLogSegments(); // The configured number of local segments is retained before their TTL expires. assertThat(logTablet.getSegments()).hasSize(2); @@ -69,7 +69,7 @@ void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throw manualClock.advanceTime(Duration.ofHours(2)); // Run local retention without updating the remote manifest or uploading another segment. - logManager.cleanupTieredLocalLogSegments(); + logManager.cleanupExpiredLocalLogSegments(); // The inactive segment is expired and deleted, while the active segment is retained. assertThat(remoteLog.allRemoteLogSegments()).hasSize(4); @@ -77,4 +77,48 @@ void testInactiveTieredLocalSegmentRemovedAfterTtl(boolean partitionTable) throw assertThat(logTablet.localLogStartOffset()).isEqualTo(40L); assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(40L); } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testExpiredLocalSegmentsRemovedWithoutRemoteLogEndOffset(boolean partitionTable) + throws Exception { + TableBucket tb = + partitionTable + ? new TableBucket(DATA1_TABLE_ID, 0L, 0) + : new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tb, partitionTable); + LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); + + addMultiSegmentsToLogTablet(logTablet, 5); + assertThat(remoteLogManager.remoteLogTablet(tb).getRemoteLogEndOffset()).isEmpty(); + + manualClock.advanceTime(Duration.ofHours(2)); + logManager.cleanupExpiredLocalLogSegments(); + + assertThat(logTablet.getSegments()).hasSize(1); + assertThat(logTablet.localLogStartOffset()).isEqualTo(40L); + assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(40L); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testTtlCleanupBoundedByRemoteLogEndOffset(boolean partitionTable) throws Exception { + TableBucket tb = + partitionTable + ? new TableBucket(DATA1_TABLE_ID, 0L, 0) + : new TableBucket(DATA1_TABLE_ID, 0); + makeLogTableAsLeader(tb, partitionTable); + LogTablet logTablet = replicaManager.getReplicaOrException(tb).getLogTablet(); + + addMultiSegmentsToLogTablet(logTablet, 5); + logTablet.updateTieredLogLocalSegments(5); + logTablet.updateRemoteLogEndOffset(20L); + + manualClock.advanceTime(Duration.ofHours(2)); + logManager.cleanupExpiredLocalLogSegments(); + + assertThat(logTablet.getSegments()).hasSize(3); + assertThat(logTablet.localLogStartOffset()).isEqualTo(20L); + assertThat(logTablet.activeLogSegment().getBaseOffset()).isEqualTo(40L); + } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index bb3f921ec0..d37e91dc6d 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -825,28 +825,6 @@ void testUpdateIsDataLakeEnabled() throws Exception { assertThat(logReplica.getLogTablet().isDataLakeEnabled()).isFalse(); } - @Test - void testUpdateActiveSegmentRollTimeMs() throws Exception { - Replica logReplica = - makeLogReplica(DATA1_PHYSICAL_TABLE_PATH, new TableBucket(DATA1_TABLE_ID, 1)); - makeLogReplicaAsLeader(logReplica); - - assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofDays(7).toMillis()); - - logReplica.updateActiveSegmentRollTimeMs(Duration.ofHours(1).toMillis()); - assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofHours(1).toMillis()); - - logReplica.updateActiveSegmentRollTimeMs(Duration.ofHours(1).toMillis()); - assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofHours(1).toMillis()); - - logReplica.updateActiveSegmentRollTimeMs(Duration.ofMinutes(30).toMillis()); - assertThat(logReplica.getLogTablet().getActiveSegmentRollTimeMs()) - .isEqualTo(Duration.ofMinutes(30).toMillis()); - } - @SuppressWarnings("unchecked") private long getLakeTieringGaugeValue(Replica replica, String metricName) { MetricGroup lakeTieringMetricGroup = replica.bucketMetrics().addGroup("lakeTiering"); From 5baa48a391fadcbd1c2d79fafea6a4fd13b59bf0 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 20 Jul 2026 16:34:37 +0800 Subject: [PATCH 17/20] fix test failure --- .../source/FlinkTableSourceBatchITCase.java | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java index 51d05fa036..3257640615 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceBatchITCase.java @@ -59,11 +59,14 @@ abstract class FlinkTableSourceBatchITCase extends FlinkTestBase { static final String CATALOG_NAME = "testcatalog"; - static final String DEFAULT_DB = "defaultdb"; protected StreamTableEnvironment tEnv; + private String databaseName; + private boolean databaseCreated; @BeforeEach void before() { + databaseName = null; + databaseCreated = false; StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); // create table environment tEnv = StreamTableEnvironment.create(execEnv, EnvironmentSettings.inBatchMode()); @@ -75,15 +78,20 @@ void before() { tEnv.executeSql("use catalog " + CATALOG_NAME); tEnv.getConfig().set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4); - // create database - tEnv.executeSql("create database " + DEFAULT_DB); - tEnv.useDatabase(DEFAULT_DB); + databaseName = "defaultdb_" + RandomUtils.nextInt(); + tEnv.executeSql("create database " + databaseName); + databaseCreated = true; + tEnv.useDatabase(databaseName); } @AfterEach void after() { + if (tEnv == null || !databaseCreated) { + return; + } tEnv.useDatabase(BUILTIN_DATABASE); - tEnv.executeSql(String.format("drop database %s cascade", DEFAULT_DB)); + tEnv.executeSql(String.format("drop database %s cascade", databaseName)); + databaseCreated = false; } @Test @@ -94,10 +102,10 @@ void testScanSingleRowFilter() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[and(=(id, 1), =(name, _UTF-16LE'name1':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name])", - tableName)); + databaseName, tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, address1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -111,10 +119,10 @@ void testScanSingleRowFilter2() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[and(=(id, 1), =(name, _UTF-16LE'name1':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name])", - tableName)); + databaseName, tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, address1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -128,10 +136,10 @@ void testScanSingleRowFilter3() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[=(id, 1)], " + "project=[id, name]]], fields=[id, name])", - tableName)); + databaseName, tableName)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = Collections.singletonList("+I[1, name1]"); assertResultsIgnoreOrder(collected, expected, true); @@ -140,7 +148,7 @@ void testScanSingleRowFilter3() throws Exception { @Test void testScanSingleRowFilterOnPartitionedTable() throws Exception { String tableName = prepareSourceTable(new String[] {"id", "dt"}, "dt"); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), tablePath); Iterator partitionIterator = @@ -152,10 +160,10 @@ void testScanSingleRowFilterOnPartitionedTable() throws Exception { assertThat(tEnv.explainSql(query)) .contains( String.format( - "TableSourceScan(table=[[testcatalog, defaultdb, %s, " + "TableSourceScan(table=[[testcatalog, %s, %s, " + "filter=[and(=(id, 1), =(dt, _UTF-16LE'%s':VARCHAR(2147483647) CHARACTER SET \"UTF-16LE\"))]]], " + "fields=[id, address, name, dt])\n", - tableName, partition1)); + databaseName, tableName, partition1)); CloseableIterator collected = tEnv.executeSql(query).collect(); List expected = @@ -194,7 +202,7 @@ void testFilterOnLookupSource() throws Exception { + " 'table.auto-partition.time-unit' = 'year')", dimTableName)); - TablePath srcTablePath = TablePath.of(DEFAULT_DB, srcTableName); + TablePath srcTablePath = TablePath.of(databaseName, srcTableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), srcTablePath); // just pick first partition to insert data @@ -212,7 +220,7 @@ void testFilterOnLookupSource() throws Exception { upsertWriter.flush(); } - TablePath dimTablePath = TablePath.of(DEFAULT_DB, dimTableName); + TablePath dimTablePath = TablePath.of(databaseName, dimTableName); // prepare dim table data try (Table dimTable = conn.getTable(dimTablePath)) { UpsertWriter upsertWriter = dimTable.newUpsert().createWriter(); @@ -263,7 +271,7 @@ void testLakeTableQueryOnLakeDisabledTable() throws Exception { .isInstanceOf(UnsupportedOperationException.class) .hasMessage( String.format( - "Table %s.%s is not datalake enabled.", DEFAULT_DB, tableName)); + "Table %s.%s is not datalake enabled.", databaseName, tableName)); } @Test @@ -461,7 +469,9 @@ void testCountPushDownWithWALMode() throws Exception { assertThatThrownBy(() -> tEnv.executeSql(query)) .hasRootCauseInstanceOf(InvalidTableException.class) .hasMessageContaining( - "Row count is disabled for this table 'defaultdb.test_count_table_with_wal'."); + String.format( + "Row count is disabled for this table '%s.test_count_table_with_wal'.", + databaseName)); } @ParameterizedTest @@ -546,7 +556,7 @@ private String prepareSourceTable(String[] keys, String partitionedKey) throws E tableName, String.join(",", keys), partitionedKey)); } - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); String partition1 = null; if (partitionedKey != null) { Map partitionNameById = @@ -587,7 +597,7 @@ private String prepareLogTable() throws Exception { + ")", tableName)); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); // prepare table data with NULL values in address column try (Table table = conn.getTable(tablePath)) { @@ -619,7 +629,7 @@ protected String preparePartitionedLogTable() throws Exception { + " 'table.auto-partition.time-unit' = 'year')", tableName)); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); Map partitionNameById = waitUntilPartitions(FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), tablePath); Collection partitions = partitionNameById.values(); @@ -657,7 +667,7 @@ private String prepareLogTableWithComplexTypes() throws Exception { + ")", tableName)); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + TablePath tablePath = TablePath.of(databaseName, tableName); // prepare table data with complex types try (Table table = conn.getTable(tablePath)) { From 236d7c01ebe0c5b3488455bfdfc3aead09e897c7 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 20 Jul 2026 20:52:40 +0800 Subject: [PATCH 18/20] fix test failure --- .../replica/KvReplicaRestoreITCase.java | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/KvReplicaRestoreITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/KvReplicaRestoreITCase.java index 9aa6142f3b..f876a8b982 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/KvReplicaRestoreITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/KvReplicaRestoreITCase.java @@ -257,14 +257,24 @@ void testRowCountRecoveryAfterFailover() throws Exception { // simulate failure and force failover int currentLeader = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(tableBucket); FLUSS_CLUSTER_EXTENSION.stopTabletServer(currentLeader); - FLUSS_CLUSTER_EXTENSION.startTabletServer(currentLeader); - - // Get the new leader replica - Replica newLeaderReplica = FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(tableBucket); - assertThat(newLeaderReplica.getLeaderId()).isNotEqualTo(currentLeader); - - // Verify the row count is restored correctly after failover and applied changelogs - assertThat(newLeaderReplica.getRowCount()).isEqualTo(recordCount); + try { + AtomicInteger newLeader = new AtomicInteger(-1); + waitUntil( + () -> { + int leader = FLUSS_CLUSTER_EXTENSION.waitAndGetLeader(tableBucket); + newLeader.set(leader); + return leader != currentLeader; + }, + Duration.ofMinutes(2), + "Fail to elect a new leader after stopping tablet server " + currentLeader); + + // Verify the row count is restored correctly after failover and applied changelogs + Replica newLeaderReplica = FLUSS_CLUSTER_EXTENSION.waitAndGetLeaderReplica(tableBucket); + assertThat(newLeaderReplica.getLeaderId()).isEqualTo(newLeader.get()); + assertThat(newLeaderReplica.getRowCount()).isEqualTo(recordCount); + } finally { + FLUSS_CLUSTER_EXTENSION.startTabletServer(currentLeader); + } } private static Configuration initConfig() { From aa2fedf84a952a57c2011b20f0e0697c73444175 Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Tue, 21 Jul 2026 00:33:30 +0800 Subject: [PATCH 19/20] revert code --- .../java/org/apache/fluss/server/log/LogTablet.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 13963f4177..875bef312a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -1338,16 +1338,14 @@ private List deletableRemoteSegments(long endOffset) { // readers is in progress. List deletableSegments = new ArrayList<>(); List logSegments = localLog.getSegments().values(); - int tierProtectedStartIndex = logSegments.size() - tieredLogLocalSegments; - for (int i = 0; i < logSegments.size() - 1; i++) { - if (logSegments.get(i + 1).getBaseOffset() > endOffset) { - break; - } - if (i >= tierProtectedStartIndex) { + // ignore the segments configured to be retained + for (int i = 0; i < logSegments.size() - tieredLogLocalSegments; i++) { + if (logSegments.get(i + 1).getBaseOffset() <= endOffset) { + deletableSegments.add(logSegments.get(i)); + } else { break; } - deletableSegments.add(logSegments.get(i)); } return deletableSegments; } From 46cb366f1bd5ab499a8e3987d973371b3a99403d Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 21 Jul 2026 14:38:05 +0800 Subject: [PATCH 20/20] fix test failure --- .../fluss/client/security/acl/FlussAuthorizationITCase.java | 1 + .../server/coordinator/event/watcher/TableChangeWatcherTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/security/acl/FlussAuthorizationITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/security/acl/FlussAuthorizationITCase.java index fb48154602..e5538f3f99 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/security/acl/FlussAuthorizationITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/security/acl/FlussAuthorizationITCase.java @@ -146,6 +146,7 @@ protected void setup() throws Exception { guestPrincipal = new FlussPrincipal("guest", "User"); // prepare default database and table + FLUSS_CLUSTER_EXTENSION.assertHasTabletServerNumber(3); rootAdmin .createDatabase( DATA1_TABLE_PATH_PK.getDatabaseName(), DatabaseDescriptor.EMPTY, true) diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java index c181e44423..1289f4852c 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java @@ -187,6 +187,7 @@ void testTableChanges() { // as it appears so the next pending drop is admitted. for (CoordinatorEvent event : expectedTableEvents) { DropTableEvent drop = (DropTableEvent) event; + retry(Duration.ofMinutes(1), () -> assertThat(eventManager.getEvents()).contains(drop)); lifecycleThrottler.onTableDropCompleted(drop.getTableId()); }