From 02e8439eedbca3306657cf38256e481123eeee78 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Sun, 13 Sep 2026 23:00:16 +0300 Subject: [PATCH 01/17] HIVE-30059: Layout and read buffers for Parquet column chunks in the LLAP cache Whoever puts a Parquet column chunk in the LLAP cache decides how the file is cut into cacheable pieces. Today that is the cache-aware stream under the vectorized reader; the native reader added later does the same. If the two cut a file differently, a chunk cached by one is not reusable by the other and the same bytes end up held twice under different keys. Define the layout once, before either uses it: the piece sizes, the largest range a vectored read may ask for, and the pooled buffers those ranges are read into. Pieces are powers of two, largest first, so each fills its buddy allocation exactly. The cache-aware stream cached a chunk as one buffer and left the allocator to round it up, so a 5 Mb chunk took 8 Mb; over a mix of chunk sizes that lost about half the cache. Any contiguous run of pieces decomposes into itself, so a gap left by eviction re-caches on the same boundaries. The range bound is the larger of 8 Mb -- where reads against S3 stop getting faster, and where Trino also splits -- and the allocator's maximum, since a cache buffer is always read whole. The buffer pool is per stream: the pool never evicts, so one shared across the daemon would pin every executor's peak on the heap for the daemon's life. --- .../hadoop/hive/llap/ParquetCacheLayout.java | 97 +++++++++++++++++++ .../hadoop/hive/llap/ParquetRangeBuffers.java | 62 ++++++++++++ .../hive/llap/TestParquetCacheLayout.java | 93 ++++++++++++++++++ .../hive/llap/TestParquetRangeBuffers.java | 81 ++++++++++++++++ 4 files changed, 333 insertions(+) create mode 100644 ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java create mode 100644 ql/src/java/org/apache/hadoop/hive/llap/ParquetRangeBuffers.java create mode 100644 ql/src/test/org/apache/hadoop/hive/llap/TestParquetCacheLayout.java create mode 100644 ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java diff --git a/ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java b/ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java new file mode 100644 index 000000000000..da04f9d238fb --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.llap; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.io.Allocator; +import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.orc.encoded.CacheChunk; + +/** + * How Parquet column chunks are laid out in the LLAP cache and fetched from the file. Everything + * that caches a chunk goes through this, so a reader finding only part of one cached re-caches + * the gap on the boundaries the first reader used. A reader builds one from the daemon's allocator + * and configuration and uses it for every chunk, which is what keeps its layout consistent. + */ +public final class ParquetCacheLayout { + + /** + * An object store fetches separate ranges concurrently but never splits one, so cutting buys + * parallelism while a large range amortises the request. Measured against S3 the knee is here: + * below 4 Mb throughput drops sharply, above 8 Mb it is flat. Trino also splits at 8 Mb. + */ + private static final int MIN_RANGE_CAP_BYTES = 8 << 20; + + public static final DiskRangeListFactory CACHE_CHUNK_FACTORY = CacheChunk::new; + + private final int maxBuffer; + private final int minBuffer; + private final int maxRange; + + /** The layout for a daemon: its allocator's maximum and the grain it caches non-ORC data at. */ + public ParquetCacheLayout(Allocator allocator, Configuration conf) { + this(allocator.getMaxAllocation(), HiveConf.getSizeVar(conf, HiveConf.ConfVars.LLAP_IO_ENCODE_ALLOC_SIZE)); + } + + /** + * @param maxAlloc largest buffer the allocator hands out; rounded down to a power of two + * @param splitFloor size below which a remainder is left as one buffer rather than split further + */ + public ParquetCacheLayout(int maxAlloc, long splitFloor) { + this.maxBuffer = Integer.highestOneBit(maxAlloc); + // Clamped, not cast: the key is validated as a size but not against the allocator's maximum. + this.minBuffer = (int) Math.min(splitFloor, maxBuffer); + this.maxRange = Math.max(MIN_RANGE_CAP_BYTES, maxAlloc); + } + + /** + * Largest range of a vectored read. A cache buffer is always fetched whole, so a cap below the + * allocator's maximum would not be one; it does not simply follow the allocator either, since + * shrinking the cache should not shrink reads. + */ + public int maxRangeBytes() { + return maxRange; + } + + /** + * Cuts a range into power-of-two buffers, largest first, so each fills its buddy allocation + * exactly; caching a chunk whole leaves the allocator to round it up, and a 5 Mb chunk then + * occupies 8 Mb. Below the floor the remainder stays one buffer, since splitting a small tail + * costs a key, a refcount and an eviction to save very little. ORC caches in fixed parts of its + * compression buffer size instead, which needs no decomposition but many more buffers. + * + * Any contiguous run of the result decomposes into itself, so a gap left by evicting some of a + * chunk's buffers is re-cached under the same keys. + */ + public int[] bufferSizes(long length) { + int count = 0; + for (long left = length; left > 0; ++count) { + left -= left < minBuffer ? left : Math.min(maxBuffer, Long.highestOneBit(left)); + } + int[] sizes = new int[count]; + long left = length; + for (int i = 0; i < count; ++i) { + sizes[i] = left < minBuffer ? (int) left : (int) Math.min(maxBuffer, Long.highestOneBit(left)); + left -= sizes[i]; + } + return sizes; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/llap/ParquetRangeBuffers.java b/ql/src/java/org/apache/hadoop/hive/llap/ParquetRangeBuffers.java new file mode 100644 index 000000000000..e636ff1bf047 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/llap/ParquetRangeBuffers.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.llap; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.io.ElasticByteBufferPool; + +/** + * Heap buffers for one stream's vectored reads. A stream returning slices of a larger read says + * so ({@link StreamCapabilities#VECTOREDIO_BUFFERS_SLICED}); those alias each other and are left + * to the collector. Other streams' buffers are exclusive and pooled for reuse across the file's + * row groups. The pool is per stream, not per JVM: {@link ElasticByteBufferPool} never evicts + * and has one lock, so a shared one would pin every executor's peak for the daemon's life. + */ +public final class ParquetRangeBuffers { + private final ElasticByteBufferPool pool; + + private ParquetRangeBuffers(boolean pooled) { + this.pool = pooled ? new ElasticByteBufferPool() : null; + } + + public static ParquetRangeBuffers forStream(FSDataInputStream stream) { + return new ParquetRangeBuffers(!stream.hasCapability(StreamCapabilities.VECTOREDIO_BUFFERS_SLICED)); + } + + public ByteBuffer allocate(int length) { + if (pool == null) { + return ByteBuffer.allocate(length); + } + ByteBuffer buffer = pool.getBuffer(false, length); + // The pool hands back anything large enough; limit it so a reader filling remaining() stops + // at the range end. + buffer.limit(length); + return buffer; + } + + public void release(ByteBuffer buffer) { + if (pool != null) { + pool.putBuffer(buffer); + } + } +} diff --git a/ql/src/test/org/apache/hadoop/hive/llap/TestParquetCacheLayout.java b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetCacheLayout.java new file mode 100644 index 000000000000..5d328b186fe6 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetCacheLayout.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.llap; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; + +import org.junit.Test; + +public class TestParquetCacheLayout { + private static final int MB = 1 << 20; + /** hive.llap.io.encode.alloc.size at its default, the grain these expectations assume. */ + private static final long FLOOR = 256 * 1024; + + @Test + public void chunksAreCutIntoPowersOfTwoLargestFirst() { + ParquetCacheLayout layout = new ParquetCacheLayout(16 * MB, FLOOR); + assertArrayEquals(new int[] {4 * MB, MB}, layout.bufferSizes(5L * MB)); + assertArrayEquals(new int[] {16 * MB, 4 * MB, MB, 512 * 1024, 100}, + layout.bufferSizes(21L * MB + 512 * 1024 + 100)); + assertArrayEquals(new int[] {16 * MB, 16 * MB}, layout.bufferSizes(32L * MB)); + assertArrayEquals(new int[0], layout.bufferSizes(0)); + } + + @Test + public void belowTheFloorTheRemainderStaysOneBuffer() { + assertArrayEquals(new int[] {200 * 1024}, new ParquetCacheLayout(16 * MB, FLOOR).bufferSizes(200 * 1024)); + } + + @Test + public void aSmallAllocatorCapsEveryBufferAndLowersTheFloorWithIt() { + assertArrayEquals(new int[] {4096, 4096, 1000}, new ParquetCacheLayout(4096, FLOOR).bufferSizes(9192)); + } + + @Test + public void anyRunOfBuffersDecomposesIntoItself() { + // The property the shared layout exists for: a gap left by evicting some of a chunk's buffers + // must re-cache on the boundaries the original split used, or the bytes are held twice under + // keys that never match. Lengths cover the floor, both sides of maxAlloc and odd tails. + long[] lengths = {1, 4095, 4096, 4097, 200 * 1024, FLOOR - 1, FLOOR, FLOOR + 1, MB - 1, MB, + 5L * MB, 15L * MB + 999, 16L * MB, 16L * MB + 1, 21L * MB + 512 * 1024 + 100, 100L * MB - 7}; + for (int maxAlloc : new int[] {4096, MB, 16 * MB}) { + ParquetCacheLayout layout = new ParquetCacheLayout(maxAlloc, FLOOR); + for (long length : lengths) { + int[] sizes = layout.bufferSizes(length); + if (sizes.length > 48) { + continue; // a tiny maxAlloc on a long range is thousands of equal buffers; nothing new to learn + } + for (int from = 0; from < sizes.length; ++from) { + long run = 0; + for (int to = from; to < sizes.length; ++to) { + run += sizes[to]; + assertArrayEquals("length " + length + " maxAlloc " + maxAlloc + " run [" + from + ", " + to + "]", + Arrays.copyOfRange(sizes, from, to + 1), layout.bufferSizes(run)); + } + } + } + } + } + + @Test + public void theRangeCapNeverFallsBelowTheAllocatorsMaximumNorBelowEightMb() { + assertEquals(8 * MB, new ParquetCacheLayout(4 * MB, FLOOR).maxRangeBytes()); + assertEquals(8 * MB, new ParquetCacheLayout(8 * MB, FLOOR).maxRangeBytes()); + assertEquals(16 * MB, new ParquetCacheLayout(16 * MB, FLOOR).maxRangeBytes()); + assertEquals(32 * MB, new ParquetCacheLayout(32 * MB, FLOOR).maxRangeBytes()); + } + + @Test + public void aFloorAboveTheAllocatorsMaximumIsClampedNotOverflowed() { + // The key is validated as a size but not against the allocator's maximum. + assertArrayEquals(new int[] {4096, 4096, 1000}, new ParquetCacheLayout(4096, 4L << 30).bufferSizes(9192)); + } +} diff --git a/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java new file mode 100644 index 000000000000..9461e497313c --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.llap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.Seekable; +import org.apache.hadoop.fs.StreamCapabilities; +import org.junit.Test; + +public class TestParquetRangeBuffers { + + @Test + public void anExclusiveStreamsBuffersAreReusedAndLimitedToTheRange() throws IOException { + ParquetRangeBuffers buffers = ParquetRangeBuffers.forStream(stream(false)); + ByteBuffer first = buffers.allocate(100); + assertEquals(100, first.limit()); + buffers.release(first); + // The pool hands back anything large enough: the same 100-byte buffer serves a 50-byte range, + // and must be limited to it or a reader filling remaining() overruns the range. + ByteBuffer second = buffers.allocate(50); + assertSame(first.array(), second.array()); + assertEquals(50, second.limit()); + assertEquals(0, second.position()); + } + + @Test + public void aSlicingStreamsBuffersAreNeverPooled() throws IOException { + ParquetRangeBuffers buffers = ParquetRangeBuffers.forStream(stream(true)); + ByteBuffer first = buffers.allocate(100); + buffers.release(first); + ByteBuffer second = buffers.allocate(50); + assertNotSame(first.array(), second.array()); + assertEquals(50, second.capacity()); + } + + /** A stream that does nothing but answer whether its vectored-read buffers are slices. */ + private static FSDataInputStream stream(boolean sliced) throws IOException { + return new FSDataInputStream(new Inert()) { + @Override + public boolean hasCapability(String capability) { + return sliced && StreamCapabilities.VECTOREDIO_BUFFERS_SLICED.equals(capability); + } + }; + } + + private static final class Inert extends InputStream implements Seekable, PositionedReadable { + @Override public int read() { return -1; } + @Override public void seek(long pos) { } + @Override public long getPos() { return 0; } + @Override public boolean seekToNewSource(long targetPos) { return false; } + @Override public int read(long position, byte[] buffer, int offset, int length) { return -1; } + @Override public void readFully(long position, byte[] buffer, int offset, int length) { } + @Override public void readFully(long position, byte[] buffer) { } + } +} From 23e1ab759161fd6320f5845f03ab75d4908491ac Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 14 Sep 2026 00:39:31 +0300 Subject: [PATCH 02/17] HIVE-30059: Extract Parquet row-group column-reader construction into its own class VectorizedParquetRecordReader built its VectorizedColumnReader array inline: walk the requested schema, match each Hive type to a Parquet column, apply the column defaults a schema-evolved file needs, and decide which nested shapes are readable at all. That logic depends only on the schemas and the Hive types, not on how the pages were obtained, but it could not be called from anywhere else. Move it to ParquetRowGroupDecoder unchanged, so a reader that gets its pages from somewhere other than a file -- an LLAP cache-backed consumer, next -- builds the same readers from the same rules rather than growing a second copy that drifts. No behaviour change: the reader now delegates to the new class and the qtests are untouched. checkListColumnSupport moves with it, so the Iceberg storage handler's reference to it follows. --- .../mr/hive/HiveIcebergStorageHandler.java | 4 +- .../vector/ParquetRowGroupDecoder.java | 275 ++++++++++++++++++ .../vector/VectorizedParquetRecordReader.java | 182 +----------- 3 files changed, 284 insertions(+), 177 deletions(-) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 5d19622f2463..0a540f746dab 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -84,7 +84,6 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.StorageFormatDescriptor; -import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; @@ -1890,7 +1889,8 @@ private static boolean hasOrcTimeInSchema(Properties tableProps, Schema tableSch /** * Vectorized reads of parquet files from columns with list or map type is only supported if the nested types are of * primitive type category - * check {@link VectorizedParquetRecordReader#checkListColumnSupport} for details on nested types under lists + * check {@link org.apache.hadoop.hive.ql.io.parquet.vector.ParquetRowGroupDecoder#checkListColumnSupport} for + * details on nested types under lists * @param tableProps iceberg table properties * @param tableSchema iceberg table schema * @return true if having nested types diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java new file mode 100644 index 000000000000..0cc6ab8de4b2 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.parquet.vector; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.parquet.ParquetRuntimeException; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.InvalidSchemaException; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; + +/** + * Reusable, stateless helper that builds the {@link VectorizedColumnReader} array for one Parquet + * row group, given an already-read {@link PageReadStore}. The logic here was lifted verbatim from + * {@link VectorizedParquetRecordReader} so that both the row-by-split reader and the LLAP + * cache-backed consumer ({@code ParquetEncodedDataConsumer}) can share a single, behavior-preserving + * implementation of the Hive-type-driven Parquet column-reader construction. + * + *

The instance only carries the {@code fileSchema} (used for the schema-evolution check). All + * per-row-group state lives in the supplied {@link PageReadStore}. + */ +public class ParquetRowGroupDecoder { + + private static final int MAP_DEFINITION_LEVEL_MAX = 3; + + private final MessageType fileSchema; + private final Map initialDefaults; + + public ParquetRowGroupDecoder(MessageType fileSchema, Map initialDefaults) { + this.fileSchema = fileSchema; + this.initialDefaults = initialDefaults; + } + + /** + * Builds the per-(requested-)column {@link VectorizedColumnReader} array for a row group. + * + * @param pages the row group's page store (from {@code reader.readRowGroup(..)} + * / {@code readNextRowGroup()}) + * @param requestedSchema the projected Parquet schema being read + * @param columnTypesList the Hive type infos for ALL table columns (indexed by table col id) + * @param colsToInclude the table column ids being read, in requested-schema field order; + * may be empty (e.g. {@code count(*)}), in which case all readers are + * null + * @param readAllColumns whether projection is "read all columns" + * @param skipTimestampConversion see {@code VectorizedParquetRecordReader} + * @param writerTimezone see {@code VectorizedParquetRecordReader} + * @param skipProlepticConversion see {@code VectorizedParquetRecordReader} + * @param legacyConversionEnabled see {@code VectorizedParquetRecordReader} + * @return one reader per requested-schema field (null entries where no reader is needed) + */ + public VectorizedColumnReader[] buildColumnReaders( + PageReadStore pages, + MessageType requestedSchema, + List columnTypesList, + List colsToInclude, + boolean readAllColumns, + boolean skipTimestampConversion, + ZoneId writerTimezone, + boolean skipProlepticConversion, + boolean legacyConversionEnabled) throws IOException { + List types = requestedSchema.getFields(); + VectorizedColumnReader[] columnReaders = new VectorizedColumnReader[types.size()]; + + if (!readAllColumns) { + // certain queries like select count(*) from table do not have + // any projected columns and still have isReadAllColumns as false + // in such cases columnReaders are not needed + // However, if colsToInclude is not empty we should initialize each columnReader + if (!colsToInclude.isEmpty()) { + for (int i = 0; i < types.size(); ++i) { + columnReaders[i] = buildVectorizedParquetReader( + columnTypesList.get(colsToInclude.get(i)), types.get(i), pages, + requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, + skipProlepticConversion, legacyConversionEnabled, 0, 0); + } + } + } else { + for (int i = 0; i < types.size(); ++i) { + columnReaders[i] = buildVectorizedParquetReader( columnTypesList.get(i), + types.get(i), pages, requestedSchema.getColumns(), skipTimestampConversion, + writerTimezone, skipProlepticConversion, legacyConversionEnabled, 0, 0); + } + } + return columnReaders; + } + + private static List getAllColumnDescriptorByType( + int depth, + Type type, + List columns) throws ParquetRuntimeException { + List res = new ArrayList<>(); + for (ColumnDescriptor descriptor : columns) { + if (depth >= descriptor.getPath().length) { + throw new InvalidSchemaException("Corrupted Parquet schema"); + } + if (type.getName().equals(descriptor.getPath()[depth])) { + res.add(descriptor); + } + } + return res; + } + + // TODO support only non nested case + private static PrimitiveType getElementType(Type type) { + if (type.isPrimitive()) { + return type.asPrimitiveType(); + } + if (type.asGroupType().getFields().size() > 1) { + throw new RuntimeException( + "Current Parquet Vectorization reader doesn't support nested type"); + } + + Type childType = type.asGroupType().getFields().get(0); + + // Parquet file generated using thrift may have child type as PrimitiveType + if (childType.isPrimitive()) { + return childType.asPrimitiveType(); + } else { + return childType.asGroupType().getFields().get(0).asPrimitiveType(); + } + } + + // Build VectorizedParquetColumnReader via Hive typeInfo and Parquet schema + private VectorizedColumnReader buildVectorizedParquetReader( + TypeInfo typeInfo, + Type type, + PageReadStore pages, + List columnDescriptors, + boolean skipTimestampConversion, + ZoneId writerTimezone, + boolean skipProlepticConversion, + boolean legacyConversionEnabled, + int depth, int currentDefLevel) throws IOException { + int typeDefLevel = currentDefLevel; + if (type.isRepetition(Type.Repetition.OPTIONAL) || type.isRepetition(Type.Repetition.REPEATED)) { + typeDefLevel++; + } + List descriptors = + getAllColumnDescriptorByType(depth, type, columnDescriptors); + // Support for schema evolution: if the column from the current + // query schema is not present in the file schema, return a dummy + // reader that produces nulls. This allows queries to proceed even + // when new columns have been added after the file was written. + if (!fileSchema.getColumns().contains(descriptors.get(0))) { + return new VectorizedDummyColumnReader(Optional.ofNullable(initialDefaults) + .map(defaults -> defaults.getOrDefault(descriptors.get(0).getPath()[0], null)).orElse(null)); + } + switch (typeInfo.getCategory()) { + case PRIMITIVE: + if (columnDescriptors == null || columnDescriptors.isEmpty()) { + throw new RuntimeException( + "Failed to find related Parquet column descriptor with type " + type); + } + return new VectorizedPrimitiveColumnReader(descriptors.get(0), + pages.getPageReader(descriptors.get(0)), skipTimestampConversion, writerTimezone, + skipProlepticConversion, legacyConversionEnabled, type, typeInfo); + case STRUCT: + StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; + List fieldReaders = new ArrayList<>(); + List fieldTypes = structTypeInfo.getAllStructFieldTypeInfos(); + List types = type.asGroupType().getFields(); + for (int i = 0; i < fieldTypes.size(); i++) { + VectorizedColumnReader r = + buildVectorizedParquetReader( fieldTypes.get(i), types.get(i), pages, + descriptors, skipTimestampConversion, writerTimezone, skipProlepticConversion, + legacyConversionEnabled, depth + 1, typeDefLevel); + if (r != null) { + fieldReaders.add(r); + } else { + throw new RuntimeException( + "Fail to build Parquet vectorized reader based on Hive type " + fieldTypes.get(i) + .getTypeName() + " and Parquet type" + types.get(i).toString()); + } + } + return new VectorizedStructColumnReader(fieldReaders, typeDefLevel); + case LIST: + checkListColumnSupport(((ListTypeInfo) typeInfo).getListElementTypeInfo()); + if (columnDescriptors == null || columnDescriptors.isEmpty()) { + throw new RuntimeException( + "Failed to find related Parquet column descriptor with type " + type); + } + + return new VectorizedListColumnReader(descriptors.get(0), + pages.getPageReader(descriptors.get(0)), skipTimestampConversion, writerTimezone, + skipProlepticConversion, legacyConversionEnabled, getElementType(type), typeInfo); + case MAP: + if (columnDescriptors == null || columnDescriptors.isEmpty()) { + throw new RuntimeException( + "Failed to find related Parquet column descriptor with type " + type); + } + + // to handle the different Map definition in Parquet, eg: + // definition has 1 group: + // repeated group map (MAP_KEY_VALUE) + // {required binary key (UTF8); optional binary value (UTF8);} + // definition has 2 groups: + // optional group m1 (MAP) { + // repeated group map (MAP_KEY_VALUE) + // {required binary key (UTF8); optional binary value (UTF8);} + // } + int nestGroup = 0; + GroupType groupType = type.asGroupType(); + // if FieldCount == 2, get types for key & value, + // otherwise, continue to get the group type until MAP_DEFINITION_LEVEL_MAX. + while (groupType.getFieldCount() < 2) { + if (nestGroup > MAP_DEFINITION_LEVEL_MAX) { + throw new RuntimeException( + "More than " + MAP_DEFINITION_LEVEL_MAX + " level is found in Map definition, " + + "Failed to get the field types for Map with type " + type); + } + groupType = groupType.getFields().get(0).asGroupType(); + nestGroup++; + } + List kvTypes = groupType.getFields(); + VectorizedListColumnReader keyListColumnReader = new VectorizedListColumnReader( + descriptors.get(0), pages.getPageReader(descriptors.get(0)), skipTimestampConversion, + writerTimezone, skipProlepticConversion, legacyConversionEnabled, kvTypes.get(0), typeInfo); + VectorizedListColumnReader valueListColumnReader = new VectorizedListColumnReader( + descriptors.get(1), pages.getPageReader(descriptors.get(1)), skipTimestampConversion, + writerTimezone, skipProlepticConversion, legacyConversionEnabled, kvTypes.get(1), typeInfo); + return new VectorizedMapColumnReader(keyListColumnReader, valueListColumnReader); + case UNION: + default: + throw new RuntimeException("Unsupported category " + typeInfo.getCategory().name()); + } + } + + /** + * Check if the element type in list is supported by vectorization read. + * Supported type: INT, BYTE, SHORT, DATE, INTERVAL_YEAR_MONTH, LONG, BOOLEAN, DOUBLE, BINARY, + * STRING, CHAR, VARCHAR, FLOAT, DECIMAL + */ + private static void checkListColumnSupport(TypeInfo elementType) { + if (elementType instanceof org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) { + switch (((org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) elementType) + .getPrimitiveCategory()) { + case INTERVAL_DAY_TIME: + case TIMESTAMP: + throw new RuntimeException("Unsupported primitive type used in list:: " + elementType); + default: + // supported + } + } else { + throw new RuntimeException("Unsupported type used in list:" + elementType); + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index 236f6f3095f0..8521228cd589 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -47,16 +47,13 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.apache.parquet.ParquetRuntimeException; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter; @@ -69,11 +66,7 @@ import org.apache.parquet.hadoop.util.HadoopStreams; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.SeekableInputStream; -import org.apache.parquet.schema.GroupType; -import org.apache.parquet.schema.InvalidSchemaException; import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +104,6 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase private Object[] partitionValues; private boolean addPartitionCols = true; private Path cacheFsPath; - private static final int MAP_DEFINITION_LEVEL_MAX = 3; /** * For each request column, the reader to read this column. This is NULL if this column @@ -446,32 +438,12 @@ private void checkEndOfRowGroup() throws IOException { throw new IOException("expecting more rows but reached last block. Read " + rowsReturned + " out of " + totalRowCount); } - List columns = requestedSchema.getColumns(); - List types = requestedSchema.getFields(); - columnReaders = new VectorizedColumnReader[columns.size()]; - - if (!ColumnProjectionUtils.isReadAllColumns(jobConf)) { - //certain queries like select count(*) from table do not have - //any projected columns and still have isReadAllColumns as false - //in such cases columnReaders are not needed - //However, if colsToInclude is not empty we should initialize each columnReader - if(!colsToInclude.isEmpty()) { - for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = - buildVectorizedParquetReader( - columnTypesList.get(colsToInclude.get(i)), types.get(i), - pages, requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, 0, 0 - ); - } - } - } else { - for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = buildVectorizedParquetReader(columnTypesList.get(i), types.get(i), pages, - requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, 0, 0); - } - } + // Delegate the (Hive-type-driven) column-reader construction to the shared helper so the + // LLAP cache-backed consumer can reuse the exact same logic. Behavior is unchanged. + columnReaders = new ParquetRowGroupDecoder(fileSchema, initialDefaults).buildColumnReaders( + pages, requestedSchema, columnTypesList, colsToInclude, + ColumnProjectionUtils.isReadAllColumns(jobConf), skipTimestampConversion, writerTimezone, + skipProlepticConversion, legacyConversionEnabled); currentRowNumInRowGroup = 0; currentRowGroupIndex++; @@ -479,146 +451,6 @@ private void checkEndOfRowGroup() throws IOException { totalCountLoadedSoFar += pages.getRowCount(); } - private List getAllColumnDescriptorByType( - int depth, - Type type, - List columns) throws ParquetRuntimeException { - List res = new ArrayList<>(); - for (ColumnDescriptor descriptor : columns) { - if (depth >= descriptor.getPath().length) { - throw new InvalidSchemaException("Corrupted Parquet schema"); - } - if (type.getName().equals(descriptor.getPath()[depth])) { - res.add(descriptor); - } - } - return res; - } - - // TODO support only non nested case - private PrimitiveType getElementType(Type type) { - if (type.isPrimitive()) { - return type.asPrimitiveType(); - } - if (type.asGroupType().getFields().size() > 1) { - throw new RuntimeException( - "Current Parquet Vectorization reader doesn't support nested type"); - } - - Type childType = type.asGroupType().getFields().get(0); - - // Parquet file generated using thrift may have child type as PrimitiveType - if (childType.isPrimitive()) { - return childType.asPrimitiveType(); - } else { - return childType.asGroupType().getFields().get(0).asPrimitiveType(); - } - } - - // Build VectorizedParquetColumnReader via Hive typeInfo and Parquet schema - private VectorizedColumnReader buildVectorizedParquetReader( - TypeInfo typeInfo, - Type type, - PageReadStore pages, - List columnDescriptors, - boolean skipTimestampConversion, - ZoneId writerTimezone, - boolean skipProlepticConversion, - boolean legacyConversionEnabled, - int depth, int currentDefLevel) throws IOException { - - int typeDefLevel = currentDefLevel; - if (type.isRepetition(Type.Repetition.OPTIONAL) || type.isRepetition(Type.Repetition.REPEATED)) { - typeDefLevel++; - } - List descriptors = - getAllColumnDescriptorByType(depth, type, columnDescriptors); - // Support for schema evolution: if the column from the current - // query schema is not present in the file schema, return a dummy - // reader that produces nulls. This allows queries to proceed even - // when new columns have been added after the file was written. - if (!fileSchema.getColumns().contains(descriptors.get(0))) { - return new VectorizedDummyColumnReader(Optional.ofNullable(initialDefaults) - .map(defaults -> defaults.getOrDefault(descriptors.get(0).getPath()[0], null)).orElse(null)); - } - switch (typeInfo.getCategory()) { - case PRIMITIVE: - if (columnDescriptors == null || columnDescriptors.isEmpty()) { - throw new RuntimeException( - "Failed to find related Parquet column descriptor with type " + type); - } - return new VectorizedPrimitiveColumnReader(descriptors.get(0), - pages.getPageReader(descriptors.get(0)), skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, type, typeInfo); - case STRUCT: - StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; - List fieldReaders = new ArrayList<>(); - List fieldTypes = structTypeInfo.getAllStructFieldTypeInfos(); - List types = type.asGroupType().getFields(); - for (int i = 0; i < fieldTypes.size(); i++) { - VectorizedColumnReader r = - buildVectorizedParquetReader(fieldTypes.get(i), types.get(i), pages, descriptors, skipTimestampConversion, - writerTimezone, skipProlepticConversion, legacyConversionEnabled, depth + 1, typeDefLevel); - if (r != null) { - fieldReaders.add(r); - } else { - throw new RuntimeException( - "Fail to build Parquet vectorized reader based on Hive type " + fieldTypes.get(i) - .getTypeName() + " and Parquet type" + types.get(i).toString()); - } - } - return new VectorizedStructColumnReader(fieldReaders, typeDefLevel); - case LIST: - checkListColumnSupport(((ListTypeInfo) typeInfo).getListElementTypeInfo()); - if (columnDescriptors == null || columnDescriptors.isEmpty()) { - throw new RuntimeException( - "Failed to find related Parquet column descriptor with type " + type); - } - - return new VectorizedListColumnReader(descriptors.get(0), - pages.getPageReader(descriptors.get(0)), skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, getElementType(type), typeInfo); - case MAP: - if (columnDescriptors == null || columnDescriptors.isEmpty()) { - throw new RuntimeException( - "Failed to find related Parquet column descriptor with type " + type); - } - - // to handle the different Map definition in Parquet, eg: - // definition has 1 group: - // repeated group map (MAP_KEY_VALUE) - // {required binary key (UTF8); optional binary value (UTF8);} - // definition has 2 groups: - // optional group m1 (MAP) { - // repeated group map (MAP_KEY_VALUE) - // {required binary key (UTF8); optional binary value (UTF8);} - // } - int nestGroup = 0; - GroupType groupType = type.asGroupType(); - // if FieldCount == 2, get types for key & value, - // otherwise, continue to get the group type until MAP_DEFINITION_LEVEL_MAX. - while (groupType.getFieldCount() < 2) { - if (nestGroup > MAP_DEFINITION_LEVEL_MAX) { - throw new RuntimeException( - "More than " + MAP_DEFINITION_LEVEL_MAX + " level is found in Map definition, " + - "Failed to get the field types for Map with type " + type); - } - groupType = groupType.getFields().get(0).asGroupType(); - nestGroup++; - } - List kvTypes = groupType.getFields(); - VectorizedListColumnReader keyListColumnReader = new VectorizedListColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0)), skipTimestampConversion, - writerTimezone, skipProlepticConversion, legacyConversionEnabled, kvTypes.get(0), typeInfo); - VectorizedListColumnReader valueListColumnReader = new VectorizedListColumnReader( - descriptors.get(1), pages.getPageReader(descriptors.get(1)), skipTimestampConversion, - writerTimezone, skipProlepticConversion, legacyConversionEnabled, kvTypes.get(1), typeInfo); - return new VectorizedMapColumnReader(keyListColumnReader, valueListColumnReader); - case UNION: - default: - throw new RuntimeException("Unsupported category " + typeInfo.getCategory().name()); - } - } /** * Check if the element type in list is supported by vectorization read. From a281e8b906f25970f2e46dbb2abb08ca5d16b87e Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 14 Sep 2026 00:53:22 +0300 Subject: [PATCH 03/17] HIVE-30059: Cache and decode Parquet column chunks in LLAP IO Parquet is cache-only in LLAP IO: the file bytes are cached, but every task re-runs the vectorized Parquet reader over them. ORC instead caches the column chunks it decoded and hands the consumer a column-vector batch. This adds the same three pieces for Parquet -- an encoded data reader, an encoded data consumer and a column vector producer -- so a Parquet scan is served from the cache in the same shape as ORC. The reader reads the footer, prunes row groups against the search argument and fetches only the projected column chunks, filling the cache with what it read. The consumer decodes a row group from a cache-backed InputFile, so a second reader of the same chunk decodes from memory and never touches the filesystem. Rather than a second set of readers, as ORC needed, the cached pages are served through parquet's own PageReadStore, so both paths run the same decode: the row-group logic moves out of VectorizedParquetRecordReader into ParquetRowGroupDecoder unchanged, and both callers use it. Off by default behind hive.llap.io.parquet.native.enabled. A projection that reaches into a nested type falls back to the vectorized Parquet reader, as does any error in the native path. --- .../org/apache/hadoop/hive/conf/HiveConf.java | 3 + .../hadoop/hive/llap/io/api/LlapIo.java | 11 + .../llap/io/api/impl/LlapInputFormat.java | 5 + .../hive/llap/io/api/impl/LlapIoImpl.java | 40 + .../llap/io/api/impl/LlapRecordReader.java | 12 + .../io/decode/ParquetCachedPageReadStore.java | 231 ++++ .../decode/ParquetColumnVectorProducer.java | 83 ++ .../io/decode/ParquetEncodedDataConsumer.java | 225 ++++ .../io/encoded/ParquetEncodedColumnBatch.java | 53 + .../io/encoded/ParquetEncodedDataReader.java | 555 +++++++++ .../io/api/impl/TestLlapRecordReader.java | 209 ++++ .../encoded/TestParquetEncodedDataReader.java | 1066 +++++++++++++++++ .../hadoop/hive/ql/io/HiveInputFormat.java | 10 +- .../io/parquet/ParquetRecordReaderBase.java | 29 +- .../apache/hadoop/hive/ql/plan/MapWork.java | 3 +- 15 files changed, 2521 insertions(+), 14 deletions(-) create mode 100644 llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java create mode 100644 llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java create mode 100644 llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java create mode 100644 llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java create mode 100644 llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java create mode 100644 llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java create mode 100644 llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 01f82922c9bb..621f96fc9765 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4976,6 +4976,9 @@ public static enum ConfVars { LLAP_IO_ENCODE_ENABLED("hive.llap.io.encode.enabled", true, "Whether LLAP should try to re-encode and cache data for non-ORC formats. This is used\n" + "on LLAP Server side to determine if the infrastructure for that is initialized."), + LLAP_IO_PARQUET_NATIVE_ENABLED("hive.llap.io.parquet.native.enabled", false, + "Whether LLAP IO caches Parquet column chunks and decodes them from the cache, like ORC.\n" + + "A projection that reaches into a nested type falls back to the vectorized Parquet reader."), LLAP_IO_ENCODE_FORMATS("hive.llap.io.encode.formats", "org.apache.hadoop.mapred.TextInputFormat,", "The table input formats for which LLAP IO should re-encode and cache data.\n" + diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java index 84b562156e43..b037fc015c80 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -107,6 +108,16 @@ MemoryBufferOrBuffers getParquetFooterBuffersFromCache(Path path, JobConf conf, RecordReader llapVectorizedOrcReaderForPath(Object fileKey, Path path, CacheTag tag, List tableIncludedCols, JobConf conf, long offset, long length, Reporter reporter) throws IOException; + /** + * Parquet counterpart of {@link #llapVectorizedOrcReaderForPath}: reads the column chunks of the split through the + * LLAP data cache. Returns null when the file cannot be served this way (native Parquet IO disabled, no MapWork, + * unsupported schema). + * @param initialDefaults - values for columns absent from the file, keyed by column name + */ + RecordReader llapVectorizedParquetReaderForPath(Object fileKey, Path path, + CacheTag tag, List tableIncludedCols, JobConf conf, long offset, long length, + Map initialDefaults, Reporter reporter) throws IOException; + /** * Extract and return the cache content metadata. */ diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java index 91278b7fd358..944f14130ec6 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java @@ -48,7 +48,9 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat.AvoidSplitCombination; +import org.apache.hadoop.hive.ql.io.LlapCacheOnlyInputFormatInterface; import org.apache.hadoop.hive.ql.io.LlapAwareSplit; import org.apache.hadoop.hive.ql.io.NullRowsInputFormat.NullRowsRecordReader; import org.apache.hadoop.hive.ql.io.SelfDescribingInputFormatInterface; @@ -123,6 +125,9 @@ public RecordReader getRecordReader( cvp, executor, sourceInputFormat, sourceSerDe, reporter, daemonConf); if (rr == null) { // Reader-specific incompatibility like SMB or schema evolution. + if (sourceInputFormat instanceof LlapCacheOnlyInputFormatInterface) { + LlapProxy.getIo().initCacheOnlyInputFormat(sourceInputFormat); + } return sourceInputFormat.getRecordReader(split, job, reporter); } // For non-vectorized operator case, wrap the reader if possible. diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index e7927eb8e02d..becfdce5a5f3 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -77,6 +78,8 @@ import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer; import org.apache.hadoop.hive.llap.io.decode.GenericColumnVectorProducer; import org.apache.hadoop.hive.llap.io.decode.OrcColumnVectorProducer; +import org.apache.hadoop.hive.llap.io.decode.ParquetColumnVectorProducer; +import org.apache.hadoop.hive.llap.io.decode.ParquetEncodedDataConsumer; import org.apache.hadoop.hive.llap.io.encoded.OrcEncodedDataReader; import org.apache.hadoop.hive.llap.io.metadata.MetadataCache; import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; @@ -87,11 +90,13 @@ import org.apache.hadoop.hive.ql.io.orc.OrcSplit; import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; import org.apache.hadoop.hive.ql.io.parquet.vector.ParquetFooterInputFromCache; import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; @@ -122,6 +127,7 @@ public class LlapIoImpl implements LlapIo, LlapIoDebugDump { // TODO: later, we may have a map private final ColumnVectorProducer orcCvp, genericCvp; + private final ColumnVectorProducer parquetCvp; private final ExecutorService executor; private final ExecutorService encodeExecutor; private final LlapDaemonCacheMetrics cacheMetrics; @@ -272,6 +278,10 @@ public void debugDumpShort(StringBuilder sb) { metadataCache, dataCache, pathCache, bufferManagerOrc, conf, cacheMetrics, ioMetrics, tracePool); this.genericCvp = isEncodeEnabled ? new GenericColumnVectorProducer( serdeCache, bufferManagerGeneric, conf, cacheMetrics, ioMetrics, tracePool, encodeExecutor) : null; + // Native Parquet IO is gated per query by the job conf at the dispatch sites. + this.parquetCvp = dataCache != null + ? new ParquetColumnVectorProducer(dataCache, bufferManagerOrc, conf, cacheMetrics, ioMetrics) + : null; LOG.info("LLAP IO initialized"); registerMXBeans(); @@ -342,6 +352,13 @@ public InputFormat getInputFormat( ColumnVectorProducer cvp = genericCvp; if (sourceInputFormat instanceof OrcInputFormat) { cvp = orcCvp; // Special-case for ORC. + } else if (sourceInputFormat instanceof MapredParquetInputFormat && sourceSerDe == null) { + // Parquet arrives without a SerDe only from HiveInputFormat's native-cache dispatch; with one + // it is the encode.formats configuration, which keeps the re-encoding producer. + if (parquetCvp == null) { + return null; + } + cvp = parquetCvp; } else if (cvp == null) { LOG.warn("LLAP encode is disabled; cannot use for " + sourceInputFormat.getClass()); return null; @@ -473,6 +490,29 @@ public RecordReader llapVectorizedOrcReaderFor } } + @Override + public RecordReader llapVectorizedParquetReaderForPath(Object fileKey, Path path, + CacheTag tag, List tableIncludedCols, JobConf conf, long offset, long length, + Map initialDefaults, Reporter reporter) throws IOException { + if (parquetCvp == null) { + return null; + } + FileSplit split = new FileSplit(path, offset, length, (String[]) null); + try { + LlapRecordReader rr = LlapRecordReader.create(conf, split, tableIncludedCols, HiveStringUtils.getHostname(), + parquetCvp, executor, null, null, reporter, daemonConf); + if (rr == null) { + return null; + } + ((ParquetEncodedDataConsumer) rr.getReadPipeline()).setInitialDefaults(initialDefaults); + rr.setPartitionValues(null); + rr.start(); + return rr; + } catch (HiveException e) { + throw new IOException(e); + } + } + @Override public MemoryBufferOrBuffers getParquetFooterBuffersFromCache(Path path, JobConf conf, @Nullable Object fileKey) throws IOException { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index afd39015b4ab..cbf9a7fad1c1 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -135,6 +135,9 @@ public static LlapRecordReader create(JobConf job, FileSplit split, if (mapWork == null) return null; // No compatible MapWork. LlapRecordReader rr = new LlapRecordReader(mapWork, job, split, tableIncludedCols, hostName, cvp, executor, sourceInputFormat, sourceSerDe, reporter, daemonConf); + if (rr.rp == null) { + return null; // The producer declined the split; the caller uses the source reader. + } if (!rr.checkOrcSchemaEvolution()) { rr.close(); throwIfCacheOnlyRead(HiveConf.getBoolVar(job, ConfVars.LLAP_IO_CACHE_ONLY)); @@ -350,6 +353,11 @@ public void start() { private boolean checkOrcSchemaEvolution() { SchemaEvolution evolution = rp.getSchemaEvolution(); + if (evolution == null) { + // No ORC-style schema evolution to validate (e.g. native parquet path); + // parquet handles its own column resolution. Nothing to check here. + return true; + } if (evolution.hasConversion() && !evolution.isOnlyImplicitConversion()) { @@ -656,6 +664,10 @@ void setPartitionValues(Object[] partitionValues) { this.partitionValues = partitionValues; } + ReadPipeline getReadPipeline() { + return rp; + } + /** This class encapsulates include-related logic for LLAP readers. It is not actually specific * to LLAP IO but in LLAP IO in particular, I want to encapsulate all this mess for now until * we have smth better like Schema Evolution v2. This can also hypothetically encapsulate diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java new file mode 100644 index 000000000000..09036b065b41 --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.io.decode; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import org.apache.hadoop.hive.llap.io.encoded.ParquetEncodedColumnBatch; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.DataPage; +import org.apache.parquet.column.page.DataPageV1; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.compression.CompressionCodecFactory; +import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor; +import org.apache.parquet.format.DataPageHeader; +import org.apache.parquet.format.DataPageHeaderV2; +import org.apache.parquet.format.DictionaryPageHeader; +import org.apache.parquet.format.PageHeader; +import org.apache.parquet.format.Util; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ColumnPath; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.schema.PrimitiveType; + +/** + * {@link PageReadStore} over one row group's cached column-chunk buffers: page headers are parsed in + * place, page bytes are views of the cache buffers and are decompressed lazily on readPage, mirroring + * parquet-mr's ParquetFileReader.Chunk.readAllPages and ColumnChunkPageReader (no offset index, no decryption). + */ +class ParquetCachedPageReadStore implements PageReadStore { + + private final Map readers = new HashMap<>(); + private final long rowCount; + + ParquetCachedPageReadStore(ParquetMetadata footer, ParquetEncodedColumnBatch batch, + CompressionCodecFactory codecFactory, ParquetMetadataConverter converter) throws IOException { + BlockMetaData block = footer.getBlocks().get(batch.rowGroupIx); + this.rowCount = block.getRowCount(); + String createdBy = footer.getFileMetaData().getCreatedBy(); + for (int pc = 0; pc < batch.chunks.length; ++pc) { + ColumnChunkMetaData chunk = batch.chunks[pc]; + readers.put(chunk.getPath(), readAllPages(chunk, chunkBuffers(batch, pc), createdBy, + codecFactory.getDecompressor(chunk.getCodec()), converter)); + } + } + + /** Slices of the cached buffers covering exactly the chunk's byte region, in file order. */ + private static List chunkBuffers(ParquetEncodedColumnBatch batch, int pc) { + long start = batch.chunks[pc].getStartingPos(), end = start + batch.chunks[pc].getTotalSize(); + List slices = new ArrayList<>(batch.columnBuffers[pc].length); + for (int i = 0; i < batch.columnBuffers[pc].length; ++i) { + long offset = batch.bufferOffsets[pc][i]; + long from = Math.max(start, offset), to = Math.min(end, offset + batch.bufferLengths[pc][i]); + ByteBuffer bb = batch.columnBuffers[pc][i].getByteBufferDup(); + bb.position(bb.position() + (int) (from - offset)); + bb.limit(bb.position() + (int) (to - from)); + slices.add(bb.slice()); + } + return slices; + } + + private static PageReader readAllPages(ColumnChunkMetaData chunk, List buffers, + String createdBy, BytesInputDecompressor decompressor, ParquetMetadataConverter converter) + throws IOException { + ByteBufferInputStream stream = ByteBufferInputStream.wrap(buffers); + PrimitiveType type = chunk.getPrimitiveType(); + List pages = new ArrayList<>(); + DictionaryPage dictionaryPage = null; + long valuesRead = 0; + while (valuesRead < chunk.getValueCount()) { + PageHeader header = Util.readPageHeader(stream); + int uncompressedSize = header.getUncompressed_page_size(); + int compressedSize = header.getCompressed_page_size(); + switch (header.getType()) { + case DICTIONARY_PAGE: + if (dictionaryPage != null) { + throw new ParquetDecodingException("more than one dictionary page in column " + chunk.getPath()); + } + DictionaryPageHeader dictHeader = header.getDictionary_page_header(); + dictionaryPage = new DictionaryPage(BytesInput.from(stream.sliceBuffers(compressedSize)), + uncompressedSize, dictHeader.getNum_values(), converter.getEncoding(dictHeader.getEncoding())); + break; + case DATA_PAGE: + DataPageHeader v1 = header.getData_page_header(); + pages.add(new DataPageV1(BytesInput.from(stream.sliceBuffers(compressedSize)), v1.getNum_values(), + uncompressedSize, converter.fromParquetStatistics(createdBy, v1.getStatistics(), type), + converter.getEncoding(v1.getRepetition_level_encoding()), + converter.getEncoding(v1.getDefinition_level_encoding()), + converter.getEncoding(v1.getEncoding()))); + valuesRead += v1.getNum_values(); + break; + case DATA_PAGE_V2: + DataPageHeaderV2 v2 = header.getData_page_header_v2(); + int dataSize = compressedSize + - v2.getRepetition_levels_byte_length() - v2.getDefinition_levels_byte_length(); + BytesInput repetitionLevels = BytesInput.from(stream.sliceBuffers(v2.getRepetition_levels_byte_length())); + BytesInput definitionLevels = BytesInput.from(stream.sliceBuffers(v2.getDefinition_levels_byte_length())); + BytesInput values = BytesInput.from(stream.sliceBuffers(dataSize)); + pages.add(new DataPageV2(v2.getNum_rows(), v2.getNum_nulls(), v2.getNum_values(), + repetitionLevels, definitionLevels, converter.getEncoding(v2.getEncoding()), values, + uncompressedSize, converter.fromParquetStatistics(createdBy, v2.getStatistics(), type), + v2.isIs_compressed())); + valuesRead += v2.getNum_values(); + break; + default: + stream.skipFully(compressedSize); + } + } + if (valuesRead != chunk.getValueCount()) { + throw new IOException("Expected " + chunk.getValueCount() + " values in column chunk " + chunk.getPath() + + " at offset " + chunk.getStartingPos() + " but got " + valuesRead + " over " + pages.size() + " pages"); + } + return new CachedChunkPageReader(decompressor, pages, dictionaryPage); + } + + @Override + public long getRowCount() { + return rowCount; + } + + @Override + public PageReader getPageReader(ColumnDescriptor descriptor) { + return readers.get(ColumnPath.get(descriptor.getPath())); + } + + private static final class CachedChunkPageReader implements PageReader { + private final BytesInputDecompressor decompressor; + private final Queue compressedPages; + private final DictionaryPage compressedDictionaryPage; + private final long valueCount; + + CachedChunkPageReader(BytesInputDecompressor decompressor, List compressedPages, + DictionaryPage compressedDictionaryPage) { + this.decompressor = decompressor; + this.compressedPages = new ArrayDeque<>(compressedPages); + this.compressedDictionaryPage = compressedDictionaryPage; + long count = 0; + for (DataPage p : compressedPages) { + count += p.getValueCount(); + } + this.valueCount = count; + } + + @Override + public long getTotalValueCount() { + return valueCount; + } + + @Override + public DataPage readPage() { + DataPage compressedPage = compressedPages.poll(); + if (compressedPage == null) { + return null; + } + return compressedPage.accept(new DataPage.Visitor() { + @Override + public DataPage visit(DataPageV1 page) { + try { + return new DataPageV1(decompressor.decompress(page.getBytes(), page.getUncompressedSize()), + page.getValueCount(), page.getUncompressedSize(), page.getStatistics(), + page.getRlEncoding(), page.getDlEncoding(), page.getValueEncoding()); + } catch (IOException e) { + throw new ParquetDecodingException("could not decompress page", e); + } + } + + @Override + public DataPage visit(DataPageV2 page) { + if (!page.isCompressed()) { + return page; + } + int uncompressedSize = Math.toIntExact(page.getUncompressedSize() + - page.getDefinitionLevels().size() - page.getRepetitionLevels().size()); + BytesInput data; + try { + data = decompressor.decompress(page.getData(), uncompressedSize); + } catch (IOException e) { + throw new ParquetDecodingException("could not decompress page", e); + } + return DataPageV2.uncompressed(page.getRowCount(), page.getNullCount(), page.getValueCount(), + page.getRepetitionLevels(), page.getDefinitionLevels(), page.getDataEncoding(), data, + page.getStatistics()); + } + }); + } + + @Override + public DictionaryPage readDictionaryPage() { + if (compressedDictionaryPage == null) { + return null; + } + try { + BytesInput bytes = decompressor.decompress( + compressedDictionaryPage.getBytes(), compressedDictionaryPage.getUncompressedSize()); + return new DictionaryPage(bytes, compressedDictionaryPage.getDictionarySize(), + compressedDictionaryPage.getEncoding()); + } catch (IOException e) { + throw new ParquetDecodingException("Could not decompress dictionary page", e); + } + } + } +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java new file mode 100644 index 000000000000..7c0b45ddccf8 --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.io.decode; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.llap.cache.BufferUsageManager; +import org.apache.hadoop.hive.llap.cache.LowLevelCache; +import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; +import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch; +import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl; +import org.apache.hadoop.hive.llap.io.encoded.ParquetEncodedDataReader; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; +import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.plan.PartitionDesc; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Reporter; + +public class ParquetColumnVectorProducer implements ColumnVectorProducer { + private final LowLevelCache lowLevelCache; + private final BufferUsageManager bufferManager; + private final Configuration conf; + private final LlapDaemonCacheMetrics cacheMetrics; + private final LlapDaemonIOMetrics ioMetrics; + + public ParquetColumnVectorProducer(LowLevelCache lowLevelCache, BufferUsageManager bufferManager, + Configuration conf, LlapDaemonCacheMetrics cacheMetrics, LlapDaemonIOMetrics ioMetrics) { + this.lowLevelCache = lowLevelCache; + this.bufferManager = bufferManager; + this.conf = conf; + this.cacheMetrics = cacheMetrics; + this.ioMetrics = ioMetrics; + } + + @Override + public ReadPipeline createReadPipeline(Consumer consumer, FileSplit split, + Includes includes, SearchArgument sarg, QueryFragmentCounters counters, + SchemaEvolutionFactory sef, InputFormat sourceInputFormat, Deserializer sourceSerDe, + Reporter reporter, JobConf job, Map parts) throws IOException { + try { + ParquetEncodedDataConsumer edc = + new ParquetEncodedDataConsumer(consumer, includes, counters, ioMetrics, job); + ParquetEncodedDataReader reader = new ParquetEncodedDataReader( + lowLevelCache, bufferManager, conf, job, split, includes, edc, counters); + reader.loadFooter(); + // Null makes LlapInputFormat fall back to the normal VectorizedParquetRecordReader. + if (reader.projectsNestedTypes()) { + LlapIoImpl.LOG.info("Parquet native cache: falling back to normal reader for {} due to " + + "nested types in the projection", split.getPath()); + return null; + } + edc.init(reader, reader); + return edc; + } catch (IOException e) { + LlapIoImpl.LOG.info("Parquet native cache: falling back to normal reader for {} due to {}", + split.getPath(), e.toString()); + return null; + } + } +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java new file mode 100644 index 000000000000..1ab26aed50d3 --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.io.decode; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.counters.LlapIOCounters; +import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; +import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch; +import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl; +import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes; +import org.apache.hadoop.hive.llap.io.encoded.ParquetEncodedColumnBatch; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; +import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; +import org.apache.hadoop.hive.ql.io.parquet.vector.ParquetRowGroupDecoder; +import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedColumnReader; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.orc.TypeDescription; +import org.apache.orc.impl.SchemaEvolution; +import org.apache.parquet.HadoopReadOptions; +import org.apache.parquet.ParquetReadOptions; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.compression.CompressionCodecFactory; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.schema.MessageType; + +import com.google.common.base.Strings; + +/** + * The Parquet counterpart of {@link OrcEncodedDataConsumer}: it turns one row group's worth of + * cached column-chunk buffers ({@link ParquetEncodedColumnBatch}) into {@link ColumnVectorBatch}es + * and hands them downstream. + * + *

Rather than reimplementing Parquet value decoding, this consumer builds a + * {@link ParquetCachedPageReadStore} directly over the cached buffers (pages are parsed in place and + * decompressed lazily, no ParquetFileReader and no whole-chunk copy) and drives the very same + * {@link VectorizedColumnReader}s the non-cached vectorized reader uses (via {@link ParquetRowGroupDecoder}). + */ +public class ParquetEncodedDataConsumer + extends EncodedDataConsumer { + + private final Configuration jobConf; + private final boolean useDecimal64ColumnVectors; + private final CompressionCodecFactory codecFactory; + private final ParquetMetadataConverter converter; + private ParquetMetadata footer; + private MessageType requestedSchema; + private Path path; + private Map initialDefaults; + + // Derived lazily (once) from the footer + job conf, then reused across row groups. + private List columnTypesList; + private List colsToInclude; + private boolean readAllColumns; + private boolean skipTimestampConversion; + private boolean skipProlepticConversion; + private boolean legacyConversionEnabled; + private ZoneId writerTimezone; + private boolean schemaInitialized = false; + + public ParquetEncodedDataConsumer(Consumer consumer, Includes includes, + QueryFragmentCounters counters, LlapDaemonIOMetrics ioMetrics, Configuration jobConf) { + super(consumer, includes.getPhysicalColumnIds().size(), ioMetrics, counters); + this.jobConf = jobConf; + this.useDecimal64ColumnVectors = HiveConf.getVar(jobConf, + ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + ParquetReadOptions options = HadoopReadOptions.builder(jobConf).build(); + this.codecFactory = options.getCodecFactory(); + this.converter = new ParquetMetadataConverter(options); + } + + public void setFileMetadata(ParquetMetadata footer, MessageType requestedSchema, Path path) { + this.footer = footer; + this.requestedSchema = requestedSchema; + this.path = path; + } + + public void setInitialDefaults(Map initialDefaults) { + this.initialDefaults = initialDefaults; + } + + /** + * Derives the Hive type / projection lists and the conversion flags the same way + * {@link org.apache.hadoop.hive.ql.io.parquet.ParquetRecordReaderBase} does from the job conf, + * so the shared {@link ParquetRowGroupDecoder} produces identical readers. + */ + private void initSchema() { + org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData = footer.getFileMetaData(); + java.util.Map kvMeta = fileMetaData.getKeyValueMetaData(); + + this.columnTypesList = + DataWritableReadSupport.getColumnTypes(jobConf.get(IOConstants.COLUMNS_TYPES)); + this.colsToInclude = ColumnProjectionUtils.getReadColumnIDs(jobConf); + this.readAllColumns = ColumnProjectionUtils.isReadAllColumns(jobConf); + + this.writerTimezone = DataWritableReadSupport.getWriterTimeZoneId(kvMeta); + if (HiveConf.getBoolVar(jobConf, ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION)) { + this.skipTimestampConversion = + !Strings.nullToEmpty(fileMetaData.getCreatedBy()).startsWith("parquet-mr"); + } else { + this.skipTimestampConversion = false; + } + Boolean proleptic = DataWritableReadSupport.getWriterDateProleptic(kvMeta); + if (proleptic == null) { + proleptic = HiveConf.getBoolVar(jobConf, ConfVars.HIVE_PARQUET_DATE_PROLEPTIC_GREGORIAN_DEFAULT); + } + this.skipProlepticConversion = proleptic; + this.legacyConversionEnabled = DataWritableReadSupport.getZoneConversionLegacy(kvMeta, jobConf); + + this.schemaInitialized = true; + } + + @Override + protected void decodeBatch(ParquetEncodedColumnBatch batch, + Consumer downstreamConsumer) throws InterruptedException { + if (!schemaInitialized) { + initSchema(); + } + + long startTime = counters.startTimeCounter(); + try { + PageReadStore pages = new ParquetCachedPageReadStore(footer, batch, codecFactory, converter); + + VectorizedColumnReader[] columnReaders = + new ParquetRowGroupDecoder(footer.getFileMetaData().getSchema(), initialDefaults).buildColumnReaders( + pages, requestedSchema, columnTypesList, colsToInclude, readAllColumns, + skipTimestampConversion, writerTimezone, skipProlepticConversion, + legacyConversionEnabled); + + long rowCount = pages.getRowCount(); + long rowsLeft = rowCount; + int batches = 0; + while (rowsLeft > 0) { + int batchSize = (int) Math.min(VectorizedRowBatch.DEFAULT_SIZE, rowsLeft); + + ColumnVectorBatch cvb = cvbPool.take(); + cvb.filterContext.reset(); + cvb.size = batchSize; + + // columnReaders[i] is requestedSchema field i, i.e. the i-th projected column. + for (int i = 0; i < columnReaders.length; ++i) { + if (columnReaders[i] == null) { + continue; + } + TypeInfo columnType = readAllColumns + ? columnTypesList.get(i) + : columnTypesList.get(colsToInclude.get(i)); + ColumnVector cv = prepareColumnVector(cvb, i, columnType, batchSize); + columnReaders[i].readBatch(batchSize, cv, columnType); + } + + downstreamConsumer.consumeData(cvb); + counters.incrCounter(LlapIOCounters.ROWS_EMITTED, batchSize); + rowsLeft -= batchSize; + ++batches; + } + counters.incrWallClockCounter(LlapIOCounters.DECODE_TIME_NS, startTime); + counters.incrCounter(LlapIOCounters.NUM_VECTOR_BATCHES, batches); + counters.incrCounter(LlapIOCounters.NUM_DECODED_BATCHES); + } catch (IOException | RuntimeException e) { + // parquet-mr reports decode failures as runtime ParquetDecodingException. + LlapIoImpl.LOG.error("Parquet decodeBatch failed for rowGroup " + batch.rowGroupIx + " of " + path, e); + downstreamConsumer.setError(e); + } finally { + // Returns the pooled Hadoop decompressors after each row group; getDecompressor re-creates them. + codecFactory.release(); + } + } + + private ColumnVector prepareColumnVector(ColumnVectorBatch cvb, int idx, TypeInfo columnType, + int batchSize) { + if (cvb.cols[idx] == null) { + cvb.cols[idx] = VectorizedBatchUtil.createColumnVector(columnType, physicalVariation(columnType)); + } + ColumnVector cv = cvb.cols[idx]; + cv.reset(); + cv.ensureSize(batchSize, false); + cv.isRepeating = true; + return cv; + } + + private DataTypePhysicalVariation physicalVariation(TypeInfo columnType) { + if (useDecimal64ColumnVectors && columnType instanceof DecimalTypeInfo + && ((DecimalTypeInfo) columnType).precision() <= TypeDescription.MAX_DECIMAL64_PRECISION) { + return DataTypePhysicalVariation.DECIMAL_64; + } + return DataTypePhysicalVariation.NONE; + } + + @Override + public SchemaEvolution getSchemaEvolution() { + return null; + } +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java new file mode 100644 index 000000000000..24e94c6e35fa --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.llap.io.encoded; + +import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; +import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; + +/** + * One row-group's worth of cached Parquet column-chunk buffers, handed from the reader to the + * consumer. Extends {@link EncodedColumnBatch} only to satisfy the {@code EncodedDataConsumer} + * generic bound; the inherited ColumnStreamData machinery is ORC-shaped and unused here. The real + * payload is in the added fields. + */ +public class ParquetEncodedColumnBatch extends EncodedColumnBatch { + + public int rowGroupIx; + /** This row group's chunk per projected column; the arrays below are indexed the same way. */ + public ColumnChunkMetaData[] chunks; + public MemoryBuffer[][] columnBuffers; + public long[][] bufferOffsets; + public int[][] bufferLengths; + + public ParquetEncodedColumnBatch() {} + + /** fileKey is the cache key; rowGroupIx is the footer index of the block within the file. */ + public void init(Object fileKey, int rowGroupIx, ColumnChunkMetaData[] chunks) { + this.batchKey = fileKey; + this.rowGroupIx = rowGroupIx; + this.chunks = chunks; + int n = chunks.length; + resetColumnArrays(n); + this.columnBuffers = new MemoryBuffer[n][]; + this.bufferOffsets = new long[n][]; + this.bufferLengths = new int[n][]; + } +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java new file mode 100644 index 000000000000..95876fd1a626 --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java @@ -0,0 +1,555 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.io.encoded; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Deque; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.io.Allocator; +import org.apache.hadoop.hive.common.io.Allocator.BufferObjectFactory; +import org.apache.hadoop.hive.common.io.CacheTag; +import org.apache.hadoop.hive.common.io.DataCache.BooleanRef; +import org.apache.hadoop.hive.common.io.DiskRange; +import org.apache.hadoop.hive.common.io.DiskRangeList; +import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; +import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.ConsumerFeedback; +import org.apache.hadoop.hive.llap.ParquetCacheLayout; +import org.apache.hadoop.hive.llap.ParquetRangeBuffers; +import org.apache.hadoop.hive.llap.LlapHiveUtils; +import org.apache.hadoop.hive.llap.cache.BufferUsageManager; +import org.apache.hadoop.hive.llap.cache.LlapDataBuffer; +import org.apache.hadoop.hive.llap.cache.LowLevelCache; +import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority; +import org.apache.hadoop.hive.llap.counters.LlapIOCounters; +import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; +import org.apache.hadoop.hive.llap.io.api.LlapProxy; +import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes; +import org.apache.hadoop.hive.llap.io.decode.ParquetEncodedDataConsumer; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.SyntheticFileId; +import org.apache.hadoop.hive.ql.io.orc.encoded.CacheChunk; +import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; +import org.apache.hadoop.hive.ql.io.parquet.vector.ParquetFooterInputFromCache; +import org.apache.hadoop.hive.ql.io.orc.encoded.StoppableAllocator; +import org.apache.hadoop.hive.ql.io.parquet.ParquetRecordReaderBase; +import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.functional.FutureIO; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.filter2.compat.FilterCompat; +import org.apache.parquet.filter2.compat.RowGroupFilter; +import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.hadoop.util.HadoopStreams; +import org.apache.parquet.io.InputFile; +import org.apache.parquet.io.SeekableInputStream; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.Type; +import org.apache.tez.common.CallableWithNdc; + +/** + * Reads one Parquet split through the LLAP cache on an IO thread. Row groups whose first data + * page falls in the split are selected and filtered by their statistics; for each one the + * projected column chunks are looked up in the cache, the missing ranges are requested in one + * vectored read, and the chunks are handed to the consumer to decode. The next row group's + * request is issued before the current one is decoded so its transfer overlaps the decode, as + * long as the two together stay within this thread's share of the cache. Every buffer in a + * consumed batch carries exactly one ref, owned by the batch until returnData. + */ +public class ParquetEncodedDataReader extends CallableWithNdc + implements ConsumerFeedback { + + private static final BufferObjectFactory DATA_BUFFER_FACTORY = LlapDataBuffer::new; + + private final LowLevelCache lowLevelCache; + private final BufferUsageManager bufferManager; + private final Configuration daemonConf; + private final ParquetCacheLayout layout; + private final JobConf jobConf; + private final FileSplit split; + private final Includes includes; + private final ParquetEncodedDataConsumer consumer; + private final QueryFragmentCounters counters; + private final UserGroupInformation ugi; + private final Path path; + private final boolean cacheOnly; + /** Bytes of column chunks one IO thread may hold across the row group in decode and the next. */ + private final long lookaheadBudget; + + private Object fileKey; + private CacheTag cacheTag; + private ParquetMetadata footer; + private MessageType requestedSchema; + private final AtomicBoolean isStopped = new AtomicBoolean(false); + + public ParquetEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager bufferManager, + Configuration daemonConf, Configuration jobConf, FileSplit split, Includes includes, + ParquetEncodedDataConsumer consumer, QueryFragmentCounters counters) throws IOException { + this.lowLevelCache = lowLevelCache; + this.bufferManager = bufferManager; + this.daemonConf = daemonConf; + this.layout = new ParquetCacheLayout(bufferManager.getAllocator(), daemonConf); + this.jobConf = (JobConf) jobConf; + this.split = split; + this.includes = includes; + this.consumer = consumer; + this.counters = counters; + this.path = split.getPath(); + this.ugi = UserGroupInformation.getCurrentUser(); + this.cacheOnly = HiveConf.getBoolVar(jobConf, ConfVars.LLAP_IO_CACHE_ONLY); + this.lookaheadBudget = HiveConf.getSizeVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE) + / Math.max(1, HiveConf.getIntVar(daemonConf, ConfVars.LLAP_IO_THREADPOOL_SIZE)); + } + + /** Reads the footer once (through the LLAP footer cache when the file has a usable key). */ + public ParquetMetadata loadFooter() throws IOException { + fileKey = SyntheticFileId.fromJobConf(jobConf); + if (fileKey == null) { + fileKey = LlapHiveUtils.createFileIdUsingFS(path.getFileSystem(jobConf), path, daemonConf); + } + if (fileKey != null) { + cacheTag = VectorizedParquetRecordReader.cacheTagOfParquetFile(path, daemonConf, jobConf); + MemoryBufferOrBuffers footerData = + LlapProxy.getIo().getParquetFooterBuffersFromCache(path, jobConf, fileKey); + footer = ParquetFileReader.readFooter( + new ParquetFooterInputFromCache(footerData), ParquetMetadataConverter.NO_FILTER); + } else { + final FileSystem fs = path.getFileSystem(jobConf); + final FileStatus stat = fs.getFileStatus(path); + InputFile inputFile = new InputFile() { + @Override + public SeekableInputStream newStream() throws IOException { + return HadoopStreams.wrap(fs.open(path)); + } + @Override + public long getLength() { + return stat.getLen(); + } + }; + footer = ParquetFileReader.readFooter(inputFile, ParquetMetadataConverter.NO_FILTER); + } + requestedSchema = DataWritableReadSupport.getRequestedSchema( + jobConf.getBoolean(DataWritableReadSupport.PARQUET_COLUMN_INDEX_ACCESS, false), + DataWritableReadSupport.getColumnNames(jobConf.get(IOConstants.COLUMNS)), + DataWritableReadSupport.getColumnTypes(jobConf.get(IOConstants.COLUMNS_TYPES)), + footer.getFileMetaData().getSchema(), jobConf); + return footer; + } + + @Override + protected Void callInternal() throws IOException, InterruptedException { + return ugi.doAs((PrivilegedExceptionAction) () -> { + try { + performDataRead(); + consumer.setDone(); + } catch (Throwable t) { + consumer.setError(t); + } + return null; + }); + } + + private void performDataRead() throws IOException, InterruptedException { + MessageType fileSchema = footer.getFileMetaData().getSchema(); + int[] projected = projectedLeaves(requestedSchema, fileSchema); + consumer.setFileMetadata(footer, requestedSchema, path); + + final Allocator allocator = bufferManager.getAllocator(); + final int maxAlloc = allocator.getMaxAllocation(); + final long splitStart = split.getStart(), splitEnd = splitStart + split.getLength(); + final List blocks = footer.getBlocks(); + List selected = new ArrayList<>(); + for (BlockMetaData block : blocks) { + long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset(); + if (firstDataPage >= splitStart && firstDataPage < splitEnd) { + selected.add(block); + } + } + FilterPredicate predicate = ParquetRecordReaderBase.toFilterPredicate(jobConf, fileSchema); + if (predicate != null) { + selected = RowGroupFilter.filterRowGroups(FilterCompat.get(predicate), selected, fileSchema); + } + Map rowGroupOf = new IdentityHashMap<>(); + for (int i = 0; i < blocks.size(); ++i) { + rowGroupOf.put(blocks.get(i), i); + } + counters.incrCounter(LlapIOCounters.SELECTED_ROWGROUPS, selected.size()); + + FileSystem fs = path.getFileSystem(jobConf); + try (FSDataInputStream fileStream = openFile(fs)) { + ParquetRangeBuffers buffers = ParquetRangeBuffers.forStream(fileStream); + Deque inFlight = new ArrayDeque<>(); + try { + for (int i = 0; i < selected.size() && !isStopped.get(); ++i) { + if (inFlight.isEmpty()) { + inFlight.add(startFetch(fileStream, buffers, allocator, maxAlloc, projected, selected.get(i), + rowGroupOf.get(selected.get(i)))); + } + // The next row group's requests go out now so its transfer overlaps this one's decode. + if (i + 1 < selected.size() && !isStopped.get() + && bytes(inFlight.peek()) + bytes(projected, selected.get(i + 1)) <= lookaheadBudget) { + inFlight.add(startFetch(fileStream, buffers, allocator, maxAlloc, projected, selected.get(i + 1), + rowGroupOf.get(selected.get(i + 1)))); + } + finishFetch(allocator, buffers, inFlight.poll()); + } + } finally { + for (Fetch fetch : inFlight) { + abandon(allocator, fetch); + } + } + } + } + + /** Whether the projection reaches into a group type, which this reader does not decode. */ + public boolean projectsNestedTypes() { + for (Type field : requestedSchema.getFields()) { + if (!field.isPrimitive()) { + return true; + } + } + return false; + } + + /** File-schema positions of the requested fields; column chunks follow the schema order. */ + private static int[] projectedLeaves(MessageType requestedSchema, MessageType fileSchema) { + List leaves = new ArrayList<>(); + for (Type field : requestedSchema.getFields()) { + if (fileSchema.containsField(field.getName())) { + leaves.add(fileSchema.getFieldIndex(field.getName())); + } + } + return leaves.stream().mapToInt(Integer::intValue).toArray(); + } + + private static long bytes(int[] projected, BlockMetaData block) { + long total = 0; + for (int leaf : projected) { + total += block.getColumns().get(leaf).getTotalSize(); + } + return total; + } + + private static long bytes(Fetch fetch) { + long total = 0; + for (ColumnChunkMetaData chunk : fetch.batch.chunks) { + total += chunk.getTotalSize(); + } + return total; + } + + /** One row group on its way in: buffers planned, missing ranges requested, not yet decoded. */ + private static final class Fetch { + private final ParquetEncodedColumnBatch batch = new ParquetEncodedColumnBatch(); + private final List columns = new ArrayList<>(); + private final List misses = new ArrayList<>(); + private final List runs = new ArrayList<>(); + } + + /** The buffers covering one column chunk, in file order, and which stretches of them are new. */ + private record ColumnPlan(List parts, List missRuns) { + ColumnPlan() { + this(new ArrayList<>(), new ArrayList<>()); + } + } + + /** One cache buffer's worth of a column chunk: a hit handed back by the cache, or a miss to fill. */ + private static final class Part { + private MemoryBuffer buffer; + private final DiskRange range; + private final boolean miss; + /** We hold one ref to release; until then a miss is a raw allocation to free. */ + private boolean owned; + + Part(MemoryBuffer buffer, DiskRange range, boolean miss) { + this.buffer = buffer; + this.range = range; + this.miss = miss; + this.owned = !miss; + } + } + + /** {@code count} consecutive parts covering one contiguous missing sub-range; cached as a unit. */ + private record MissRun(int firstPart, int count) { + } + + /** One vectored range and the cache buffers it fills, in file order. */ + private record Run(FileRange range, List parts) { + } + + private Fetch startFetch(FSDataInputStream fileStream, ParquetRangeBuffers buffers, Allocator allocator, + int maxAlloc, int[] projected, BlockMetaData block, int rg) + throws IOException { + Fetch fetch = new Fetch(); + ColumnChunkMetaData[] chunks = new ColumnChunkMetaData[projected.length]; + for (int pc = 0; pc < projected.length; ++pc) { + chunks[pc] = block.getColumns().get(projected[pc]); + } + fetch.batch.init(fileKey, rg, chunks); + try { + for (ColumnChunkMetaData chunk : chunks) { + ColumnPlan column = new ColumnPlan(); + fetch.columns.add(column); + planColumnChunk(allocator, maxAlloc, chunk.getStartingPos(), + chunk.getStartingPos() + chunk.getTotalSize(), column, fetch.misses); + } + requestMisses(fileStream, buffers, fetch, layout.maxRangeBytes()); + } catch (Throwable t) { + abandon(allocator, fetch); + throw t; + } + return fetch; + } + + private void finishFetch(Allocator allocator, ParquetRangeBuffers buffers, Fetch fetch) + throws IOException, InterruptedException { + try { + receiveMisses(buffers, fetch); + for (ColumnPlan column : fetch.columns) { + putColumn(allocator, column); + } + for (int pc = 0; pc < fetch.columns.size(); ++pc) { + assemble(fetch.batch, pc, fetch.columns.get(pc).parts); + } + // consumeData returns the batch on success; after a throw it is still ours. + consumer.consumeData(fetch.batch); + } catch (Throwable t) { + abandon(allocator, fetch); + throw t; + } + } + + /** Drops a fetch that will not be decoded: outstanding requests are cancelled, buffers released. */ + private void abandon(Allocator allocator, Fetch fetch) { + for (Run run : fetch.runs) { + run.range.getData().cancel(true); + } + for (ColumnPlan column : fetch.columns) { + for (Part part : column.parts) { + if (part.owned) { + bufferManager.decRefBuffer(part.buffer); + } else { + allocator.deallocate(part.buffer); + } + } + } + } + + /** Package-private so a test can observe the read pattern. */ + FSDataInputStream openFile(FileSystem fs) throws IOException { + return fs.open(path); + } + + /** Lets the allocator abandon a wait for memory once the fragment is cancelled. */ + private void allocateMultiple(Allocator allocator, MemoryBuffer[] dest, int size) { + if (allocator instanceof StoppableAllocator) { + ((StoppableAllocator) allocator).allocateMultiple(dest, size, DATA_BUFFER_FACTORY, isStopped); + } else { + allocator.allocateMultiple(dest, size, DATA_BUFFER_FACTORY); + } + } + + /** + * Works out which buffers cover column chunk {@code [start, end)} without touching the file: cache + * hits as returned by getFileData, and freshly allocated power-of-two buffers for everything + * missing. Misses are appended to {@code allMisses} so the whole row group can be read at once. + */ + private void planColumnChunk(Allocator allocator, int maxAlloc, long start, long end, + ColumnPlan column, List allMisses) throws IOException { + DiskRangeList head = new DiskRangeList(start, end); + if (fileKey != null) { + head = lowLevelCache.getFileData(fileKey, head, 0, ParquetCacheLayout.CACHE_CHUNK_FACTORY, + counters, new BooleanRef()); + } + DiskRangeList current = head; + try { + for (; current != null; current = current.next) { + if (current.hasData()) { + column.parts.add(new Part(((CacheChunk) current).getBuffer(), current, false)); + continue; + } + LlapHiveUtils.throwIfCacheOnlyRead(cacheOnly); + int[] sizes = layout.bufferSizes(current.getEnd() - current.getOffset()); + column.missRuns.add(new MissRun(column.parts.size(), sizes.length)); + long partFrom = current.getOffset(); + for (int size : sizes) { + MemoryBuffer[] one = new MemoryBuffer[1]; + allocateMultiple(allocator, one, size); + // The cache accounts and serves the bytes up to the buffer's limit. + ByteBuffer raw = one[0].getByteBufferRaw(); + raw.limit(raw.position() + size); + Part part = new Part(one[0], new DiskRange(partFrom, partFrom + size), true); + column.parts.add(part); + allMisses.add(part); + partFrom += size; + } + } + } catch (Throwable t) { + // Hits past the failure point are still locked by getFileData; the caller only knows about + // the parts already recorded. + for (current = current.next; current != null; current = current.next) { + if (current.hasData()) { + bufferManager.decRefBuffer(((CacheChunk) current).getBuffer()); + } + } + throw t; + } + } + + /** + * Requests every missing buffer of a row group in one vectored call, one range per run of + * adjacent buffers. Column chunks sit back to back in the file, so a projection that keeps + * neighbouring columns reads them together; runs are capped so the row group arrives as + * several concurrent requests rather than one. A filesystem without a vectored implementation + * reads the ranges in turn. + */ + private static void requestMisses(FSDataInputStream fileStream, ParquetRangeBuffers buffers, Fetch fetch, + int maxRange) throws IOException { + List misses = fetch.misses; + if (misses.isEmpty()) { + return; + } + misses.sort(Comparator.comparingLong(part -> part.range.getOffset())); + List runs = new ArrayList<>(); + for (int i = 0; i < misses.size(); ) { + long from = misses.get(i).range.getOffset(); + long to = misses.get(i).range.getEnd(); + int j = i + 1; + for (; j < misses.size(); ++j) { + DiskRange next = misses.get(j).range; + if (next.getOffset() != to || next.getEnd() - from > maxRange) { + break; + } + to = next.getEnd(); + } + runs.add(new Run(FileRange.createFileRange(from, (int) (to - from)), misses.subList(i, j))); + i = j; + } + List ranges = new ArrayList<>(runs.size()); + for (Run run : runs) { + ranges.add(run.range); + } + fileStream.readVectored(ranges, buffers::allocate, buffers::release); + // Only requests the stream accepted are the fetch's to wait for or cancel. + fetch.runs.addAll(runs); + } + + /** Waits for the requested runs and copies each into the cache buffers it covers. */ + private static void receiveMisses(ParquetRangeBuffers buffers, Fetch fetch) throws IOException { + for (Run run : fetch.runs) { + ByteBuffer data = FutureIO.awaitFuture(run.range.getData()); + for (Part part : run.parts) { + int length = part.range.getLength(); + ByteBuffer src = data.duplicate(); + src.position(data.position() + (int) (part.range.getOffset() - run.range.getOffset())); + src.limit(src.position() + length); + part.buffer.getByteBufferRaw().duplicate().put(src); + } + buffers.release(data); + } + } + + /** Hands one column's freshly read buffers to the cache; every part then carries a ref we own. */ + private void putColumn(Allocator allocator, ColumnPlan column) { + for (MissRun run : column.missRuns) { + List parts = column.parts.subList(run.firstPart, run.firstPart + run.count); + if (fileKey == null) { + for (Part part : parts) { + bufferManager.incRefBuffer(part.buffer); + } + } else { + MemoryBuffer[] fresh = new MemoryBuffer[run.count]; + MemoryBuffer[] cached = new MemoryBuffer[run.count]; + DiskRange[] ranges = new DiskRange[run.count]; + for (int i = 0; i < run.count; ++i) { + fresh[i] = cached[i] = parts.get(i).buffer; + ranges[i] = parts.get(i).range; + } + lowLevelCache.putFileData(fileKey, ranges, cached, 0, Priority.NORMAL, counters, cacheTag); + for (int i = 0; i < run.count; ++i) { + if (cached[i] != fresh[i]) { + // The cache kept its own buffer (locked for us) and unlocked ours without freeing it. + allocator.deallocate(fresh[i]); + parts.get(i).buffer = cached[i]; + } + } + } + for (Part part : parts) { + part.owned = true; + } + } + } + + private static void assemble(ParquetEncodedColumnBatch batch, int pc, List parts) { + int n = parts.size(); + batch.columnBuffers[pc] = new MemoryBuffer[n]; + batch.bufferOffsets[pc] = new long[n]; + batch.bufferLengths[pc] = new int[n]; + for (int i = 0; i < n; ++i) { + Part part = parts.get(i); + batch.columnBuffers[pc][i] = part.buffer; + batch.bufferOffsets[pc][i] = part.range.getOffset(); + batch.bufferLengths[pc][i] = part.range.getLength(); + } + } + + @Override + public void returnData(ParquetEncodedColumnBatch batch) { + for (MemoryBuffer[] column : batch.columnBuffers) { + for (MemoryBuffer buffer : column) { + bufferManager.decRefBuffer(buffer); + } + } + } + + @Override + public void pause() { + } + + @Override + public void unpause() { + } + + @Override + public void stop() { + isStopped.set(true); + } +} diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java new file mode 100644 index 000000000000..eaa5cbfbcd11 --- /dev/null +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hive.llap.io.api.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; +import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer; +import org.apache.hadoop.hive.llap.io.decode.ReadPipeline; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.plan.MapWork; +import org.apache.hadoop.hive.ql.plan.PartitionDesc; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Reporter; +import org.apache.orc.impl.SchemaEvolution; +import org.junit.Test; + +/** + * {@link LlapRecordReader} over a scripted read pipeline: when its IO starts, and how a decoded + * batch reaches the row batch. + */ +public class TestLlapRecordReader { + + private static final int ROWS = 5; + + private final InlineExecutor executor = new InlineExecutor(); + private ColumnVectorBatch cvb; + + /** A two-column plan. */ + private static JobConf job() { + JobConf job = new JobConf(new HiveConf()); + job.set(IOConstants.COLUMNS, "id,val"); + job.set(IOConstants.COLUMNS_TYPES, "bigint,bigint"); + ColumnProjectionUtils.setReadColumns(job, List.of(0, 1)); + HiveConf.setVar(job, HiveConf.ConfVars.PLAN, "//tmp"); + MapWork mapWork = new MapWork(); + TypeInfo[] types = {TypeInfoFactory.longTypeInfo, TypeInfoFactory.longTypeInfo}; + mapWork.setVectorizedRowBatchCtx(new VectorizedRowBatchCtx(new String[] {"id", "val"}, types, null, + new int[] {0, 1}, 0, 0, new VirtualColumn[0], new String[0], null)); + Utilities.setMapWork(job, mapWork); + return job; + } + + /** {@code id} is the physical row, {@code val} ten times that. */ + private static ColumnVectorBatch batch() { + ColumnVectorBatch cvb = new ColumnVectorBatch(2); + for (int c = 0; c < 2; c++) { + LongColumnVector column = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE); + for (int i = 0; i < ROWS; i++) { + column.vector[i] = c == 0 ? i : 10L * i; + } + cvb.cols[c] = column; + } + cvb.size = ROWS; + return cvb; + } + + private LlapRecordReader reader(JobConf job) throws Exception { + LlapRecordReader reader = LlapRecordReader.create(job, new FileSplit(new Path("/data"), 0, 1, (String[]) null), + List.of(0, 1), "host", new ScriptedProducer(), executor, null, null, Reporter.NULL, new HiveConf()); + assertNotNull(reader); + return reader; + } + + @Test + public void aDecodedBatchReachesTheRowBatch() throws Exception { + cvb = batch(); + LlapRecordReader reader = reader(job()); + reader.start(); + VectorizedRowBatch vrb = reader.createValue(); + assertTrue(reader.next(NullWritable.get(), vrb)); + assertFalse(vrb.selectedInUse); + assertEquals(ROWS, vrb.size); + for (int i = 0; i < ROWS; i++) { + assertEquals(i, ((LongColumnVector) vrb.cols[0]).vector[i]); + assertEquals(10L * i, ((LongColumnVector) vrb.cols[1]).vector[i]); + } + assertFalse(reader.next(NullWritable.get(), vrb)); + reader.close(); + } + + @Test + public void readerStartsItsIoOnStart() throws Exception { + cvb = batch(); + LlapRecordReader reader = reader(job()); + assertEquals(0, executor.runs); + reader.start(); + assertEquals(1, executor.runs); + reader.close(); + } + + /** Hands {@link #cvb} to the reader and finishes, all on the caller's thread. */ + private final class ScriptedProducer implements ColumnVectorProducer { + + @Override + public ReadPipeline createReadPipeline(Consumer consumer, FileSplit split, + ColumnVectorProducer.Includes includes, SearchArgument sarg, QueryFragmentCounters counters, + ColumnVectorProducer.SchemaEvolutionFactory sef, InputFormat sourceInputFormat, + Deserializer sourceSerDe, Reporter reporter, JobConf job, Map parts) { + return new ReadPipeline() { + @Override + public Callable getReadCallable() { + return () -> { + consumer.consumeData(cvb); + consumer.setDone(); + return null; + }; + } + + @Override + public SchemaEvolution getSchemaEvolution() { + return null; + } + + @Override + public void pause() { + } + + @Override + public void unpause() { + } + + @Override + public void stop() { + } + + @Override + public void returnData(ColumnVectorBatch data) { + } + }; + } + } + + /** Runs every submitted task on the submitting thread and counts them. */ + private static final class InlineExecutor extends AbstractExecutorService { + int runs; + + @Override + public void execute(Runnable command) { + runs++; + command.run(); + } + + @Override + public void shutdown() { + } + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + } +} diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java new file mode 100644 index 000000000000..a4f63b4e6959 --- /dev/null +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -0,0 +1,1066 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.io.encoded; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.IntFunction; +import java.util.function.Predicate; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.hive.common.io.Allocator; +import org.apache.hadoop.hive.common.io.CacheTag; +import org.apache.hadoop.hive.common.io.DataCache.BooleanRef; +import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory; +import org.apache.hadoop.hive.common.io.DiskRange; +import org.apache.hadoop.hive.common.io.DiskRangeList; +import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.llap.ParquetCacheLayout; +import org.apache.hadoop.hive.llap.cache.BuddyAllocator; +import org.apache.hadoop.hive.llap.cache.BufferUsageManager; +import org.apache.hadoop.hive.llap.cache.LlapAllocatorBuffer; +import org.apache.hadoop.hive.llap.cache.LlapDataBuffer; +import org.apache.hadoop.hive.llap.cache.LowLevelCache; +import org.apache.hadoop.hive.llap.cache.LowLevelCacheCounters; +import org.apache.hadoop.hive.llap.cache.LowLevelCacheImpl; +import org.apache.hadoop.hive.llap.cache.LowLevelLrfuCachePolicy; +import org.apache.hadoop.hive.llap.cache.TestBuddyAllocatorForceEvict; +import org.apache.hadoop.hive.llap.counters.LlapIOCounters; +import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; +import org.apache.hadoop.hive.llap.io.api.LlapProxy; +import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch; +import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes; +import org.apache.hadoop.hive.llap.io.decode.ParquetEncodedDataConsumer; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.orc.encoded.CacheChunk; +import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; +import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.orc.TypeDescription; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; +import org.apache.tez.common.counters.TezCounters; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Drives {@link ParquetEncodedDataReader} + {@link ParquetEncodedDataConsumer} over an in-process + * BuddyAllocator/LowLevelCacheImpl against a real three-row-group Parquet file. The footer goes through + * the real LLAP footer cache (LlapProxy in cache mode); the data cache is per test. + */ +public class TestParquetEncodedDataReader { + + private static final String COLUMNS = "id,big,dec,name,ratio,flag"; + private static final String TYPES = "int,bigint,decimal(7,2),string,double,boolean"; + private static final int ROWS_PER_GROUP = 1500; + private static final int ROW_GROUPS = 3; + private static final int ROWS = ROWS_PER_GROUP * ROW_GROUPS; + // Smaller than every column chunk so each chunk spans several cache buffers. + private static final int MAX_ALLOC = 4096; + + private static final MessageType SCHEMA = Types.buildMessage() + .optional(PrimitiveTypeName.INT32).named("id") + .optional(PrimitiveTypeName.INT64).named("big") + .optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(4) + .as(LogicalTypeAnnotation.decimalType(2, 7)).named("dec") + .optional(PrimitiveTypeName.BINARY).as(LogicalTypeAnnotation.stringType()).named("name") + .optional(PrimitiveTypeName.DOUBLE).named("ratio") + .optional(PrimitiveTypeName.BOOLEAN).named("flag") + .named("hive_schema"); + + private static final LlapDaemonCacheMetrics CACHE_METRICS = + LlapDaemonCacheMetrics.create("TestParquetEncodedDataReader-cache", "1"); + private static final LlapDaemonIOMetrics IO_METRICS = + LlapDaemonIOMetrics.create("TestParquetEncodedDataReader-io", "1", null); + + private static HiveConf daemonConf; + private static java.nio.file.Path tmpDir; + private static Path file; + private static long fileLength; + private static ParquetMetadata footer; + + private LowLevelCacheImpl cache; + private Ledger ledger; + /** hive.llap.io.encode.alloc.size at its default, the grain these expectations assume. */ + private static final int FLOOR = 256 * 1024; + + private boolean failFirstRange; + private boolean unslicedBuffers; + private boolean stopAfterFirstBatch; + private boolean failDecode; + private ParquetEncodedDataReader reader; + private int decoded; + private final List decodedAtRequest = new ArrayList<>(); + + @BeforeClass + public static void setUpClass() throws Exception { + daemonConf = new HiveConf(); + HiveConf.setVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MODE, "cache"); + HiveConf.setVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE, "64Mb"); + HiveConf.setBoolVar(daemonConf, ConfVars.LLAP_TRACK_CACHE_USAGE, false); + HiveConf.setIntVar(daemonConf, ConfVars.LLAP_LRFU_BP_WRAPPER_SIZE, 1); + + tmpDir = Files.createTempDirectory("llap-parquet-native"); + file = new Path(tmpDir.toString(), "data.parquet"); + writeFile(file, daemonConf); + FileSystem fs = file.getFileSystem(daemonConf); + fileLength = fs.getFileStatus(file).getLen(); + footer = ParquetFileReader.readFooter( + HadoopInputFile.fromPath(file, daemonConf), ParquetMetadataConverter.NO_FILTER); + assertEquals(ROW_GROUPS, footer.getBlocks().size()); + for (BlockMetaData block : footer.getBlocks()) { + assertEquals(ROWS_PER_GROUP, block.getRowCount()); + } + + LlapProxy.setDaemon(true); + LlapProxy.initializeLlapIo(daemonConf); + } + + @AfterClass + public static void tearDownClass() throws IOException { + LlapProxy.close(); + FileSystem.getLocal(daemonConf).delete(new Path(tmpDir.toString()), true); + } + + @Before + public void setUp() { + LowLevelLrfuCachePolicy policy = new LowLevelLrfuCachePolicy(MAX_ALLOC, 64L << 20, daemonConf); + BuddyAllocator allocator = TestBuddyAllocatorForceEvict.create(MAX_ALLOC, 2, 8 << 20, true, false); + cache = new LowLevelCacheImpl(CACHE_METRICS, policy, allocator, true); + ledger = new Ledger(cache); + failFirstRange = false; + unslicedBuffers = false; + stopAfterFirstBatch = false; + failDecode = false; + decoded = 0; + decodedAtRequest.clear(); + } + + @Test + public void testFullReadAllColumns() throws Exception { + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + for (int i : new int[] {0, 1, 7, 11, 77, 1499, 1500, 2999, 3000, ROWS - 1}) { + assertArrayEquals("row " + i, expectedRow(i), run.rows.get(i)); + } + assertTrue(run.firstBatchCols[2] instanceof Decimal64ColumnVector); + assertEquals(2, ((Decimal64ColumnVector) run.firstBatchCols[2]).scale); + assertEquals(Arrays.asList(1024, 476, 1024, 476, 1024, 476), run.batchSizes); + assertEquals(ROW_GROUPS, run.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + assertEquals(ROWS, run.counter(LlapIOCounters.ROWS_EMITTED)); + } + + @Test + public void testSplitCoveringSecondRowGroup() throws Exception { + List blocks = footer.getBlocks(); + long start = blocks.get(1).getStartingPos(), end = blocks.get(2).getStartingPos(); + Run run = read(jobConf(COLUMNS, TYPES, 0, 3), new FileSplit(file, start, end - start, (String[]) null)); + + run.assertClean(); + assertEquals(1, run.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + assertEquals(ROWS_PER_GROUP, run.rows.size()); + for (int i = 0; i < ROWS_PER_GROUP; ++i) { + int row = ROWS_PER_GROUP + i; + assertArrayEquals("row " + row, project(expectedRow(row), 0, 3), run.rows.get(i)); + } + } + + @Test + public void testDisjointSplitsEmitEveryRowOnce() throws Exception { + long cut = footer.getBlocks().get(2).getStartingPos(); + Run first = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), new FileSplit(file, 0, cut, (String[]) null)); + Run second = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), + new FileSplit(file, cut, fileLength - cut, (String[]) null)); + + first.assertClean(); + second.assertClean(); + assertEquals(2, first.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + assertEquals(1, second.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + List all = new ArrayList<>(first.rows); + all.addAll(second.rows); + assertEquals(ROWS, all.size()); + for (int i = 0; i < ROWS; ++i) { + assertArrayEquals("row " + i, expectedRow(i), all.get(i)); + } + } + + @Test + public void testProjectionOrderAndSubset() throws Exception { + Run run = read(jobConf(COLUMNS, TYPES, 4, 0, 3), wholeFile()); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + assertEquals(3, run.firstBatchCols.length); + assertTrue(run.firstBatchCols[0] instanceof DoubleColumnVector); + assertTrue(run.firstBatchCols[1] instanceof LongColumnVector); + assertTrue(run.firstBatchCols[2] instanceof BytesColumnVector); + for (int i : new int[] {0, 7, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, project(expectedRow(i), 4, 0, 3), run.rows.get(i)); + } + } + + @Test + public void testCacheAllocationMatchesBytesRead() throws Exception { + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + run.assertClean(); + long used = run.counter(LlapIOCounters.ALLOCATED_USED_BYTES); + long allocated = run.counter(LlapIOCounters.ALLOCATED_BYTES); + assertEquals(run.counter(LlapIOCounters.CACHE_MISS_BYTES), used); + // Every buffer but the sub-floor remainder of each chunk is a power of two, so the rounding + // overhead is bounded by one MAX_ALLOC per column chunk. + long chunks = (long) ROW_GROUPS * 6; + assertTrue("allocated " + allocated + " vs used " + used, allocated - used <= chunks * MAX_ALLOC); + } + + @Test + public void testAdjacentColumnChunksReadInOneRun() throws Exception { + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2), wholeFile()); + + run.assertClean(); + // This fixture allocates in 4 Kb buffers, far below the cap, so what the loop below really + // pins is that adjacent chunks merge -- the cap itself never binds here. + int maxRun = new ParquetCacheLayout(MAX_ALLOC, FLOOR).maxRangeBytes(); + long previousEnd = -1; + for (long[] read : run.reads) { + assertTrue("read of " + read[1] + " exceeds the largest a range may be", read[1] <= maxRun); + assertTrue("reads must move forward", read[0] > previousEnd - maxRun); + previousEnd = read[0] + read[1]; + } + // Chunks of id, big and dec sit back to back: within a row group every read starts where the + // previous one ended, so the only jumps are the three row-group starts. + int jumps = 0; + previousEnd = -1; + for (long[] read : run.reads) { + if (read[0] != previousEnd) { + ++jumps; + } + previousEnd = read[0] + read[1]; + } + assertEquals(ROW_GROUPS, jumps); + boolean crossesColumns = false; + for (BlockMetaData block : footer.getBlocks()) { + long bigStart = block.getColumns().get(1).getStartingPos(); + for (long[] read : run.reads) { + crossesColumns |= read[0] < bigStart && read[0] + read[1] > bigStart; + } + } + assertTrue("no read spans the id/big chunk boundary", crossesColumns); + assertTrue("reads " + run.reads.size() + " should be far fewer than buffers " + run.buffers.size(), + run.reads.size() * 3 <= run.buffers.size()); + for (int i : new int[] {0, 11, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, project(expectedRow(i), 0, 1, 2), run.rows.get(i)); + } + } + + @Test + public void testGapBetweenProjectedChunksSplitsTheRun() throws Exception { + Run run = read(jobConf(COLUMNS, TYPES, 0, 2), wholeFile()); + + run.assertClean(); + for (BlockMetaData block : footer.getBlocks()) { + long bigStart = block.getColumns().get(1).getStartingPos(); + long bigEnd = bigStart + block.getColumns().get(1).getTotalSize(); + for (long[] read : run.reads) { + long end = read[0] + read[1]; + assertFalse("read [" + read[0] + "," + end + ") crosses the unprojected chunk", + read[0] < bigEnd && end > bigStart); + } + } + for (int i : new int[] {0, 11, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, project(expectedRow(i), 0, 2), run.rows.get(i)); + } + } + + @Test + public void testCachedBuffersAreNotReread() throws Exception { + read(jobConf(COLUMNS, TYPES, 1), wholeFile()).assertClean(); + Run second = read(jobConf(COLUMNS, TYPES, 0, 1, 2), wholeFile()); + + second.assertClean(); + for (BlockMetaData block : footer.getBlocks()) { + long bigStart = block.getColumns().get(1).getStartingPos(); + long bigEnd = bigStart + block.getColumns().get(1).getTotalSize(); + for (long[] read : second.reads) { + long end = read[0] + read[1]; + assertFalse("read [" + read[0] + "," + end + ") re-read the cached chunk", + read[0] < bigEnd && end > bigStart); + } + } + assertTrue(second.counter(LlapIOCounters.CACHE_HIT_BYTES) > 0); + for (int i : new int[] {0, 11, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, project(expectedRow(i), 0, 1, 2), second.rows.get(i)); + } + } + + @Test + public void testNextRowGroupIsRequestedBeforeCurrentDecodes() throws Exception { + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + run.assertClean(); + // Row group 1's request goes out before row group 0 is decoded; row group 2's after it. + assertEquals(Arrays.asList(0, 0, 1), decodedAtRequest); + } + + @Test + public void testPooledBuffersReadEveryRow() throws Exception { + unslicedBuffers = true; + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + for (int i : new int[] {0, 1, 1499, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, expectedRow(i), run.rows.get(i)); + } + } + + @Test + public void testPooledBufferIsLimitedToItsRange() throws Exception { + unslicedBuffers = true; + // The second read of a run takes a pooled buffer that the first left longer than the range. + read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + Run run = read(jobConf(COLUMNS, TYPES, 3), wholeFile()); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + for (int i : new int[] {0, 1499, ROWS - 1}) { + assertArrayEquals("row " + i, project(expectedRow(i), 3), run.rows.get(i)); + } + } + + @Test + public void testFailedRangeAbortsTheSplitWithoutLeaks() throws Exception { + failFirstRange = true; + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + assertNotNull("the failed range must surface", run.error); + assertTrue(String.valueOf(run.error), String.valueOf(run.error).contains("boom")); + assertEquals(0, run.rows.size()); + ledger.assertNothingLeaked(); + } + + @Test + public void testStopWithNextRowGroupInFlightReleasesIt() throws Exception { + stopAfterFirstBatch = true; + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + assertTrue("stopped after the first row group", run.rows.size() < ROWS); + assertTrue("row group 1 had been requested", run.reads.size() > 1); + ledger.assertNothingLeaked(); + } + + @Test + public void testDecodeFailureReleasesCachedBuffers() throws Exception { + failDecode = true; + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2), wholeFile()); + + assertNotNull(run.error); + assertTrue(String.valueOf(run.error), String.valueOf(run.error).contains("decode failed")); + ledger.assertNothingLeaked(); + } + + @Test + public void testCollisionKeepsTheCachedCopyAndFreesOurs() throws Exception { + ledger.collide = true; + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + for (int i : new int[] {0, 11, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, expectedRow(i), run.rows.get(i)); + } + assertEquals("every buffer we allocated was refused and freed", ledger.allocated.size(), ledger.freed.size()); + assertTrue("nothing of ours reached the cache", ledger.accepted.isEmpty()); + } + + @Test + public void testSargPrunesAllRowGroups() throws Exception { + JobConf job = jobConf(COLUMNS, TYPES, 0, 3); + setSarg(job, SearchArgumentFactory.newBuilder() + .equals("id", PredicateLeaf.Type.LONG, (long) ROWS + 1).build()); + Run run = read(job, wholeFile()); + + run.assertClean(); + assertEquals(0, run.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + assertEquals(0, run.rows.size()); + assertEquals(0, run.counter(LlapIOCounters.ROWS_EMITTED)); + } + + @Test + public void testSargKeepsMatchingRowGroup() throws Exception { + JobConf job = jobConf(COLUMNS, TYPES, 0, 3); + long lo = 2 * ROWS_PER_GROUP + 100, hi = lo + 50; + setSarg(job, SearchArgumentFactory.newBuilder().between("id", PredicateLeaf.Type.LONG, lo, hi).build()); + Run run = read(job, wholeFile()); + + run.assertClean(); + assertEquals(1, run.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + assertEquals(ROWS_PER_GROUP, run.rows.size()); + assertArrayEquals(project(expectedRow(2 * ROWS_PER_GROUP), 0, 3), run.rows.get(0)); + } + + @Test + public void testSecondReadHitsCache() throws Exception { + long chunkBytes = projectedChunkBytes(1, 3); + Run cold = read(jobConf(COLUMNS, TYPES, 1, 3), wholeFile()); + Run warm = read(jobConf(COLUMNS, TYPES, 1, 3), wholeFile()); + + cold.assertClean(); + warm.assertClean(); + assertEquals(0, cold.counter(LlapIOCounters.CACHE_HIT_BYTES)); + assertEquals(chunkBytes, cold.counter(LlapIOCounters.CACHE_MISS_BYTES)); + assertEquals(chunkBytes, warm.counter(LlapIOCounters.CACHE_HIT_BYTES)); + assertEquals(0, warm.counter(LlapIOCounters.CACHE_MISS_BYTES)); + assertTrue(CACHE_METRICS.getCacheHitBytes() >= chunkBytes); + + assertEquals(cold.rows.size(), warm.rows.size()); + for (int i = 0; i < ROWS; ++i) { + assertArrayEquals("row " + i, cold.rows.get(i), warm.rows.get(i)); + } + assertEquals(new HashSet<>(cold.buffers), new HashSet<>(warm.buffers)); + assertTrue(cold.buffers.size() > 1); + for (MemoryBuffer b : warm.buffers) { + assertFalse(b.toString(), ((LlapAllocatorBuffer) b).isLocked()); + } + } + + @Test + public void testMissingTrailingColumnReadsAsNulls() throws Exception { + Run run = read(jobConf(COLUMNS + ",extra", TYPES + ",string", 0, 6), wholeFile()); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + assertTrue(run.firstBatchCols[1] instanceof BytesColumnVector); + for (int i : new int[] {0, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, new Object[] {(long) i, null}, run.rows.get(i)); + } + } + + @Test + public void testEvolvedColumnsReorderedDefaultedAndRecreated() throws Exception { + // Hive order differs from the file (ratio before id), "added" is absent with an initial default, + // and a recreated field carries the Iceberg placeholder name that no file column matches. + String columns = "ratio,id,<>,added,name"; + Run run = read(jobConf(columns, "double,int,string,int,string", 0, 1, 2, 3, 4), wholeFile(), + Map.of("added", 42)); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + assertTrue(run.firstBatchCols[2] instanceof BytesColumnVector); + assertTrue(run.firstBatchCols[3] instanceof LongColumnVector); + assertTrue(run.firstBatchCols[3].isRepeating); + for (int i : new int[] {0, 7, 1500, ROWS - 1}) { + assertArrayEquals("row " + i, new Object[] {ratio(i), (long) i, null, 42L, i % 7 == 0 ? null : "name-" + i}, + run.rows.get(i)); + } + } + + @Test + public void testStartRowInFileSkipsPrunedRowGroup() throws Exception { + JobConf job = jobConf(COLUMNS, TYPES, 0, 3); + setSarg(job, SearchArgumentFactory.newBuilder() + .in("id", PredicateLeaf.Type.LONG, 100L, (long) 2 * ROWS_PER_GROUP + 1000).build()); + Run run = read(job, wholeFile()); + + run.assertClean(); + assertEquals(2, run.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + assertArrayEquals(project(expectedRow(3000), 0, 3), run.rows.get(ROWS_PER_GROUP)); + } + + @Test + public void testCacheOnlyReadThrowsOnColdData() throws Exception { + read(jobConf(COLUMNS, TYPES, 0, 3), wholeFile()).assertClean(); + JobConf job = jobConf(COLUMNS, TYPES, 1); + HiveConf.setBoolVar(job, ConfVars.LLAP_IO_CACHE_ONLY, true); + Run run = read(job, wholeFile()); + + assertTrue(String.valueOf(run.error), run.error instanceof IOException); + assertTrue(run.error.getMessage(), run.error.getMessage().contains("cache only")); + assertFalse(run.done); + assertEquals(0, run.rows.size()); + } + + @Test + public void testNoFileKeyReadsUncached() throws Exception { + HiveConf noKeyConf = new HiveConf(daemonConf); + HiveConf.setBoolVar(noKeyConf, ConfVars.LLAP_CACHE_ALLOW_SYNTHETIC_FILEID, false); + Run run = read(jobConf(COLUMNS, TYPES, 0, 2), wholeFile(), noKeyConf); + + run.assertClean(); + assertEquals(ROWS, run.rows.size()); + assertEquals(0, run.counter(LlapIOCounters.CACHE_HIT_BYTES)); + assertEquals(0, run.counter(LlapIOCounters.CACHE_MISS_BYTES)); + for (int i : new int[] {0, 11, ROWS - 1}) { + assertArrayEquals("row " + i, project(expectedRow(i), 0, 2), run.rows.get(i)); + } + for (MemoryBuffer b : run.buffers) { + assertFalse(((LlapAllocatorBuffer) b).isLocked()); + } + } + + // ---- fixture ---- + + private static void writeFile(Path path, Configuration conf) throws IOException { + try (ParquetWriter writer = ExampleParquetWriter.builder(path) + .withConf(conf) + .withType(SCHEMA) + .withCompressionCodec(CompressionCodecName.SNAPPY) + .withRowGroupSize(1024) + .withMinRowCountForPageSizeCheck(ROWS_PER_GROUP) + .withMaxRowCountForPageSizeCheck(ROWS_PER_GROUP) + .build()) { + SimpleGroupFactory factory = new SimpleGroupFactory(SCHEMA); + for (int i = 0; i < ROWS; ++i) { + Group g = factory.newGroup(); + g.append("id", i); + g.append("big", bigValue(i)); + if (i % 11 != 0) { + g.append("dec", Binary.fromConstantByteArray(ByteBuffer.allocate(4).putInt((int) decUnscaled(i)).array())); + } + if (i % 7 != 0) { + g.append("name", "name-" + i); + } + g.append("ratio", ratio(i)); + g.append("flag", i % 3 == 0); + writer.write(g); + } + } + } + + private static long bigValue(int i) { + return i * 1_000_003L; + } + + private static long decUnscaled(int i) { + return i * 125L + 1; + } + + private static double ratio(int i) { + return i / 8.0; + } + + /** Values as the capturing consumer reads them back: longs for int/bigint/boolean/decimal64. */ + private static Object[] expectedRow(int i) { + return new Object[] { + (long) i, + bigValue(i), + i % 11 == 0 ? null : decUnscaled(i), + i % 7 == 0 ? null : "name-" + i, + ratio(i), + i % 3 == 0 ? 1L : 0L}; + } + + private static Object[] project(Object[] row, int... cols) { + Object[] out = new Object[cols.length]; + for (int i = 0; i < cols.length; ++i) { + out[i] = row[cols[i]]; + } + return out; + } + + private static FileSplit wholeFile() { + return new FileSplit(file, 0, fileLength, (String[]) null); + } + + private static long projectedChunkBytes(int... fileCols) { + long total = 0; + for (BlockMetaData block : footer.getBlocks()) { + for (int c : fileCols) { + ColumnChunkMetaData ccm = block.getColumns().get(c); + total += ccm.getTotalSize(); + } + } + return total; + } + + private static JobConf jobConf(String columns, String types, int... readColumnIds) { + JobConf job = new JobConf(daemonConf); + job.set(IOConstants.COLUMNS, columns); + job.set(IOConstants.COLUMNS_TYPES, types); + List ids = new ArrayList<>(); + for (int id : readColumnIds) { + ids.add(id); + } + ColumnProjectionUtils.setReadColumns(job, ids); + HiveConf.setVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED, "decimal_64"); + return job; + } + + private static void setSarg(JobConf job, SearchArgument sarg) { + job.set(ConvertAstToSearchArg.SARG_PUSHDOWN, ConvertAstToSearchArg.sargToKryo(sarg)); + } + + private Run read(JobConf job, FileSplit split) throws Exception { + return read(job, split, daemonConf, null); + } + + private Run read(JobConf job, FileSplit split, Map initialDefaults) throws Exception { + return read(job, split, daemonConf, initialDefaults); + } + + private Run read(JobConf job, FileSplit split, Configuration readerDaemonConf) throws Exception { + return read(job, split, readerDaemonConf, null); + } + + private Run read(JobConf job, FileSplit split, Configuration readerDaemonConf, + Map initialDefaults) throws Exception { + List projection = ColumnProjectionUtils.getReadColumnIDs(job); + TezCounters tezCounters = new TezCounters(); + QueryFragmentCounters counters = new QueryFragmentCounters(job, tezCounters); + CapturingConsumer downstream = new CapturingConsumer(); + List buffers = new ArrayList<>(); + ParquetEncodedDataConsumer edc = new ParquetEncodedDataConsumer( + downstream, includes(projection), counters, IO_METRICS, job) { + @Override + protected void decodeBatch(ParquetEncodedColumnBatch batch, Consumer consumer) + throws InterruptedException { + for (MemoryBuffer[] col : batch.columnBuffers) { + Collections.addAll(buffers, col); + } + if (failDecode) { + throw new IllegalStateException("decode failed"); + } + super.decodeBatch(batch, consumer); + if (++decoded == 1 && stopAfterFirstBatch) { + reader.stop(); + } + } + }; + edc.setInitialDefaults(initialDefaults); + List reads = new ArrayList<>(); + ParquetEncodedDataReader reader = new ParquetEncodedDataReader( + ledger, ledger, readerDaemonConf, job, split, includes(projection), edc, counters) { + @Override + FSDataInputStream openFile(FileSystem fs) throws IOException { + return new RecordingStream(super.openFile(fs), reads); + } + }; + this.reader = reader; + edc.init(reader, reader); + reader.loadFooter(); + reader.call(); + return new Run(downstream, tezCounters, buffers, reads, ledger); + } + + + /** + * Records every range of every vectored read as {offset, length} and how many batches had been + * decoded when the request went out; can fail the first range of the first request. + */ + private final class RecordingStream extends FSDataInputStream { + private final List reads; + private boolean first = true; + + RecordingStream(FSDataInputStream delegate, List reads) { + super(delegate); + this.reads = reads; + } + + /** Reports the wrapped stream's capabilities, less the ones this test suppresses. */ + @Override + public boolean hasCapability(String capability) { + if (unslicedBuffers && StreamCapabilities.VECTOREDIO_BUFFERS_SLICED.equals(capability)) { + return false; + } + return super.hasCapability(capability); + } + + @Override + public void readVectored(List ranges, IntFunction allocate) + throws IOException { + record(ranges); + super.readVectored(ranges, allocate); + injectFailure(ranges); + } + + @Override + public void readVectored(List ranges, IntFunction allocate, + java.util.function.Consumer release) throws IOException { + record(ranges); + super.readVectored(ranges, allocate, release); + injectFailure(ranges); + } + + private void record(List ranges) { + for (FileRange range : ranges) { + reads.add(new long[] {range.getOffset(), range.getLength()}); + } + decodedAtRequest.add(decoded); + } + + private void injectFailure(List ranges) { + if (first && failFirstRange) { + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new IOException("boom")); + ranges.get(0).setData(failed); + } + first = false; + } + } + + /** + * Stands between the reader and the cache to account for every buffer: allocated by the + * reader, freed by it, accepted by the cache, or handed out as a hit. + */ + private static final class Ledger implements LowLevelCache, BufferUsageManager, Allocator { + private final LowLevelCacheImpl cache; + private final Allocator allocator; + private final Set allocated = Collections.newSetFromMap(new IdentityHashMap<>()); + private final Set freed = Collections.newSetFromMap(new IdentityHashMap<>()); + private final Set accepted = Collections.newSetFromMap(new IdentityHashMap<>()); + private final Set hits = Collections.newSetFromMap(new IdentityHashMap<>()); + /** Inserts its own copy of every range first, so the reader's put collides. */ + boolean collide; + + Ledger(LowLevelCacheImpl cache) { + this.cache = cache; + this.allocator = cache.getAllocator(); + } + + void assertNothingLeaked() { + Set raw = Collections.newSetFromMap(new IdentityHashMap<>()); + raw.addAll(allocated); + raw.removeAll(freed); + raw.removeAll(accepted); + assertTrue("raw allocations never freed nor cached: " + raw, raw.isEmpty()); + Set both = Collections.newSetFromMap(new IdentityHashMap<>()); + both.addAll(freed); + both.retainAll(accepted); + assertTrue("cache-owned buffers raw-freed: " + both, both.isEmpty()); + for (MemoryBuffer b : allocated) { + assertFalse("still locked: " + b, ((LlapAllocatorBuffer) b).isLocked()); + } + for (MemoryBuffer b : hits) { + assertFalse("hit still locked: " + b, ((LlapAllocatorBuffer) b).isLocked()); + } + } + + @Override + public DiskRangeList getFileData(Object fileKey, DiskRangeList range, long baseOffset, + DiskRangeListFactory factory, LowLevelCacheCounters qfCounters, BooleanRef gotAllData) { + DiskRangeList result = cache.getFileData(fileKey, range, baseOffset, factory, qfCounters, gotAllData); + for (DiskRangeList r = result; r != null; r = r.next) { + if (r.hasData()) { + hits.add(((CacheChunk) r).getBuffer()); + } + } + return result; + } + + @Override + public long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] chunks, long baseOffset, + Priority priority, LowLevelCacheCounters qfCounters, CacheTag tag) { + if (collide) { + MemoryBuffer[] copies = new MemoryBuffer[chunks.length]; + for (int i = 0; i < chunks.length; ++i) { + MemoryBuffer[] one = new MemoryBuffer[1]; + allocator.allocateMultiple(one, ranges[i].getLength(), LlapDataBuffer::new); + ByteBuffer src = chunks[i].getByteBufferRaw().duplicate(); + src.limit(src.position() + ranges[i].getLength()); + ByteBuffer dest = one[0].getByteBufferRaw(); + dest.limit(dest.position() + ranges[i].getLength()); + dest.duplicate().put(src); + copies[i] = one[0]; + } + cache.putFileData(fileKey, ranges, copies, baseOffset, priority, qfCounters, tag); + cache.decRefBuffers(java.util.Arrays.asList(copies)); + } + long[] result = cache.putFileData(fileKey, ranges, chunks, baseOffset, priority, qfCounters, tag); + for (MemoryBuffer b : chunks) { + if (allocated.contains(b)) { + accepted.add(b); + } + } + return result; + } + + @Override + public void notifyEvicted(MemoryBuffer buffer) { + cache.notifyEvicted(buffer); + } + + @Override + public long markBuffersForProactiveEviction(Predicate predicate, boolean isInstantDeallocation) { + return cache.markBuffersForProactiveEviction(predicate, isInstantDeallocation); + } + + @Override + public Allocator getAllocator() { + return this; + } + + @Override + public void decRefBuffer(MemoryBuffer buffer) { + cache.decRefBuffer(buffer); + } + + @Override + public void decRefBuffers(List buffers) { + cache.decRefBuffers(buffers); + } + + @Override + public boolean incRefBuffer(MemoryBuffer buffer) { + // A raw allocation handed to the batch this way is freed by the cache on its last decref. + if (allocated.contains(buffer)) { + accepted.add(buffer); + } + return cache.incRefBuffer(buffer); + } + + @Override + public void allocateMultiple(MemoryBuffer[] dest, int size) { + allocator.allocateMultiple(dest, size); + Collections.addAll(allocated, dest); + } + + @Override + public void allocateMultiple(MemoryBuffer[] dest, int size, BufferObjectFactory factory) { + allocator.allocateMultiple(dest, size, factory); + Collections.addAll(allocated, dest); + } + + @Override + public MemoryBuffer createUnallocated() { + return allocator.createUnallocated(); + } + + @Override + public void deallocate(MemoryBuffer buffer) { + freed.add(buffer); + allocator.deallocate(buffer); + } + + @Override + public boolean isDirectAlloc() { + return allocator.isDirectAlloc(); + } + + @Override + public int getMaxAllocation() { + return allocator.getMaxAllocation(); + } + } + + private static final class Run { + final List rows; + final List batchSizes; + final ColumnVector[] firstBatchCols; + final boolean done; + final Throwable error; + final TezCounters counters; + final List buffers; + final List reads; + final Ledger ledger; + + Run(CapturingConsumer c, TezCounters counters, List buffers, List reads, + Ledger ledger) { + this.rows = c.rows; + this.batchSizes = c.batchSizes; + this.firstBatchCols = c.firstBatchCols; + this.done = c.done; + this.error = c.error; + this.counters = counters; + this.buffers = buffers; + this.reads = reads; + this.ledger = ledger; + } + + long counter(LlapIOCounters counter) { + return counters.findCounter(counter).getValue(); + } + + void assertClean() { + assertNull(error == null ? null : error.toString(), error); + assertTrue("setDone not reached", done); + for (MemoryBuffer b : buffers) { + assertFalse(b.toString(), ((LlapAllocatorBuffer) b).isLocked()); + } + ledger.assertNothingLeaked(); + } + } + + private static final class CapturingConsumer implements Consumer { + final List rows = new ArrayList<>(); + final List batchSizes = new ArrayList<>(); + ColumnVector[] firstBatchCols; + boolean done; + Throwable error; + + @Override + public void consumeData(ColumnVectorBatch cvb) { + if (firstBatchCols == null) { + firstBatchCols = cvb.cols.clone(); + } + batchSizes.add(cvb.size); + for (int r = 0; r < cvb.size; ++r) { + Object[] row = new Object[cvb.cols.length]; + for (int c = 0; c < cvb.cols.length; ++c) { + row[c] = value(cvb.cols[c], r); + } + rows.add(row); + } + } + + private static Object value(ColumnVector cv, int r) { + int ix = cv.isRepeating ? 0 : r; + if (!cv.noNulls && cv.isNull[ix]) { + return null; + } + if (cv instanceof LongColumnVector l) { + return l.vector[ix]; + } + if (cv instanceof DoubleColumnVector d) { + return d.vector[ix]; + } + if (cv instanceof BytesColumnVector b) { + return b.toString(ix); + } + throw new AssertionError("Unexpected vector " + cv.getClass()); + } + + @Override + public void setDone() { + done = true; + } + + @Override + public void setError(Throwable t) { + error = t; + } + } + + /** Only getPhysicalColumnIds is consulted by the Parquet pipeline. */ + private static Includes includes(List physicalColumnIds) { + return new Includes() { + @Override + public List getPhysicalColumnIds() { + return physicalColumnIds; + } + + @Override + public List getReaderLogicalColumnIds() { + return physicalColumnIds; + } + + @Override + public List getLogicalOrderedColumnIds() { + return physicalColumnIds; + } + + @Override + public boolean[] generateFileIncludes(TypeDescription fileSchema) { + throw new UnsupportedOperationException(); + } + + @Override + public TypeDescription[] getBatchReaderTypes(TypeDescription fileSchema) { + throw new UnsupportedOperationException(); + } + + @Override + public String[] getOriginalColumnNames(TypeDescription fileSchema) { + throw new UnsupportedOperationException(); + } + + @Override + public String getQueryId() { + return "test-query"; + } + + @Override + public boolean isProbeDecodeEnabled() { + return false; + } + + @Override + public byte getProbeMjSmallTablePos() { + return -1; + } + + @Override + public String getProbeCacheKey() { + return null; + } + + @Override + public String getProbeColName() { + return null; + } + + @Override + public int getProbeColIdx() { + return -1; + } + }; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 45b6c0e2b919..6c69f0afa69c 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.exec.tez.HashableInputSplit; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.io.NullRowsInputFormat.NullRowsRecordReader; +import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler; @@ -296,7 +297,8 @@ public static InputFormat wrapForLlap( return inputFormat; // LLAP not enabled, no-op. } String ifName = inputFormat.getClass().getCanonicalName(); - boolean isSupported = inputFormat instanceof LlapWrappableInputFormatInterface; + boolean isSupported = inputFormat instanceof LlapWrappableInputFormatInterface + || usesNativeParquetLlapIo(inputFormat.getClass(), conf); boolean isCacheOnly = inputFormat instanceof LlapCacheOnlyInputFormatInterface; boolean isVectorized = Utilities.getIsVectorized(conf); if (!isVectorized) { @@ -353,6 +355,12 @@ public static InputFormat wrapForLlap( return inputFormat; } + /** Parquet reads through LLAP IO natively when the flag is on; otherwise it is cache-only. */ + public static boolean usesNativeParquetLlapIo(Class inputFormatClass, Configuration conf) { + return MapredParquetInputFormat.class.isAssignableFrom(inputFormatClass) + && HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_PARQUET_NATIVE_ENABLED); + } + public static boolean checkInputFormatForLlapEncode(Configuration conf, String ifName) { String formatList = HiveConf.getVar(conf, ConfVars.LLAP_IO_ENCODE_FORMATS); LOG.debug("Checking {} against {}", ifName, formatList); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java index 50c30e2941a3..6cb13da2b8bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java @@ -187,6 +187,21 @@ protected ParquetMetadata getParquetMetadata(Path path, JobConf conf) throws IOE } public FilterCompat.Filter setFilter(final JobConf conf, MessageType schema) { + FilterPredicate p = toFilterPredicate(conf, schema); + if (p != null) { + // Filter may have sensitive information. Do not send to debug. + LOG.debug("PARQUET predicate push down generated."); + ParquetInputFormat.setFilterPredicate(conf, p); + return FilterCompat.get(p); + } else { + // Filter may have sensitive information. Do not send to debug. + LOG.debug("No PARQUET predicate push down is generated."); + return null; + } + } + + /** The pushed-down SARG as a Parquet predicate over the file's columns, or null if there is none. */ + public static FilterPredicate toFilterPredicate(final JobConf conf, MessageType schema) { SearchArgument sarg = ConvertAstToSearchArg.createFromConf(conf); if (sarg == null) { return null; @@ -204,20 +219,10 @@ public FilterCompat.Filter setFilter(final JobConf conf, MessageType schema) { // Create the Parquet FilterPredicate without including columns that do not exist // on the schema (such as partition columns). MessageType newSchema = getSchemaWithoutPartitionColumns(conf, schema); - FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, newSchema, columns); - if (p != null) { - // Filter may have sensitive information. Do not send to debug. - LOG.debug("PARQUET predicate push down generated."); - ParquetInputFormat.setFilterPredicate(conf, p); - return FilterCompat.get(p); - } else { - // Filter may have sensitive information. Do not send to debug. - LOG.debug("No PARQUET predicate push down is generated."); - return null; - } + return ParquetFilterPredicateConverter.toFilterPredicate(sarg, newSchema, columns); } - private MessageType getSchemaWithoutPartitionColumns(JobConf conf, MessageType schema) { + private static MessageType getSchemaWithoutPartitionColumns(JobConf conf, MessageType schema) { List partCols = Utilities.getPartitionColumnNames(conf); if (partCols.isEmpty()) { return schema; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index 515b2c7fa7a2..b02cc0d1cfdd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -301,7 +301,8 @@ public void deriveLlap(Configuration conf, boolean isExecDriver) { for (PartitionDesc part : pathToPartitionInfo.values()) { Class inputFormatClass = part.getInputFileFormatClass(); boolean isUsingLlapIo = canWrapAny && (HiveInputFormat.canWrapForLlap(inputFormatClass, doCheckIfs) - || HiveInputFormat.checkInputFormatForLlapEncode(conf, inputFormatClass.getCanonicalName())); + || HiveInputFormat.checkInputFormatForLlapEncode(conf, inputFormatClass.getCanonicalName()) + || HiveInputFormat.usesNativeParquetLlapIo(inputFormatClass, conf)); if (isUsingLlapIo) { if (part.getTableDesc() != null && AcidUtils.isTablePropertyTransactional(part.getTableDesc().getProperties())) { From 6d2e1e994dc64960dcc295804ff5af8a5215e6fc Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 15 Sep 2026 15:09:22 +0200 Subject: [PATCH 04/17] HIVE-30059: Count Parquet footer cache hits and misses in the LLAP IO summary (META_HIT, META_MISS) The summary's META_HIT / META_MISS columns are wired to METADATA_CACHE_HIT and METADATA_CACHE_MISS, which OrcEncodedDataReader bumps around its file tail lookups. The Parquet path uses the same FileMetadataCache (LlapIoImpl. getParquetFooterBuffersFromCache calls getFileMetadata / putFileMetadata on it) but never bumped the counters, so a Parquet-only run showed zeroes in those columns. Bump them from the same site. LlapIoImpl cannot see QueryFragmentCounters (it lives in llap-server; LlapIo in llap-client), so the impl reports the outcome through a nullable BooleanRef out-parameter -- the same signalling pattern storage-api already uses -- and ParquetEncodedDataReader.loadFooter increments the counter it already holds. The two non-native callers (VectorizedParquetRecordReader on the vectorized fallback, HiveVectorizedReader on the Iceberg path) do not have per-fragment counters plumbed and pass null; their footer traffic continues to go unaccounted, as it did before. TestParquetEncodedDataReader reads the file twice and asserts the second read reports one hit and zero misses; only the second read is asserted because the footer cache is a @BeforeClass singleton and any earlier test in the class may have populated it. Co-Authored-By: Claude Code --- .../mr/hive/vector/HiveVectorizedReader.java | 4 +++- .../org/apache/hadoop/hive/llap/io/api/LlapIo.java | 9 +++++++-- .../hadoop/hive/llap/io/api/impl/LlapIoImpl.java | 8 ++++++-- .../llap/io/encoded/ParquetEncodedDataReader.java | 7 ++++++- .../io/encoded/TestParquetEncodedDataReader.java | 13 +++++++++++++ .../vector/VectorizedParquetRecordReader.java | 5 ++++- 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java index 0bbc5aa8f08e..40ded314cc41 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java @@ -261,7 +261,9 @@ private static RecordReader parquetRecordReade if (HiveConf.getBoolVar(job, HiveConf.ConfVars.LLAP_IO_ENABLED, LlapProxy.isDaemon()) && LlapProxy.getIo() != null && LlapProxy.getIo().usingLowLevelCache()) { LlapProxy.getIo().initCacheOnlyInputFormat(inputFormat); - footerData = LlapProxy.getIo().getParquetFooterBuffersFromCache(path, job, fileId); + // No per-fragment counters on the Iceberg vectorized path yet; footer lookups are + // recorded as cache traffic but not as META hits/misses in the LLAP IO summary. + footerData = LlapProxy.getIo().getParquetFooterBuffersFromCache(path, job, fileId, null); } ParquetMetadata parquetMetadata = HiveParquetUtil.readFooter(task.file(), io, job, footerData); diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java index b037fc015c80..d96b61584a6d 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.io.CacheTag; +import org.apache.hadoop.hive.common.io.DataCache.BooleanRef; import org.apache.hadoop.hive.common.io.encoded.MemoryBufferOrBuffers; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -75,11 +76,15 @@ InputFormat getInputFormat( * @param fileKey fileId of the Parquet file (either the Long fileId of HDFS or the SyntheticFileId). * Optional, if it is not provided, it will be generated, see: * org.apache.hadoop.hive.ql.io.HdfsUtils#getFileId() + * @param cacheHit optional out-parameter; when supplied, its {@code value} field is written + * with {@code true} on a cache hit and {@code false} on a miss, so the caller + * (which lives in llap-server and holds {@code QueryFragmentCounters}) can bump + * {@code METADATA_CACHE_HIT} / {@code METADATA_CACHE_MISS} for the LLAP IO summary. * @return * @throws IOException */ - MemoryBufferOrBuffers getParquetFooterBuffersFromCache(Path path, JobConf conf, @Nullable Object fileKey) - throws IOException; + MemoryBufferOrBuffers getParquetFooterBuffersFromCache(Path path, JobConf conf, @Nullable Object fileKey, + @Nullable BooleanRef cacheHit) throws IOException; /** * Handles request to evict entities specified in the request object. diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index becfdce5a5f3..adae60713f39 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -50,6 +50,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.io.Allocator; import org.apache.hadoop.hive.common.io.DataCache; +import org.apache.hadoop.hive.common.io.DataCache.BooleanRef; import org.apache.hadoop.hive.common.io.DiskRange; import org.apache.hadoop.hive.common.io.DiskRangeList; import org.apache.hadoop.hive.common.io.FileMetadataCache; @@ -514,8 +515,8 @@ public RecordReader llapVectorizedParquetReade } @Override - public MemoryBufferOrBuffers getParquetFooterBuffersFromCache(Path path, JobConf conf, @Nullable Object fileKey) - throws IOException { + public MemoryBufferOrBuffers getParquetFooterBuffersFromCache(Path path, JobConf conf, @Nullable Object fileKey, + @Nullable BooleanRef cacheHit) throws IOException { Preconditions.checkNotNull(fileMetadataCache, "Metadata cache must not be null"); @@ -524,6 +525,9 @@ public MemoryBufferOrBuffers getParquetFooterBuffersFromCache(Path path, JobConf MemoryBufferOrBuffers footerData = (fileKey == null ) ? null : fileMetadataCache.getFileMetadata(fileKey); + if (cacheHit != null) { + cacheHit.value = footerData != null; + } if (footerData != null) { LOG.info("Found the footer in cache for " + fileKey); try { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java index 95876fd1a626..c38ce7fb3294 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java @@ -147,8 +147,13 @@ public ParquetMetadata loadFooter() throws IOException { } if (fileKey != null) { cacheTag = VectorizedParquetRecordReader.cacheTagOfParquetFile(path, daemonConf, jobConf); + // Bumps METADATA_CACHE_HIT / METADATA_CACHE_MISS so the LLAP IO summary accounts for the + // Parquet footer lookup the same way it does for ORC's file tail. + BooleanRef cacheHit = new BooleanRef(); MemoryBufferOrBuffers footerData = - LlapProxy.getIo().getParquetFooterBuffersFromCache(path, jobConf, fileKey); + LlapProxy.getIo().getParquetFooterBuffersFromCache(path, jobConf, fileKey, cacheHit); + counters.incrCounter(cacheHit.value + ? LlapIOCounters.METADATA_CACHE_HIT : LlapIOCounters.METADATA_CACHE_MISS); footer = ParquetFileReader.readFooter( new ParquetFooterInputFromCache(footerData), ParquetMetadataConverter.NO_FILTER); } else { diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java index a4f63b4e6959..704892df9645 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -362,6 +362,19 @@ public void testCachedBuffersAreNotReread() throws Exception { } } + @Test + public void testFooterLookupBumpsMetadataCacheCounters() throws Exception { + // The footer cache is a @BeforeClass singleton, so any earlier test may have populated it; + // only the second read of this test is order-independent. That call finds the footer we just + // put in on the first read, so META_HIT is 1 and META_MISS is 0 whatever came before. + read(jobConf(COLUMNS, TYPES, 0), wholeFile()).assertClean(); + + Run second = read(jobConf(COLUMNS, TYPES, 0), wholeFile()); + second.assertClean(); + assertEquals(1, second.counter(LlapIOCounters.METADATA_CACHE_HIT)); + assertEquals(0, second.counter(LlapIOCounters.METADATA_CACHE_MISS)); + } + @Test public void testNextRowGroupIsRequestedBeforeCurrentDecodes() throws Exception { Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index 8521228cd589..6647b22e8a68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -310,7 +310,10 @@ private ParquetMetadata readSplitFooter(JobConf configuration, final Path file, FileStatus stat = fs.getFileStatus(file); return readFooterFromFile(file, fs, stat, filter); } else { - MemoryBufferOrBuffers footerData = LlapProxy.getIo().getParquetFooterBuffersFromCache(file, configuration, cacheKey); + // Fallback path outside the native encoded reader: no per-fragment counters here, so the + // footer lookup doesn't contribute to META_HIT / META_MISS in the LLAP IO summary. + MemoryBufferOrBuffers footerData = + LlapProxy.getIo().getParquetFooterBuffersFromCache(file, configuration, cacheKey, null); return ParquetFileReader.readFooter(new ParquetFooterInputFromCache(footerData), filter); } } From 471599fc9e74708a868740f434a39bd39599222d Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 16 Sep 2026 15:34:45 +0200 Subject: [PATCH 05/17] unused import --- .../hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index 6647b22e8a68..cbdbfb232e9c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -78,7 +78,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; From e367a845fe5fb8901bde5c956081998efe42fcbe Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 16 Sep 2026 15:39:47 +0200 Subject: [PATCH 06/17] clarifying comment on LlapCacheAwareFs --- .../hadoop/hive/llap/LlapCacheAwareFs.java | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java index 3c25491902ef..cdf2c739edf9 100644 --- a/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java +++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapCacheAwareFs.java @@ -52,9 +52,51 @@ import org.apache.orc.impl.RecordReaderUtils; /** - * This is currently only used by Parquet; however, a generally applicable approach is used - - * you pass in a set of offset pairs for a file, and the file is cached with these boundaries. - * Don't add anything format specific here. + * A shim {@link FileSystem} that transparently interposes on reads so that byte ranges backed by + * the LLAP {@link DataCache} are served from memory and only genuine misses go to the underlying + * (real) file system. + * + *

What it does

+ * It exposes a virtual {@code llapcache://} URI in place of the real file path. Reads issued + * against that URI are routed through {@link CacheAwareInputStream}, which consults the + * {@link DataCache} for each requested chunk: hits are copied straight out of cache buffers, + * misses are fetched from the wrapped {@link FileSystem}, handed back to the caller, and + * simultaneously inserted into the cache so subsequent reads of the same chunk are served from + * memory. All non-read {@code FileSystem} operations (open/append/create/delete/rename/ + * getFileStatus/listStatus/mkdirs) are simply delegated to the wrapped file system after + * translating the virtual path back to the real one. + * + *

How it works

+ * The reader that wants caching calls {@link #registerFile(DataCache, Path, Object, TreeMap, + * Configuration, CacheTag)} with the real path, a stable {@code fileKey}, and a chunk index + * (a {@code TreeMap} of {@code startOffset -> endOffset} boundaries covering the ranges the + * reader intends to read - typically the column-chunk ranges of the projected columns). The + * shim assigns a unique {@code splitId}, stashes the per-split state in a static map, registers + * itself as the handler for the {@code llapcache} scheme via {@code fs.llapcache.impl}, and + * returns a virtual {@code llapcache://llapcache/<splitId>} path. The caller hands this + * virtual path to the underlying format library (e.g. the Parquet reader); when the library + * opens the path and issues positioned reads, this class receives them, splits them along the + * pre-registered chunk boundaries, and either serves each chunk from cache or reads-through and + * populates it. When the reader is finished it must call {@link #unregisterFile(Path)} to drop + * the per-split state. + * + *

When it is used

+ * Today the only caller is the vectorized Parquet path in + * {@code VectorizedParquetRecordReader.wrapPathForCache}, which engages the shim when LLAP IO + * is enabled and a {@link DataCache} was injected into the reader via + * {@code LlapCacheOnlyInputFormatInterface.injectCaches}. Concretely that means: + *
    + *
  • LLAP daemon splits where the native Parquet cache pipeline + * ({@code ParquetColumnVectorProducer} / {@code ParquetEncodedDataReader}) declines the + * split and {@code LlapInputFormat} falls back to the source + * {@code VectorizedParquetInputFormat} - e.g. nested-type projections or setup errors. + *
  • Iceberg vectorized Parquet reads via {@code HiveVectorizedReader.parquetRecordReader}, + * which always uses {@code VectorizedParquetInputFormat} directly and injects caches when + * LLAP is on - the native Parquet cache pipeline is not reachable from that entry point. + *
+ * Non-LLAP execution (no daemon, no {@code DataCache} injected) skips this shim entirely and + * reads directly from the real file system. The class is deliberately format-agnostic - the + * boundaries are supplied by the caller, so nothing Parquet-specific should be added here. */ public class LlapCacheAwareFs extends FileSystem { public static final String SCHEME = "llapcache"; From dafb7542f49c16419d1ba5589bb6c75c818edf72 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 16 Sep 2026 15:48:37 +0200 Subject: [PATCH 07/17] cacheMetrics.incrCacheReadRequests for parity with OrcColumnVectorProducer --- .../hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java index 7c0b45ddccf8..a303290ac6ce 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java @@ -61,6 +61,7 @@ public ReadPipeline createReadPipeline(Consumer consumer, Fil SchemaEvolutionFactory sef, InputFormat sourceInputFormat, Deserializer sourceSerDe, Reporter reporter, JobConf job, Map parts) throws IOException { try { + cacheMetrics.incrCacheReadRequests(); ParquetEncodedDataConsumer edc = new ParquetEncodedDataConsumer(consumer, includes, counters, ioMetrics, job); ParquetEncodedDataReader reader = new ParquetEncodedDataReader( From 185fe73cbdfe8c0e9a36aecda1df3fdb36a1ad8c Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 16 Sep 2026 15:49:00 +0200 Subject: [PATCH 08/17] createReadPipeline doesn't throw exception --- .../hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java index a303290ac6ce..9e159b166c7f 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java @@ -59,7 +59,7 @@ public ParquetColumnVectorProducer(LowLevelCache lowLevelCache, BufferUsageManag public ReadPipeline createReadPipeline(Consumer consumer, FileSplit split, Includes includes, SearchArgument sarg, QueryFragmentCounters counters, SchemaEvolutionFactory sef, InputFormat sourceInputFormat, Deserializer sourceSerDe, - Reporter reporter, JobConf job, Map parts) throws IOException { + Reporter reporter, JobConf job, Map parts) { try { cacheMetrics.incrCacheReadRequests(); ParquetEncodedDataConsumer edc = From a1b4ecada9af966c9ef3d74af0bb5052edf85b52 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 17 Sep 2026 08:32:37 +0200 Subject: [PATCH 09/17] final metrics fields in OrcColumnVectorProducer --- .../hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java index 28b2958876f2..1db5d6377ba5 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java @@ -59,8 +59,8 @@ public class OrcColumnVectorProducer implements ColumnVectorProducer { private final PathCache pathCache; private final BufferUsageManager bufferManager; private final Configuration conf; - private LlapDaemonCacheMetrics cacheMetrics; - private LlapDaemonIOMetrics ioMetrics; + private final LlapDaemonCacheMetrics cacheMetrics; + private final LlapDaemonIOMetrics ioMetrics; // TODO: if using in multiple places, e.g. SerDe cache, pass this in. // TODO: should this rather use a threadlocal for NUMA affinity? private final FixedSizedObjectPool tracePool; From 6b1f41c7031047c2cc1e4e035a31c013d81ecf45 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 17 Sep 2026 08:45:29 +0200 Subject: [PATCH 10/17] QueryFragmentCounters parity with ORC --- .../io/encoded/ParquetEncodedDataReader.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java index c38ce7fb3294..ef66853bbf7a 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java @@ -157,8 +157,12 @@ public ParquetMetadata loadFooter() throws IOException { footer = ParquetFileReader.readFooter( new ParquetFooterInputFromCache(footerData), ParquetMetadataConverter.NO_FILTER); } else { + // Fallback path: no cache key, so we read status + footer straight from HDFS. Time both + // under HDFS_TIME_NS to match how ORC accounts for cache-miss footer reads. final FileSystem fs = path.getFileSystem(jobConf); + long hdfsStart = counters.startTimeCounter(); final FileStatus stat = fs.getFileStatus(path); + counters.recordHdfsTime(hdfsStart); InputFile inputFile = new InputFile() { @Override public SeekableInputStream newStream() throws IOException { @@ -169,7 +173,9 @@ public long getLength() { return stat.getLen(); } }; + hdfsStart = counters.startTimeCounter(); footer = ParquetFileReader.readFooter(inputFile, ParquetMetadataConverter.NO_FILTER); + counters.recordHdfsTime(hdfsStart); } requestedSchema = DataWritableReadSupport.getRequestedSchema( jobConf.getBoolean(DataWritableReadSupport.PARQUET_COLUMN_INDEX_ACCESS, false), @@ -182,17 +188,28 @@ public long getLength() { @Override protected Void callInternal() throws IOException, InterruptedException { return ugi.doAs((PrivilegedExceptionAction) () -> { + long startTime = counters.startTimeCounter(); try { performDataRead(); consumer.setDone(); } catch (Throwable t) { consumer.setError(t); + } finally { + counters.incrWallClockCounter(LlapIOCounters.TOTAL_IO_TIME_NS, startTime); } return null; }); } private void performDataRead() throws IOException, InterruptedException { + // The LLAP IO summary keys off TABLE / FILE / STRIPES; set them the same way ORC does so a + // native Parquet fragment shows up in the summary with the same fields populated. + if (cacheTag != null) { + counters.setDesc(QueryFragmentCounters.Desc.TABLE, cacheTag.getTableName()); + } + counters.setDesc(QueryFragmentCounters.Desc.FILE, path + + (fileKey == null ? "" : " (" + fileKey + ")")); + MessageType fileSchema = footer.getFileMetaData().getSchema(); int[] projected = projectedLeaves(requestedSchema, fileSchema); consumer.setFileMetadata(footer, requestedSchema, path); @@ -216,6 +233,9 @@ private void performDataRead() throws IOException, InterruptedException { for (int i = 0; i < blocks.size(); ++i) { rowGroupOf.put(blocks.get(i), i); } + // STRIPES is the ORC term; for Parquet the analog is row groups. Reuse the same descriptor so + // the summary layout stays common and we don't have to teach the reporter about a new field. + counters.setDesc(QueryFragmentCounters.Desc.STRIPES, "0," + selected.size()); counters.incrCounter(LlapIOCounters.SELECTED_ROWGROUPS, selected.size()); FileSystem fs = path.getFileSystem(jobConf); @@ -336,7 +356,10 @@ private Fetch startFetch(FSDataInputStream fileStream, ParquetRangeBuffers buffe planColumnChunk(allocator, maxAlloc, chunk.getStartingPos(), chunk.getStartingPos() + chunk.getTotalSize(), column, fetch.misses); } + // The vectored dispatch is where non-async FS impls actually read; count it under HDFS_TIME. + long hdfsStart = counters.startTimeCounter(); requestMisses(fileStream, buffers, fetch, layout.maxRangeBytes()); + counters.recordHdfsTime(hdfsStart); } catch (Throwable t) { abandon(allocator, fetch); throw t; @@ -347,7 +370,10 @@ private Fetch startFetch(FSDataInputStream fileStream, ParquetRangeBuffers buffe private void finishFetch(Allocator allocator, ParquetRangeBuffers buffers, Fetch fetch) throws IOException, InterruptedException { try { + // Awaiting the vectored futures is where the miss bytes actually arrive from HDFS. + long hdfsStart = counters.startTimeCounter(); receiveMisses(buffers, fetch); + counters.recordHdfsTime(hdfsStart); for (ColumnPlan column : fetch.columns) { putColumn(allocator, column); } From f959112b7775668cf62620e457cf9e096af10a2e Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 17 Sep 2026 08:50:06 +0200 Subject: [PATCH 11/17] Simple test for QueryFragmentCounters parity with ORC Verifies the descriptors and time counters wired into ParquetEncodedDataReader in the previous commit: SELECTED_ROWGROUPS, TOTAL_IO_TIME_NS, HDFS_TIME_NS, METADATA_CACHE_HIT/MISS, plus FILE and STRIPES descriptors visible via the QueryFragmentCounters summary string. Co-Authored-By: Claude Code --- .../encoded/TestParquetEncodedDataReader.java | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java index 704892df9645..3ba0afbdd702 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -375,6 +375,31 @@ public void testFooterLookupBumpsMetadataCacheCounters() throws Exception { assertEquals(0, second.counter(LlapIOCounters.METADATA_CACHE_MISS)); } + @Test + public void testFragmentCountersParityWithOrc() throws Exception { + // The LLAP IO summary is emitted from QueryFragmentCounters#toString(). ORC populates FILE, + // STRIPES, TOTAL_IO_TIME_NS and HDFS_TIME_NS for every fragment; the native Parquet reader + // should populate the same fields so a Parquet fragment shows up in the summary the same way. + // (TABLE is only set when LLAP_TRACK_CACHE_USAGE is on, which this test's fixture disables.) + Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); + + run.assertClean(); + assertEquals(ROW_GROUPS, run.counter(LlapIOCounters.SELECTED_ROWGROUPS)); + assertTrue("TOTAL_IO_TIME_NS not recorded", + run.counter(LlapIOCounters.TOTAL_IO_TIME_NS) > 0); + assertTrue("HDFS_TIME_NS not recorded", + run.counter(LlapIOCounters.HDFS_TIME_NS) > 0); + assertEquals("exactly one of metadata hit/miss should be bumped per fragment", 1, + run.counter(LlapIOCounters.METADATA_CACHE_HIT) + + run.counter(LlapIOCounters.METADATA_CACHE_MISS)); + + String summary = run.fragmentCounters.toString(); + assertTrue("FILE descriptor missing from summary: " + summary, + summary.contains(file.toString())); + assertTrue("STRIPES descriptor missing from summary: " + summary, + summary.contains("0," + ROW_GROUPS)); + } + @Test public void testNextRowGroupIsRequestedBeforeCurrentDecodes() throws Exception { Run run = read(jobConf(COLUMNS, TYPES, 0, 1, 2, 3, 4, 5), wholeFile()); @@ -721,7 +746,7 @@ FSDataInputStream openFile(FileSystem fs) throws IOException { edc.init(reader, reader); reader.loadFooter(); reader.call(); - return new Run(downstream, tezCounters, buffers, reads, ledger); + return new Run(downstream, counters, tezCounters, buffers, reads, ledger); } @@ -930,18 +955,20 @@ private static final class Run { final ColumnVector[] firstBatchCols; final boolean done; final Throwable error; + final QueryFragmentCounters fragmentCounters; final TezCounters counters; final List buffers; final List reads; final Ledger ledger; - Run(CapturingConsumer c, TezCounters counters, List buffers, List reads, - Ledger ledger) { + Run(CapturingConsumer c, QueryFragmentCounters fragmentCounters, TezCounters counters, + List buffers, List reads, Ledger ledger) { this.rows = c.rows; this.batchSizes = c.batchSizes; this.firstBatchCols = c.firstBatchCols; this.done = c.done; this.error = c.error; + this.fragmentCounters = fragmentCounters; this.counters = counters; this.buffers = buffers; this.reads = reads; From a9f994fd1b1433a212275c6152cefe7e89dfc7d4 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 17 Sep 2026 09:04:08 +0200 Subject: [PATCH 12/17] Address Denys's PR review comments - HiveIcebergStorageHandler: import ParquetRowGroupDecoder so the javadoc {@link} can use the short name. - LlapIoImpl: rename local bufferManagerOrc -> bufferManagerData; the buffer manager is now shared with the native Parquet producer, so the Orc suffix is stale. - LlapRecordReader: rename checkOrcSchemaEvolution -> checkSchemaEvolution since the method already tolerates producers (like native Parquet) that do not expose an ORC-style SchemaEvolution. - ParquetCachedPageReadStore.chunkBuffers: split the two-per-line locals, and rename from/to -> sliceStart/sliceEnd (with chunkStart/chunkEnd and bufferStart/bufferEnd for the surrounding bounds). Co-Authored-By: Claude Code --- .../iceberg/mr/hive/HiveIcebergStorageHandler.java | 3 ++- .../hadoop/hive/llap/io/api/impl/LlapIoImpl.java | 10 +++++----- .../hive/llap/io/api/impl/LlapRecordReader.java | 9 +++++---- .../llap/io/decode/ParquetCachedPageReadStore.java | 13 ++++++++----- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 0a540f746dab..729fb0e268e6 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -84,6 +84,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.StorageFormatDescriptor; +import org.apache.hadoop.hive.ql.io.parquet.vector.ParquetRowGroupDecoder; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; @@ -1889,7 +1890,7 @@ private static boolean hasOrcTimeInSchema(Properties tableProps, Schema tableSch /** * Vectorized reads of parquet files from columns with list or map type is only supported if the nested types are of * primitive type category - * check {@link org.apache.hadoop.hive.ql.io.parquet.vector.ParquetRowGroupDecoder#checkListColumnSupport} for + * check {@link ParquetRowGroupDecoder#checkListColumnSupport} for * details on nested types under lists * @param tableProps iceberg table properties * @param tableSchema iceberg table schema diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index adae60713f39..d95b43d7e2fb 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -178,7 +178,7 @@ private LlapIoImpl(Configuration conf) throws IOException { MetadataCache metadataCache = null; SerDeLowLevelCacheImpl serdeCache = null; // TODO: extract interface when needed - BufferUsageManager bufferManagerOrc = null, bufferManagerGeneric = null; + BufferUsageManager bufferManagerData = null, bufferManagerGeneric = null; boolean isEncodeEnabled = useLowLevelCache && HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ENCODE_ENABLED); if (useLowLevelCache) { @@ -226,7 +226,7 @@ private LlapIoImpl(Configuration conf) throws IOException { cachePolicyWrapper.setEvictionListener(e); cacheImpl.startThreads(); // Start the cache threads. - bufferManager = bufferManagerOrc = cacheImpl; // Cache also serves as buffer manager. + bufferManager = bufferManagerData = cacheImpl; // Cache also serves as buffer manager. bufferManagerGeneric = serdeCache; if (trackUsage) { debugDumpComponents.add(cachePolicyWrapper); // Cache contents tracker. @@ -245,7 +245,7 @@ private LlapIoImpl(Configuration conf) throws IOException { this.allocator = new SimpleAllocator(conf); fileMetadataCache = null; SimpleBufferManager sbm = new SimpleBufferManager(allocator, cacheMetrics); - bufferManager = bufferManagerOrc = bufferManagerGeneric = sbm; + bufferManager = bufferManagerData = bufferManagerGeneric = sbm; dataCache = sbm; this.memoryManager = null; debugDumpComponents.add(new LlapIoDebugDump() { @@ -276,12 +276,12 @@ public void debugDumpShort(StringBuilder sb) { // TODO: this should depends on input format and be in a map, or something. this.orcCvp = new OrcColumnVectorProducer( - metadataCache, dataCache, pathCache, bufferManagerOrc, conf, cacheMetrics, ioMetrics, tracePool); + metadataCache, dataCache, pathCache, bufferManagerData, conf, cacheMetrics, ioMetrics, tracePool); this.genericCvp = isEncodeEnabled ? new GenericColumnVectorProducer( serdeCache, bufferManagerGeneric, conf, cacheMetrics, ioMetrics, tracePool, encodeExecutor) : null; // Native Parquet IO is gated per query by the job conf at the dispatch sites. this.parquetCvp = dataCache != null - ? new ParquetColumnVectorProducer(dataCache, bufferManagerOrc, conf, cacheMetrics, ioMetrics) + ? new ParquetColumnVectorProducer(dataCache, bufferManagerData, conf, cacheMetrics, ioMetrics) : null; LOG.info("LLAP IO initialized"); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index cbf9a7fad1c1..01fa9cd4705e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -138,7 +138,7 @@ public static LlapRecordReader create(JobConf job, FileSplit split, if (rr.rp == null) { return null; // The producer declined the split; the caller uses the source reader. } - if (!rr.checkOrcSchemaEvolution()) { + if (!rr.checkSchemaEvolution()) { rr.close(); throwIfCacheOnlyRead(HiveConf.getBoolVar(job, ConfVars.LLAP_IO_CACHE_ONLY)); return null; @@ -351,11 +351,12 @@ public void start() { executor.submit(rp.getReadCallable()); } - private boolean checkOrcSchemaEvolution() { + private boolean checkSchemaEvolution() { SchemaEvolution evolution = rp.getSchemaEvolution(); if (evolution == null) { - // No ORC-style schema evolution to validate (e.g. native parquet path); - // parquet handles its own column resolution. Nothing to check here. + // Only the ORC pipeline hangs an ORC SchemaEvolution off the ReadPipeline; other formats + // (native Parquet today) handle column resolution themselves, so there is nothing to + // validate here. return true; } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java index 09036b065b41..a35c770d09cb 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java @@ -75,14 +75,17 @@ class ParquetCachedPageReadStore implements PageReadStore { /** Slices of the cached buffers covering exactly the chunk's byte region, in file order. */ private static List chunkBuffers(ParquetEncodedColumnBatch batch, int pc) { - long start = batch.chunks[pc].getStartingPos(), end = start + batch.chunks[pc].getTotalSize(); + long chunkStart = batch.chunks[pc].getStartingPos(); + long chunkEnd = chunkStart + batch.chunks[pc].getTotalSize(); List slices = new ArrayList<>(batch.columnBuffers[pc].length); for (int i = 0; i < batch.columnBuffers[pc].length; ++i) { - long offset = batch.bufferOffsets[pc][i]; - long from = Math.max(start, offset), to = Math.min(end, offset + batch.bufferLengths[pc][i]); + long bufferStart = batch.bufferOffsets[pc][i]; + long bufferEnd = bufferStart + batch.bufferLengths[pc][i]; + long sliceStart = Math.max(chunkStart, bufferStart); + long sliceEnd = Math.min(chunkEnd, bufferEnd); ByteBuffer bb = batch.columnBuffers[pc][i].getByteBufferDup(); - bb.position(bb.position() + (int) (from - offset)); - bb.limit(bb.position() + (int) (to - from)); + bb.position(bb.position() + (int) (sliceStart - bufferStart)); + bb.limit(bb.position() + (int) (sliceEnd - sliceStart)); slices.add(bb.slice()); } return slices; From 59ae554460436bdc87507fd2ea9d1d373b69ae45 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 17 Sep 2026 09:11:30 +0200 Subject: [PATCH 13/17] HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64 --- .../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 9 ++++++++- .../hadoop/hive/llap/io/api/impl/LlapRecordReader.java | 3 ++- .../hive/llap/io/decode/ParquetEncodedDataConsumer.java | 3 ++- .../hive/llap/io/encoded/OrcEncodedDataReader.java | 3 ++- .../hive/llap/io/encoded/SerDeEncodedDataReader.java | 3 ++- .../hive/llap/io/encoded/VectorDeserializeOrcWriter.java | 3 ++- .../llap/io/encoded/TestParquetEncodedDataReader.java | 3 ++- .../apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 3 ++- .../hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java | 6 ++++-- .../org/apache/hadoop/hive/ql/io/orc/WriterImpl.java | 2 +- .../hadoop/hive/ql/io/orc/TestInputOutputFormat.java | 6 ++++-- 11 files changed, 31 insertions(+), 13 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 621f96fc9765..3f28c636e1e3 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -261,6 +261,13 @@ private static URL checkConfigFile(File f) { public static final String HIVE_SERVER2_AUTHENTICATION_LDAP_USERMEMBERSHIPKEY_NAME = "hive.server2.authentication.ldap.userMembershipKey"; + /** + * Token that {@link ConfVars#HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED} must contain for + * decimal_64 vectorization to be enabled. Extracted here so call sites can reference the same + * constant instead of repeating the literal. + */ + public static final String HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64 = "decimal_64"; + /** * dbVars are the parameters can be set per database. If these * parameters are set as a database property, when switching to that @@ -4641,7 +4648,7 @@ public static enum ConfVars { "evaluate call and turn them into NULLs. Assume, by default, this is not needed"), HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED( "hive.vectorized.input.format.supports.enabled", - "decimal_64", + HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64, "Which vectorized input format support features are enabled for vectorization.\n" + "That is, if a VectorizedInputFormat input format does support \"decimal_64\" for example\n" + "this variable must enable that to be used in vectorization"), diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index 01fa9cd4705e..af837e1c553f 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -188,7 +188,8 @@ private LlapRecordReader(MapWork mapWork, JobConf job, FileSplit split, final boolean decimal64Support = - HiveConf.getVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + HiveConf.getVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); int limit = determineQueueLimit(bestEffortSize, diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java index 1ab26aed50d3..d25f665c8e76 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java @@ -94,7 +94,8 @@ public ParquetEncodedDataConsumer(Consumer consumer, Includes super(consumer, includes.getPhysicalColumnIds().size(), ioMetrics, counters); this.jobConf = jobConf; this.useDecimal64ColumnVectors = HiveConf.getVar(jobConf, - ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); ParquetReadOptions options = HadoopReadOptions.builder(jobConf).build(); this.codecFactory = options.getCodecFactory(); this.converter = new ParquetMetadataConverter(options); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 24306d5db1fa..8e8a917f20b9 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -261,7 +261,8 @@ public OrcEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager buff } consumer.setUseDecimal64ColumnVectors(HiveConf.getVar(jobConf, - ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64")); + ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64)); consumer.setFileMetadata(fileMetadata); consumer.setSchemaEvolution(evolution); isReadCacheOnly = HiveConf.getBoolVar(jobConf, ConfVars.LLAP_IO_CACHE_ONLY); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java index e3dc6806b423..0b454d002584 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java @@ -232,7 +232,8 @@ public MemoryBuffer create() { this.reporter = reporter; this.jobConf = jobConf; final boolean useDecimal64ColumnVectors = HiveConf.getVar(jobConf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); consumer.setUseDecimal64ColumnVectors(useDecimal64ColumnVectors); this.schema = schema; this.writerIncludes = OrcInputFormat.genIncludedColumns(schema, columnIds); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java index 46e8a12bdb51..4181118daca0 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java @@ -235,7 +235,8 @@ public void startAsync(AsyncCallback callback) { private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, final Properties tblProps, final Configuration conf) throws IOException { final boolean useDecimal64ColumnVectors = HiveConf.getVar(conf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); final String serde = tblProps.getProperty(serdeConstants.SERIALIZATION_LIB); final String inputFormat = tblProps.getProperty(hive_metastoreConstants.FILE_INPUT_FORMAT); final boolean isTextFormat = TextInputFormat.class.getName().equals(inputFormat) diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java index 3ba0afbdd702..8a8a4ac1bf67 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -689,7 +689,8 @@ private static JobConf jobConf(String columns, String types, int... readColumnId ids.add(id); } ColumnProjectionUtils.setReadColumns(job, ids); - HiveConf.setVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED, "decimal_64"); + HiveConf.setVar(job, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED, + HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); return job; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 190bec882185..25d00b315c16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -68,7 +68,8 @@ protected RecordReaderImpl(ReaderImpl fileReader, Reader.Options options, final Configuration conf) throws IOException { super(fileReader, options); final boolean useDecimal64ColumnVectors = conf != null && HiveConf.getVar(conf, - HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors){ batch = this.schema.createRowBatchV2(); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index c4a92f624e2c..20d2c07e3df4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -212,7 +212,8 @@ public float getProgress() throws IOException { } }; final boolean useDecimal64ColumnVectors = HiveConf - .getVar(conf, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .getVar(conf, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors) { this.vectorizedRowBatchBase = ((RecordReaderImpl) innerReader).createRowBatch(true); } else { @@ -1514,7 +1515,8 @@ static class DeleteReaderValue { this.bucketForSplit = bucket; final boolean useDecimal64ColumnVector = HiveConf.getVar(conf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED).equalsIgnoreCase("decimal_64"); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVector) { this.batch = acidEmptyStructSchema.createRowBatchV2(); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java index 24083b3ab84e..8b9865513d48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java @@ -97,7 +97,7 @@ public class WriterImpl extends org.apache.orc.impl.WriterImpl implements Writer this.inspector = opts.getInspector(); boolean useDecimal64ColumnVectors = opts.getConfiguration() != null && HiveConf.getVar(opts.getConfiguration(), HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase("decimal_64"); + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors) { this.internalBatch = opts.getSchema().createRowBatch(TypeDescription.RowBatchVersion.USE_DECIMAL64, opts.getBatchSize()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index a3b663964004..afc4e796d2b7 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -3874,7 +3874,8 @@ public void testRowNumberUniquenessInDifferentSplits() throws Exception { public void testSchemaEvolutionOldDecimal() throws Exception { TypeDescription fileSchema = TypeDescription.fromString("struct,d:string>"); - conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, "decimal_64"); + conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, + HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .fileSystem(fs) @@ -3946,7 +3947,8 @@ public void testSchemaEvolutionOldDecimal() throws Exception { public void testSchemaEvolutionDecimal64() throws Exception { TypeDescription fileSchema = TypeDescription.fromString("struct,d:string>"); - conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, "decimal_64"); + conf.set(ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED.varname, + HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .fileSystem(fs) From 4995f03078d29e70201eb03f0adb51b03413ad15 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 17 Sep 2026 09:24:52 +0200 Subject: [PATCH 14/17] Address Copilot review comments on PR #6793 Three Critical findings on the native Parquet cache path: - projectedLeaves used fileSchema.getFieldIndex to pick column chunks, but BlockMetaData.getColumns() is in flat leaf order. A file schema like 'group nested {a,b}, x' projecting 'x' would hit nested.b instead of x. Map each requested top-level primitive to its single-segment leaf in fileSchema.getColumns() instead. Add a regression test. - putColumn set part.owned only after processing all missing ranges for the column, so a mid-run throw from putFileData (e.g. its length-mismatch guard) left already-inserted cache buffers looking like raw allocations and finishFetch would allocator.deallocate cache-owned memory. Insert one range at a time and flip ownership per part. - ParquetEncodedDataReader.loadFooter builds requestedSchema without the row-lineage columns the fallback reader adds via RowLineageUtils.getRequestedSchemaWithRowLineageColumns, so with row lineage on the native path would silently emit nulls for ROW__LINEAGE__ID / LAST__UPDATED__SEQUENCE__NUMBER. Detect that case in the producer and fall back, matching the nested-projection fallback. Co-Authored-By: Claude Code --- .../decode/ParquetColumnVectorProducer.java | 18 ++++ .../io/encoded/ParquetEncodedDataReader.java | 85 ++++++++++++++----- .../encoded/TestParquetEncodedDataReader.java | 42 +++++++++ 3 files changed, 123 insertions(+), 22 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java index 9e159b166c7f..a4e0a9e6a8b5 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java @@ -73,6 +73,24 @@ public ReadPipeline createReadPipeline(Consumer consumer, Fil + "nested types in the projection", split.getPath()); return null; } + /* + * Fallback path still caches (footer via LlapProxy, data via LlapCacheAwareFs); what's lost + * for this split is vectored reads, row-group lookahead, and its LLAP IO summary counters. + * Guard fires per split only for Iceberg-Parquet files that physically carry _row_id / + * _last_updated_sequence_number while the query projects those virtual columns. + * + * Table shape Fires? + * ------------------------------------------------------------ ------ + * Non-Iceberg Parquet never + * Iceberg-Parquet, row lineage disabled never + * Iceberg-Parquet, row lineage on, file has no _row_id yet never + * Iceberg-Parquet, row lineage on, file has lineage columns yes, per split + */ + if (reader.needsRowLineage()) { + LlapIoImpl.LOG.info("Parquet native cache: falling back to normal reader for {} due to " + + "row-lineage virtual columns", split.getPath()); + return null; + } edc.init(reader, reader); return edc; } catch (IOException e) { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java index ef66853bbf7a..94406db8de43 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java @@ -66,6 +66,10 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.StoppableAllocator; import org.apache.hadoop.hive.ql.io.parquet.ParquetRecordReaderBase; import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.ql.metadata.RowLineageUtils; +import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.UserGroupInformation; @@ -81,6 +85,7 @@ import org.apache.parquet.hadoop.util.HadoopStreams; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.SeekableInputStream; +import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; import org.apache.tez.common.CallableWithNdc; @@ -274,15 +279,49 @@ public boolean projectsNestedTypes() { return false; } - /** File-schema positions of the requested fields; column chunks follow the schema order. */ - private static int[] projectedLeaves(MessageType requestedSchema, MessageType fileSchema) { - List leaves = new ArrayList<>(); + /** + * Whether the query needs row-lineage virtual columns ({@code ROW__LINEAGE__ID} / + * {@code LAST__UPDATED__SEQUENCE__NUMBER}) that the fallback reader augments the requested schema + * with via {@link RowLineageUtils#getRequestedSchemaWithRowLineageColumns}. The native pipeline + * doesn't propagate that augmentation through {@code Includes} yet, so we defer to the fallback + * reader instead of quietly emitting nulls in the lineage slots. + */ + public boolean needsRowLineage() { + VectorizedRowBatchCtx rbCtx = Utilities.getVectorizedRowBatchCtx(jobConf); + if (rbCtx == null) { + return false; + } + MessageType fileSchema = footer.getFileMetaData().getSchema(); + return RowLineageUtils.isRowLineageColumnPresent(rbCtx, fileSchema, VirtualColumn.ROW_LINEAGE_ID) + || RowLineageUtils.isRowLineageColumnPresent(rbCtx, fileSchema, + VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER); + } + + /** + * Leaf-column positions of the requested fields; {@code BlockMetaData.getColumns()} is a flat list + * of leaves in file-schema order, so a top-level primitive at ordinal {@code k} in the file schema + * may sit at a very different leaf index (e.g. a nested group of two leaves before it shifts it + * from {@code 1} to {@code 2}). We only reach this method when every requested field is a + * top-level primitive ({@link #projectsNestedTypes()} would have forced a fallback otherwise), so + * each requested field maps to exactly one leaf whose path is a single segment. + */ + static int[] projectedLeaves(MessageType requestedSchema, MessageType fileSchema) { + List allLeaves = fileSchema.getColumns(); + List selected = new ArrayList<>(); for (Type field : requestedSchema.getFields()) { - if (fileSchema.containsField(field.getName())) { - leaves.add(fileSchema.getFieldIndex(field.getName())); + if (!fileSchema.containsField(field.getName())) { + continue; + } + for (int i = 0; i < allLeaves.size(); ++i) { + String[] path = allLeaves.get(i).getPath(); + // Single-segment path == top-level primitive; skip leaves buried inside a group. + if (path.length == 1 && path[0].equals(field.getName())) { + selected.add(i); + break; + } } } - return leaves.stream().mapToInt(Integer::intValue).toArray(); + return selected.stream().mapToInt(Integer::intValue).toArray(); } private static long bytes(int[] projected, BlockMetaData block) { @@ -525,25 +564,27 @@ private void putColumn(Allocator allocator, ColumnPlan column) { if (fileKey == null) { for (Part part : parts) { bufferManager.incRefBuffer(part.buffer); + part.owned = true; } - } else { - MemoryBuffer[] fresh = new MemoryBuffer[run.count]; - MemoryBuffer[] cached = new MemoryBuffer[run.count]; - DiskRange[] ranges = new DiskRange[run.count]; - for (int i = 0; i < run.count; ++i) { - fresh[i] = cached[i] = parts.get(i).buffer; - ranges[i] = parts.get(i).range; - } - lowLevelCache.putFileData(fileKey, ranges, cached, 0, Priority.NORMAL, counters, cacheTag); - for (int i = 0; i < run.count; ++i) { - if (cached[i] != fresh[i]) { - // The cache kept its own buffer (locked for us) and unlocked ours without freeing it. - allocator.deallocate(fresh[i]); - parts.get(i).buffer = cached[i]; - } - } + // No cache key => no putFileData; each part is already on the ref-counted side + // (incRefBuffer + owned=true), so cleanup will decRef rather than deallocate. Skip the + // cache-put loop. + continue; } + // One range at a time so a mid-run throw from putFileData (e.g. the length-mismatch guard + // in LowLevelCacheImpl) leaves ownership clean: parts already handled are marked owned so + // cleanup calls decRefBuffer on cache-owned memory, while the current and later ones stay + // raw allocations that cleanup can safely deallocate. for (Part part : parts) { + MemoryBuffer fresh = part.buffer; + MemoryBuffer[] pair = new MemoryBuffer[] { fresh }; + DiskRange[] range = new DiskRange[] { part.range }; + lowLevelCache.putFileData(fileKey, range, pair, 0, Priority.NORMAL, counters, cacheTag); + if (pair[0] != fresh) { + // The cache kept its own buffer (locked for us) and unlocked ours without freeing it. + allocator.deallocate(fresh); + part.buffer = pair[0]; + } part.owned = true; } } diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java index 8a8a4ac1bf67..811d2031965e 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -605,6 +605,48 @@ public void testNoFileKeyReadsUncached() throws Exception { } } + @Test + public void testProjectedLeavesSkipsPrecedingNestedGroup() { + /* + * File schema: struct nested { a, b }, x. Column chunks are in leaf order — nested.a, + * nested.b, x — so x's top-level ordinal (1) diverges from its leaf ordinal (2); the + * method must return the leaf ordinal. + */ + MessageType fileSchema = Types.buildMessage() + .requiredGroup() + .required(PrimitiveTypeName.INT32).named("a") + .required(PrimitiveTypeName.INT32).named("b") + .named("nested") + .required(PrimitiveTypeName.INT32).named("x") + .named("file_schema"); + MessageType requested = Types.buildMessage() + .required(PrimitiveTypeName.INT32).named("x") + .named("requested"); + + int[] leaves = ParquetEncodedDataReader.projectedLeaves(requested, fileSchema); + + // x sits at leaf index 2 (nested.a=0, nested.b=1, x=2), not at its top-level ordinal 1. + assertArrayEquals(new int[] { 2 }, leaves); + } + + @Test + public void testProjectedLeavesMissingFieldSkipped() { + // File schema: a, b. Requested: a, missing. The method must resolve a and drop the + // unknown field silently rather than returning a placeholder or throwing. + MessageType fileSchema = Types.buildMessage() + .required(PrimitiveTypeName.INT32).named("a") + .required(PrimitiveTypeName.INT32).named("b") + .named("file_schema"); + MessageType requested = Types.buildMessage() + .required(PrimitiveTypeName.INT32).named("a") + .required(PrimitiveTypeName.INT32).named("missing") + .named("requested"); + + // Only a is present in the file schema (leaf 0); "missing" contributes nothing. + assertArrayEquals(new int[] { 0 }, + ParquetEncodedDataReader.projectedLeaves(requested, fileSchema)); + } + // ---- fixture ---- private static void writeFile(Path path, Configuration conf) throws IOException { From 276fe05b646bb3fb2acada335a6f57d46cbf50c3 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 17 Sep 2026 12:00:50 +0200 Subject: [PATCH 15/17] Address SonarCloud findings on PR #6793 Fix the mechanical issues flagged on files this PR modifies. Skips the 15 findings inside verbatim extractions from VectorizedParquetRecordReader (kept behavior-preserving) and the switch/case indentation reports. Notable non-trivial fixes: - LlapIo.llapVectorizedParquetReaderForPath: 9 params -> 3, collected into a new LlapParquetReadRequest record. - LlapInputFormat.getRecordReader: extract wrapOrFallback() to bring cognitive complexity under threshold. - ParquetEncodedDataReader: extract planMissRun() from planColumnChunk; drop unused 'includes' field/param, unused maxAlloc param, and Part.miss; narrow catch(Throwable) -> catch(Exception) at four sites and preserve InterruptedException on the thread instead of masking it. - ParquetCacheLayout / ParquetEncodedDataReader: replace loop-counter mutation with while loops. - CacheChunk: drop stale @VisibleForTesting; it is part of the cache's public read surface for both ORC and Parquet. - checkstyle/suppressions.xml: suppress VisibilityModifier for ParquetEncodedColumnBatch (matches parent EncodedColumnBatch pattern). - Delete stale checkListColumnSupport from VectorizedParquetRecordReader (moved to ParquetRowGroupDecoder). Style-only elsewhere: pattern-instanceof, split multi-decls, empty-body 'why' comments, header rewraps, paren-pad, restricted-identifier rename (record() -> recordRanges()). Verified: mvn checkstyle:check on llap-server (0 violations) and the existing test suites: TestParquetEncodedDataReader (27/27), TestLlapRecordReader (2/2), TestParquetRangeBuffers (2/2). Co-Authored-By: Claude Code --- .../hadoop/hive/llap/io/api/LlapIo.java | 10 +- .../llap/io/api/LlapParquetReadRequest.java | 44 +++++++ .../llap/io/api/impl/LlapInputFormat.java | 33 +++-- .../hive/llap/io/api/impl/LlapIoImpl.java | 16 +-- .../io/decode/ParquetCachedPageReadStore.java | 11 +- .../decode/ParquetColumnVectorProducer.java | 13 +- .../io/decode/ParquetEncodedDataConsumer.java | 25 ++-- .../io/encoded/ParquetEncodedColumnBatch.java | 16 ++- .../io/encoded/ParquetEncodedDataReader.java | 113 ++++++++++-------- .../io/api/impl/TestLlapRecordReader.java | 5 + .../encoded/TestParquetEncodedDataReader.java | 36 +++--- .../hadoop/hive/llap/ParquetCacheLayout.java | 6 +- .../hive/ql/io/orc/encoded/CacheChunk.java | 8 +- .../vector/ParquetRowGroupDecoder.java | 15 +-- .../vector/VectorizedParquetRecordReader.java | 19 --- .../hive/llap/TestParquetRangeBuffers.java | 37 ++++-- 16 files changed, 245 insertions(+), 162 deletions(-) create mode 100644 llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapParquetReadRequest.java diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java index d96b61584a6d..da12206e4394 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java @@ -21,10 +21,8 @@ import java.io.IOException; import java.util.List; -import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.io.CacheTag; import org.apache.hadoop.hive.common.io.DataCache.BooleanRef; @@ -116,12 +114,10 @@ RecordReader llapVectorizedOrcReaderForPath(Ob /** * Parquet counterpart of {@link #llapVectorizedOrcReaderForPath}: reads the column chunks of the split through the * LLAP data cache. Returns null when the file cannot be served this way (native Parquet IO disabled, no MapWork, - * unsupported schema). - * @param initialDefaults - values for columns absent from the file, keyed by column name + * unsupported schema). See {@link LlapParquetReadRequest} for the identity / split / projection fields. */ - RecordReader llapVectorizedParquetReaderForPath(Object fileKey, Path path, - CacheTag tag, List tableIncludedCols, JobConf conf, long offset, long length, - Map initialDefaults, Reporter reporter) throws IOException; + RecordReader llapVectorizedParquetReaderForPath( + LlapParquetReadRequest request, JobConf conf, Reporter reporter) throws IOException; /** * Extract and return the cache content metadata. diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapParquetReadRequest.java b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapParquetReadRequest.java new file mode 100644 index 000000000000..67d1078687e3 --- /dev/null +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapParquetReadRequest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.llap.io.api; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.io.CacheTag; + +/** + * Everything that identifies a single Parquet split served through the LLAP data cache: the file + * (its optional {@code fileKey}, {@code path} and {@code tag}), the split ({@code offset} and + * {@code length}), the projection ({@code tableIncludedCols}), and the defaults for columns + * absent from the file ({@code initialDefaults}). Kept together so + * {@link LlapIo#llapVectorizedParquetReaderForPath} stays a three-parameter call with the job + * conf and reporter alongside. + */ +public record LlapParquetReadRequest( + Object fileKey, + Path path, + CacheTag tag, + List tableIncludedCols, + long offset, + long length, + Map initialDefaults) { +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java index 944f14130ec6..dd0512cce3d4 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java @@ -130,17 +130,10 @@ public RecordReader getRecordReader( } return sourceInputFormat.getRecordReader(split, job, reporter); } - // For non-vectorized operator case, wrap the reader if possible. - RecordReader result = rr; - if (!Utilities.getIsVectorized(job)) { - result = null; - if (HiveConf.getBoolVar(job, ConfVars.LLAP_IO_ROW_WRAPPER_ENABLED)) { - result = wrapLlapReader(tableIncludedCols, rr, split); - } - if (result == null) { - // Cannot wrap a reader for non-vectorized pipeline. - return sourceInputFormat.getRecordReader(split, job, reporter); - } + RecordReader result = + wrapOrFallback(rr, tableIncludedCols, split, job); + if (result == null) { + return sourceInputFormat.getRecordReader(split, job, reporter); } // This starts the reader in the background. rr.start(); @@ -159,6 +152,24 @@ public RecordReader getRecordReader( } } + /** + * Adapts the LLAP reader (which always produces vectorized batches) to what the surrounding + * operator pipeline expects. In a vectorized pipeline it is handed back as-is; in a + * non-vectorized one a row wrapper is attempted, and if that isn't possible (wrapper disabled + * or unavailable for this reader) null is returned so the caller falls back to + * {@link #sourceInputFormat}. + */ + private RecordReader wrapOrFallback(LlapRecordReader rr, + List tableIncludedCols, InputSplit split, JobConf job) throws IOException { + if (Utilities.getIsVectorized(job)) { + return rr; + } + if (!HiveConf.getBoolVar(job, ConfVars.LLAP_IO_ROW_WRAPPER_ENABLED)) { + return null; + } + return wrapLlapReader(tableIncludedCols, rr, split); + } + private boolean checkLimitReached(JobConf job) { /* * 2 assumptions here when using "tez.mapreduce.vertex.name" diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index d95b43d7e2fb..7c2981c9dd58 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -76,6 +76,7 @@ import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos; import org.apache.hadoop.hive.llap.io.api.LlapIo; +import org.apache.hadoop.hive.llap.io.api.LlapParquetReadRequest; import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer; import org.apache.hadoop.hive.llap.io.decode.GenericColumnVectorProducer; import org.apache.hadoop.hive.llap.io.decode.OrcColumnVectorProducer; @@ -492,20 +493,19 @@ public RecordReader llapVectorizedOrcReaderFor } @Override - public RecordReader llapVectorizedParquetReaderForPath(Object fileKey, Path path, - CacheTag tag, List tableIncludedCols, JobConf conf, long offset, long length, - Map initialDefaults, Reporter reporter) throws IOException { + public RecordReader llapVectorizedParquetReaderForPath( + LlapParquetReadRequest request, JobConf conf, Reporter reporter) throws IOException { if (parquetCvp == null) { return null; } - FileSplit split = new FileSplit(path, offset, length, (String[]) null); + FileSplit split = new FileSplit(request.path(), request.offset(), request.length(), (String[]) null); try { - LlapRecordReader rr = LlapRecordReader.create(conf, split, tableIncludedCols, HiveStringUtils.getHostname(), - parquetCvp, executor, null, null, reporter, daemonConf); - if (rr == null) { + LlapRecordReader rr = LlapRecordReader.create(conf, split, request.tableIncludedCols(), + HiveStringUtils.getHostname(), parquetCvp, executor, null, null, reporter, daemonConf); + if (rr == null) { // NOSONAR - S2583: create() has null-return paths Sonar's data-flow does not model. return null; } - ((ParquetEncodedDataConsumer) rr.getReadPipeline()).setInitialDefaults(initialDefaults); + ((ParquetEncodedDataConsumer) rr.getReadPipeline()).setInitialDefaults(request.initialDefaults()); rr.setPartitionValues(null); rr.start(); return rr; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java index a35c770d09cb..00be2b5a140f 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java @@ -9,11 +9,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.llap.io.decode; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java index a4e0a9e6a8b5..1814ce8bd34e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetColumnVectorProducer.java @@ -9,11 +9,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.llap.io.decode; @@ -65,7 +66,7 @@ public ReadPipeline createReadPipeline(Consumer consumer, Fil ParquetEncodedDataConsumer edc = new ParquetEncodedDataConsumer(consumer, includes, counters, ioMetrics, job); ParquetEncodedDataReader reader = new ParquetEncodedDataReader( - lowLevelCache, bufferManager, conf, job, split, includes, edc, counters); + lowLevelCache, bufferManager, conf, job, split, edc, counters); reader.loadFooter(); // Null makes LlapInputFormat fall back to the normal VectorizedParquetRecordReader. if (reader.projectsNestedTypes()) { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java index d25f665c8e76..46b3e7bc89c3 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java @@ -9,11 +9,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.llap.io.decode; @@ -174,9 +175,7 @@ protected void decodeBatch(ParquetEncodedColumnBatch batch, if (columnReaders[i] == null) { continue; } - TypeInfo columnType = readAllColumns - ? columnTypesList.get(i) - : columnTypesList.get(colsToInclude.get(i)); + TypeInfo columnType = columnTypesList.get(readAllColumns ? i : colsToInclude.get(i)); ColumnVector cv = prepareColumnVector(cvb, i, columnType, batchSize); columnReaders[i].readBatch(batchSize, cv, columnType); } @@ -194,8 +193,10 @@ protected void decodeBatch(ParquetEncodedColumnBatch batch, LlapIoImpl.LOG.error("Parquet decodeBatch failed for rowGroup " + batch.rowGroupIx + " of " + path, e); downstreamConsumer.setError(e); } finally { - // Returns the pooled Hadoop decompressors after each row group; getDecompressor re-creates them. - codecFactory.release(); + // Returns the pooled Hadoop decompressors after each row group; getDecompressor re-creates + // them. codecFactory is a per-consumer field with a per-batch release() (not close()) — + // try-with-resources would tie it to the outer decodeBatch scope, which is wrong here. + codecFactory.release(); // NOSONAR - see comment above (S2093 does not apply) } } @@ -212,8 +213,8 @@ private ColumnVector prepareColumnVector(ColumnVectorBatch cvb, int idx, TypeInf } private DataTypePhysicalVariation physicalVariation(TypeInfo columnType) { - if (useDecimal64ColumnVectors && columnType instanceof DecimalTypeInfo - && ((DecimalTypeInfo) columnType).precision() <= TypeDescription.MAX_DECIMAL64_PRECISION) { + if (useDecimal64ColumnVectors && columnType instanceof DecimalTypeInfo decimalType + && decimalType.precision() <= TypeDescription.MAX_DECIMAL64_PRECISION) { return DataTypePhysicalVariation.DECIMAL_64; } return DataTypePhysicalVariation.NONE; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java index 24e94c6e35fa..bb950817dbcb 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java @@ -9,11 +9,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.llap.io.encoded; @@ -37,7 +38,10 @@ public class ParquetEncodedColumnBatch extends EncodedColumnBatch { public long[][] bufferOffsets; public int[][] bufferLengths; - public ParquetEncodedColumnBatch() {} + public ParquetEncodedColumnBatch() { + // No-arg constructor for pooling / reflection-based construction; fields are populated later + // by init(...). Left empty on purpose. + } /** fileKey is the cache key; rowGroupIx is the footer index of the block within the file. */ public void init(Object fileKey, int rowGroupIx, ColumnChunkMetaData[] chunks) { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java index 94406db8de43..bbb995c9e9f6 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java @@ -9,11 +9,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.llap.io.encoded; @@ -56,7 +57,6 @@ import org.apache.hadoop.hive.llap.counters.LlapIOCounters; import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; import org.apache.hadoop.hive.llap.io.api.LlapProxy; -import org.apache.hadoop.hive.llap.io.decode.ColumnVectorProducer.Includes; import org.apache.hadoop.hive.llap.io.decode.ParquetEncodedDataConsumer; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.SyntheticFileId; @@ -110,7 +110,6 @@ public class ParquetEncodedDataReader extends CallableWithNdc private final ParquetCacheLayout layout; private final JobConf jobConf; private final FileSplit split; - private final Includes includes; private final ParquetEncodedDataConsumer consumer; private final QueryFragmentCounters counters; private final UserGroupInformation ugi; @@ -126,7 +125,7 @@ public class ParquetEncodedDataReader extends CallableWithNdc private final AtomicBoolean isStopped = new AtomicBoolean(false); public ParquetEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager bufferManager, - Configuration daemonConf, Configuration jobConf, FileSplit split, Includes includes, + Configuration daemonConf, Configuration jobConf, FileSplit split, ParquetEncodedDataConsumer consumer, QueryFragmentCounters counters) throws IOException { this.lowLevelCache = lowLevelCache; this.bufferManager = bufferManager; @@ -134,7 +133,6 @@ public ParquetEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager this.layout = new ParquetCacheLayout(bufferManager.getAllocator(), daemonConf); this.jobConf = (JobConf) jobConf; this.split = split; - this.includes = includes; this.consumer = consumer; this.counters = counters; this.path = split.getPath(); @@ -197,8 +195,13 @@ protected Void callInternal() throws IOException, InterruptedException { try { performDataRead(); consumer.setDone(); - } catch (Throwable t) { - consumer.setError(t); + } catch (Exception e) { + // A shutdown-triggered InterruptedException must not be silently reported as an ordinary + // consumer error: preserve the interrupt on the thread so any caller sees it. + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + consumer.setError(e); } finally { counters.incrWallClockCounter(LlapIOCounters.TOTAL_IO_TIME_NS, startTime); } @@ -220,8 +223,8 @@ private void performDataRead() throws IOException, InterruptedException { consumer.setFileMetadata(footer, requestedSchema, path); final Allocator allocator = bufferManager.getAllocator(); - final int maxAlloc = allocator.getMaxAllocation(); - final long splitStart = split.getStart(), splitEnd = splitStart + split.getLength(); + final long splitStart = split.getStart(); + final long splitEnd = splitStart + split.getLength(); final List blocks = footer.getBlocks(); List selected = new ArrayList<>(); for (BlockMetaData block : blocks) { @@ -250,13 +253,13 @@ private void performDataRead() throws IOException, InterruptedException { try { for (int i = 0; i < selected.size() && !isStopped.get(); ++i) { if (inFlight.isEmpty()) { - inFlight.add(startFetch(fileStream, buffers, allocator, maxAlloc, projected, selected.get(i), + inFlight.add(startFetch(fileStream, buffers, allocator, projected, selected.get(i), rowGroupOf.get(selected.get(i)))); } // The next row group's requests go out now so its transfer overlaps this one's decode. if (i + 1 < selected.size() && !isStopped.get() && bytes(inFlight.peek()) + bytes(projected, selected.get(i + 1)) <= lookaheadBudget) { - inFlight.add(startFetch(fileStream, buffers, allocator, maxAlloc, projected, selected.get(i + 1), + inFlight.add(startFetch(fileStream, buffers, allocator, projected, selected.get(i + 1), rowGroupOf.get(selected.get(i + 1)))); } finishFetch(allocator, buffers, inFlight.poll()); @@ -359,15 +362,13 @@ private record ColumnPlan(List parts, List missRuns) { private static final class Part { private MemoryBuffer buffer; private final DiskRange range; - private final boolean miss; /** We hold one ref to release; until then a miss is a raw allocation to free. */ private boolean owned; - Part(MemoryBuffer buffer, DiskRange range, boolean miss) { + Part(MemoryBuffer buffer, DiskRange range, boolean owned) { this.buffer = buffer; this.range = range; - this.miss = miss; - this.owned = !miss; + this.owned = owned; } } @@ -380,7 +381,7 @@ private record Run(FileRange range, List parts) { } private Fetch startFetch(FSDataInputStream fileStream, ParquetRangeBuffers buffers, Allocator allocator, - int maxAlloc, int[] projected, BlockMetaData block, int rg) + int[] projected, BlockMetaData block, int rg) throws IOException { Fetch fetch = new Fetch(); ColumnChunkMetaData[] chunks = new ColumnChunkMetaData[projected.length]; @@ -392,16 +393,16 @@ private Fetch startFetch(FSDataInputStream fileStream, ParquetRangeBuffers buffe for (ColumnChunkMetaData chunk : chunks) { ColumnPlan column = new ColumnPlan(); fetch.columns.add(column); - planColumnChunk(allocator, maxAlloc, chunk.getStartingPos(), + planColumnChunk(allocator, chunk.getStartingPos(), chunk.getStartingPos() + chunk.getTotalSize(), column, fetch.misses); } // The vectored dispatch is where non-async FS impls actually read; count it under HDFS_TIME. long hdfsStart = counters.startTimeCounter(); requestMisses(fileStream, buffers, fetch, layout.maxRangeBytes()); counters.recordHdfsTime(hdfsStart); - } catch (Throwable t) { + } catch (Exception e) { abandon(allocator, fetch); - throw t; + throw e; } return fetch; } @@ -421,9 +422,9 @@ private void finishFetch(Allocator allocator, ParquetRangeBuffers buffers, Fetch } // consumeData returns the batch on success; after a throw it is still ours. consumer.consumeData(fetch.batch); - } catch (Throwable t) { + } catch (Exception e) { abandon(allocator, fetch); - throw t; + throw e; } } @@ -450,8 +451,8 @@ FSDataInputStream openFile(FileSystem fs) throws IOException { /** Lets the allocator abandon a wait for memory once the fragment is cancelled. */ private void allocateMultiple(Allocator allocator, MemoryBuffer[] dest, int size) { - if (allocator instanceof StoppableAllocator) { - ((StoppableAllocator) allocator).allocateMultiple(dest, size, DATA_BUFFER_FACTORY, isStopped); + if (allocator instanceof StoppableAllocator stoppable) { + stoppable.allocateMultiple(dest, size, DATA_BUFFER_FACTORY, isStopped); } else { allocator.allocateMultiple(dest, size, DATA_BUFFER_FACTORY); } @@ -462,7 +463,7 @@ private void allocateMultiple(Allocator allocator, MemoryBuffer[] dest, int size * hits as returned by getFileData, and freshly allocated power-of-two buffers for everything * missing. Misses are appended to {@code allMisses} so the whole row group can be read at once. */ - private void planColumnChunk(Allocator allocator, int maxAlloc, long start, long end, + private void planColumnChunk(Allocator allocator, long start, long end, ColumnPlan column, List allMisses) throws IOException { DiskRangeList head = new DiskRangeList(start, end); if (fileKey != null) { @@ -473,26 +474,12 @@ private void planColumnChunk(Allocator allocator, int maxAlloc, long start, long try { for (; current != null; current = current.next) { if (current.hasData()) { - column.parts.add(new Part(((CacheChunk) current).getBuffer(), current, false)); - continue; - } - LlapHiveUtils.throwIfCacheOnlyRead(cacheOnly); - int[] sizes = layout.bufferSizes(current.getEnd() - current.getOffset()); - column.missRuns.add(new MissRun(column.parts.size(), sizes.length)); - long partFrom = current.getOffset(); - for (int size : sizes) { - MemoryBuffer[] one = new MemoryBuffer[1]; - allocateMultiple(allocator, one, size); - // The cache accounts and serves the bytes up to the buffer's limit. - ByteBuffer raw = one[0].getByteBufferRaw(); - raw.limit(raw.position() + size); - Part part = new Part(one[0], new DiskRange(partFrom, partFrom + size), true); - column.parts.add(part); - allMisses.add(part); - partFrom += size; + column.parts.add(new Part(((CacheChunk) current).getBuffer(), current, true)); + } else { + planMissRun(allocator, current, column, allMisses); } } - } catch (Throwable t) { + } catch (Exception e) { // Hits past the failure point are still locked by getFileData; the caller only knows about // the parts already recorded. for (current = current.next; current != null; current = current.next) { @@ -500,7 +487,30 @@ private void planColumnChunk(Allocator allocator, int maxAlloc, long start, long bufferManager.decRefBuffer(((CacheChunk) current).getBuffer()); } } - throw t; + throw e; + } + } + + /** + * Records one missing sub-range as a run of freshly allocated power-of-two buffers, and appends + * each part to {@code allMisses} so the caller can request them all in one vectored call. + */ + private void planMissRun(Allocator allocator, DiskRangeList missing, ColumnPlan column, + List allMisses) throws IOException { + LlapHiveUtils.throwIfCacheOnlyRead(cacheOnly); + int[] sizes = layout.bufferSizes(missing.getEnd() - missing.getOffset()); + column.missRuns.add(new MissRun(column.parts.size(), sizes.length)); + long partFrom = missing.getOffset(); + for (int size : sizes) { + MemoryBuffer[] one = new MemoryBuffer[1]; + allocateMultiple(allocator, one, size); + // The cache accounts and serves the bytes up to the buffer's limit. + ByteBuffer raw = one[0].getByteBufferRaw(); + raw.limit(raw.position() + size); + Part part = new Part(one[0], new DiskRange(partFrom, partFrom + size), false); + column.parts.add(part); + allMisses.add(part); + partFrom += size; } } @@ -519,16 +529,18 @@ private static void requestMisses(FSDataInputStream fileStream, ParquetRangeBuff } misses.sort(Comparator.comparingLong(part -> part.range.getOffset())); List runs = new ArrayList<>(); - for (int i = 0; i < misses.size(); ) { + int i = 0; + while (i < misses.size()) { long from = misses.get(i).range.getOffset(); long to = misses.get(i).range.getEnd(); int j = i + 1; - for (; j < misses.size(); ++j) { + while (j < misses.size()) { DiskRange next = misses.get(j).range; if (next.getOffset() != to || next.getEnd() - from > maxRange) { break; } to = next.getEnd(); + ++j; } runs.add(new Run(FileRange.createFileRange(from, (int) (to - from)), misses.subList(i, j))); i = j; @@ -614,10 +626,13 @@ public void returnData(ParquetEncodedColumnBatch batch) { @Override public void pause() { + // The reader has no pausable state: the IO thread runs one row group at a time and yields on + // finishFetch() naturally. Kept to satisfy the ConsumerFeedback contract. } @Override public void unpause() { + // See pause(): no state to resume. } @Override diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java index eaa5cbfbcd11..18e36943dd8a 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/api/impl/TestLlapRecordReader.java @@ -155,18 +155,22 @@ public SchemaEvolution getSchemaEvolution() { @Override public void pause() { + // Scripted read pipeline: no state to pause. } @Override public void unpause() { + // Scripted read pipeline: no state to resume. } @Override public void stop() { + // Scripted read pipeline: no work in flight to stop. } @Override public void returnData(ColumnVectorBatch data) { + // Scripted read pipeline: batches are test-owned; nothing to reclaim. } }; } @@ -184,6 +188,7 @@ public void execute(Runnable command) { @Override public void shutdown() { + // Test executor: no threads to interrupt. } @Override diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java index 811d2031965e..bc9c9eb708e2 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -9,11 +9,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.llap.io.encoded; @@ -364,9 +365,10 @@ public void testCachedBuffersAreNotReread() throws Exception { @Test public void testFooterLookupBumpsMetadataCacheCounters() throws Exception { - // The footer cache is a @BeforeClass singleton, so any earlier test may have populated it; - // only the second read of this test is order-independent. That call finds the footer we just - // put in on the first read, so META_HIT is 1 and META_MISS is 0 whatever came before. + // Footer cache is a @BeforeClass singleton, so an earlier test may have already populated it. + // We only assert on the *second* read here — its counters are order-independent because the + // second read always finds the footer that the first read of this test put in (META_HIT is 1 + // and META_MISS is 0 regardless of prior state). read(jobConf(COLUMNS, TYPES, 0), wholeFile()).assertClean(); Run second = read(jobConf(COLUMNS, TYPES, 0), wholeFile()); @@ -778,17 +780,17 @@ protected void decodeBatch(ParquetEncodedColumnBatch batch, Consumer reads = new ArrayList<>(); - ParquetEncodedDataReader reader = new ParquetEncodedDataReader( - ledger, ledger, readerDaemonConf, job, split, includes(projection), edc, counters) { + ParquetEncodedDataReader parquetReader = new ParquetEncodedDataReader( + ledger, ledger, readerDaemonConf, job, split, edc, counters) { @Override FSDataInputStream openFile(FileSystem fs) throws IOException { return new RecordingStream(super.openFile(fs), reads); } }; - this.reader = reader; - edc.init(reader, reader); - reader.loadFooter(); - reader.call(); + this.reader = parquetReader; + edc.init(parquetReader, parquetReader); + parquetReader.loadFooter(); + parquetReader.call(); return new Run(downstream, counters, tezCounters, buffers, reads, ledger); } @@ -818,7 +820,7 @@ public boolean hasCapability(String capability) { @Override public void readVectored(List ranges, IntFunction allocate) throws IOException { - record(ranges); + recordRanges(ranges); super.readVectored(ranges, allocate); injectFailure(ranges); } @@ -826,12 +828,12 @@ public void readVectored(List ranges, IntFunction ranges, IntFunction allocate, java.util.function.Consumer release) throws IOException { - record(ranges); + recordRanges(ranges); super.readVectored(ranges, allocate, release); injectFailure(ranges); } - private void record(List ranges) { + private void recordRanges(List ranges) { for (FileRange range : ranges) { reads.add(new long[] {range.getOffset(), range.getLength()}); } diff --git a/ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java b/ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java index da04f9d238fb..736467287936 100644 --- a/ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java +++ b/ql/src/java/org/apache/hadoop/hive/llap/ParquetCacheLayout.java @@ -83,8 +83,10 @@ public int maxRangeBytes() { */ public int[] bufferSizes(long length) { int count = 0; - for (long left = length; left > 0; ++count) { - left -= left < minBuffer ? left : Math.min(maxBuffer, Long.highestOneBit(left)); + long remaining = length; + while (remaining > 0) { + remaining -= remaining < minBuffer ? remaining : Math.min(maxBuffer, Long.highestOneBit(remaining)); + ++count; } int[] sizes = new int[count]; long left = length; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/CacheChunk.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/CacheChunk.java index 9b3199e504c8..1174672cc1fd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/CacheChunk.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/CacheChunk.java @@ -26,13 +26,11 @@ import org.apache.hadoop.hive.common.io.DiskRangeList; import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; -import com.google.common.annotations.VisibleForTesting; - /** - * DiskRange containing encoded, uncompressed data from cache. - * It should be hidden inside EncodedReaderImpl, but we need to expose it for tests. + * DiskRange containing encoded, uncompressed data from cache. Beyond ORC's own EncodedReaderImpl, + * it is part of the cache's public read surface: any reader consuming cached chunks — ORC and + * Parquet alike — receives them typed as CacheChunk from the LowLevelCache. */ -@VisibleForTesting public class CacheChunk extends DiskRangeList { protected MemoryBuffer buffer; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java index 0cc6ab8de4b2..b62f8592b25f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java @@ -9,11 +9,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.ql.io.parquet.vector; @@ -104,7 +105,7 @@ public VectorizedColumnReader[] buildColumnReaders( } } else { for (int i = 0; i < types.size(); ++i) { - columnReaders[i] = buildVectorizedParquetReader( columnTypesList.get(i), + columnReaders[i] = buildVectorizedParquetReader(columnTypesList.get(i), types.get(i), pages, requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, skipProlepticConversion, legacyConversionEnabled, 0, 0); } @@ -189,7 +190,7 @@ private VectorizedColumnReader buildVectorizedParquetReader( List types = type.asGroupType().getFields(); for (int i = 0; i < fieldTypes.size(); i++) { VectorizedColumnReader r = - buildVectorizedParquetReader( fieldTypes.get(i), types.get(i), pages, + buildVectorizedParquetReader(fieldTypes.get(i), types.get(i), pages, descriptors, skipTimestampConversion, writerTimezone, skipProlepticConversion, legacyConversionEnabled, depth + 1, typeDefLevel); if (r != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index cbdbfb232e9c..d6bdc3826695 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -48,7 +48,6 @@ import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; @@ -452,22 +451,4 @@ private void checkEndOfRowGroup() throws IOException { totalCountLoadedSoFar += pages.getRowCount(); } - - - /** - * Check if the element type in list is supported by vectorization read. - * Supported type: INT, BYTE, SHORT, DATE, INTERVAL_YEAR_MONTH, LONG, BOOLEAN, DOUBLE, BINARY, STRING, CHAR, VARCHAR, - * FLOAT, DECIMAL - */ - private void checkListColumnSupport(TypeInfo elementType) { - if (elementType instanceof PrimitiveTypeInfo) { - switch (((PrimitiveTypeInfo)elementType).getPrimitiveCategory()) { - case INTERVAL_DAY_TIME: - case TIMESTAMP: - throw new RuntimeException("Unsupported primitive type used in list:: " + elementType); - } - } else { - throw new RuntimeException("Unsupported type used in list:" + elementType); - } - } } diff --git a/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java index 9461e497313c..76a289d556a0 100644 --- a/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java +++ b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java @@ -60,7 +60,7 @@ public void aSlicingStreamsBuffersAreNeverPooled() throws IOException { } /** A stream that does nothing but answer whether its vectored-read buffers are slices. */ - private static FSDataInputStream stream(boolean sliced) throws IOException { + private static FSDataInputStream stream(boolean sliced) { return new FSDataInputStream(new Inert()) { @Override public boolean hasCapability(String capability) { @@ -70,12 +70,33 @@ public boolean hasCapability(String capability) { } private static final class Inert extends InputStream implements Seekable, PositionedReadable { - @Override public int read() { return -1; } - @Override public void seek(long pos) { } - @Override public long getPos() { return 0; } - @Override public boolean seekToNewSource(long targetPos) { return false; } - @Override public int read(long position, byte[] buffer, int offset, int length) { return -1; } - @Override public void readFully(long position, byte[] buffer, int offset, int length) { } - @Override public void readFully(long position, byte[] buffer) { } + @Override + public int read() { + return -1; + } + @Override + public void seek(long pos) { + // Test double: no state to seek. + } + @Override + public long getPos() { + return 0; + } + @Override + public boolean seekToNewSource(long targetPos) { + return false; + } + @Override + public int read(long position, byte[] buffer, int offset, int length) { + return -1; + } + @Override + public void readFully(long position, byte[] buffer, int offset, int length) { + // Test double: no bytes to fill. + } + @Override + public void readFully(long position, byte[] buffer) { + // Test double: no bytes to fill. + } } } From 2ac8454e786a610b49b0f0b354fdf6350c3a617a Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Fri, 18 Sep 2026 09:16:33 +0200 Subject: [PATCH 16/17] hive.llap.io.parquet.native.enable=true default --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3f28c636e1e3..1ed2110b9347 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4983,7 +4983,7 @@ public static enum ConfVars { LLAP_IO_ENCODE_ENABLED("hive.llap.io.encode.enabled", true, "Whether LLAP should try to re-encode and cache data for non-ORC formats. This is used\n" + "on LLAP Server side to determine if the infrastructure for that is initialized."), - LLAP_IO_PARQUET_NATIVE_ENABLED("hive.llap.io.parquet.native.enabled", false, + LLAP_IO_PARQUET_NATIVE_ENABLED("hive.llap.io.parquet.native.enabled", true, "Whether LLAP IO caches Parquet column chunks and decodes them from the cache, like ORC.\n" + "A projection that reaches into a nested type falls back to the vectorized Parquet reader."), LLAP_IO_ENCODE_FORMATS("hive.llap.io.encode.formats", From bfaa4eea89953a29c6bbb72fe445f0c5a2b59a9c Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Fri, 18 Sep 2026 09:56:58 +0200 Subject: [PATCH 17/17] Address SonarCloud findings on PR #6793 (round 2) - ORC dot-continuation lines: fix 4-space continuation indent - LlapIoImpl: split combined declarations, remove chained assignments - ParquetEncodedDataConsumer: NOSONAR on try line for S2093 - ParquetEncodedColumnBatch: make fields private, add accessors - ParquetEncodedDataReader / TestParquetEncodedDataReader: fix '{ '-after-brace whitespace, use accessors - ParquetCachedPageReadStore: use accessors - ParquetRowGroupDecoder: reduce parameter counts via TimestampConversionOptions record; replace generic RuntimeException with UnsupportedOperationException / IllegalStateException / InvalidSchemaException; pattern-match instanceof; rewrite commented-out MAP schema example; merge case labels; retire the moved TODO - TestParquetRangeBuffers: drop unused throws IOException Co-Authored-By: Claude Code --- .../hive/llap/io/api/impl/LlapIoImpl.java | 12 +- .../io/decode/ParquetCachedPageReadStore.java | 26 ++-- .../io/decode/ParquetEncodedDataConsumer.java | 10 +- .../llap/io/encoded/OrcEncodedDataReader.java | 4 +- .../io/encoded/ParquetEncodedColumnBatch.java | 34 +++-- .../io/encoded/ParquetEncodedDataReader.java | 23 ++-- .../io/encoded/SerDeEncodedDataReader.java | 4 +- .../encoded/VectorDeserializeOrcWriter.java | 4 +- .../encoded/TestParquetEncodedDataReader.java | 6 +- .../hive/ql/io/orc/RecordReaderImpl.java | 4 +- .../orc/VectorizedOrcAcidRowBatchReader.java | 8 +- .../vector/ParquetRowGroupDecoder.java | 117 +++++++++--------- .../vector/VectorizedParquetRecordReader.java | 5 +- .../hive/llap/TestParquetRangeBuffers.java | 5 +- 14 files changed, 147 insertions(+), 115 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index 7c2981c9dd58..3687133f56b6 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -179,7 +178,8 @@ private LlapIoImpl(Configuration conf) throws IOException { MetadataCache metadataCache = null; SerDeLowLevelCacheImpl serdeCache = null; // TODO: extract interface when needed - BufferUsageManager bufferManagerData = null, bufferManagerGeneric = null; + BufferUsageManager bufferManagerData = null; + BufferUsageManager bufferManagerGeneric = null; boolean isEncodeEnabled = useLowLevelCache && HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_ENCODE_ENABLED); if (useLowLevelCache) { @@ -227,7 +227,9 @@ private LlapIoImpl(Configuration conf) throws IOException { cachePolicyWrapper.setEvictionListener(e); cacheImpl.startThreads(); // Start the cache threads. - bufferManager = bufferManagerData = cacheImpl; // Cache also serves as buffer manager. + // Cache also serves as buffer manager for both data and generic (encoded/serde) paths. + bufferManagerData = cacheImpl; + bufferManager = cacheImpl; bufferManagerGeneric = serdeCache; if (trackUsage) { debugDumpComponents.add(cachePolicyWrapper); // Cache contents tracker. @@ -246,7 +248,9 @@ private LlapIoImpl(Configuration conf) throws IOException { this.allocator = new SimpleAllocator(conf); fileMetadataCache = null; SimpleBufferManager sbm = new SimpleBufferManager(allocator, cacheMetrics); - bufferManager = bufferManagerData = bufferManagerGeneric = sbm; + bufferManager = sbm; + bufferManagerData = sbm; + bufferManagerGeneric = sbm; dataCache = sbm; this.memoryManager = null; debugDumpComponents.add(new LlapIoDebugDump() { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java index 00be2b5a140f..9094e6120364 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetCachedPageReadStore.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Queue; +import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; import org.apache.hadoop.hive.llap.io.encoded.ParquetEncodedColumnBatch; import org.apache.parquet.bytes.ByteBufferInputStream; import org.apache.parquet.bytes.BytesInput; @@ -64,11 +65,12 @@ class ParquetCachedPageReadStore implements PageReadStore { ParquetCachedPageReadStore(ParquetMetadata footer, ParquetEncodedColumnBatch batch, CompressionCodecFactory codecFactory, ParquetMetadataConverter converter) throws IOException { - BlockMetaData block = footer.getBlocks().get(batch.rowGroupIx); + BlockMetaData block = footer.getBlocks().get(batch.rowGroupIx()); this.rowCount = block.getRowCount(); String createdBy = footer.getFileMetaData().getCreatedBy(); - for (int pc = 0; pc < batch.chunks.length; ++pc) { - ColumnChunkMetaData chunk = batch.chunks[pc]; + ColumnChunkMetaData[] chunks = batch.chunks(); + for (int pc = 0; pc < chunks.length; ++pc) { + ColumnChunkMetaData chunk = chunks[pc]; readers.put(chunk.getPath(), readAllPages(chunk, chunkBuffers(batch, pc), createdBy, codecFactory.getDecompressor(chunk.getCodec()), converter)); } @@ -76,15 +78,19 @@ class ParquetCachedPageReadStore implements PageReadStore { /** Slices of the cached buffers covering exactly the chunk's byte region, in file order. */ private static List chunkBuffers(ParquetEncodedColumnBatch batch, int pc) { - long chunkStart = batch.chunks[pc].getStartingPos(); - long chunkEnd = chunkStart + batch.chunks[pc].getTotalSize(); - List slices = new ArrayList<>(batch.columnBuffers[pc].length); - for (int i = 0; i < batch.columnBuffers[pc].length; ++i) { - long bufferStart = batch.bufferOffsets[pc][i]; - long bufferEnd = bufferStart + batch.bufferLengths[pc][i]; + ColumnChunkMetaData chunk = batch.chunks()[pc]; + long chunkStart = chunk.getStartingPos(); + long chunkEnd = chunkStart + chunk.getTotalSize(); + MemoryBuffer[] columnBuffers = batch.columnBuffers()[pc]; + long[] bufferOffsets = batch.bufferOffsets()[pc]; + int[] bufferLengths = batch.bufferLengths()[pc]; + List slices = new ArrayList<>(columnBuffers.length); + for (int i = 0; i < columnBuffers.length; ++i) { + long bufferStart = bufferOffsets[i]; + long bufferEnd = bufferStart + bufferLengths[i]; long sliceStart = Math.max(chunkStart, bufferStart); long sliceEnd = Math.min(chunkEnd, bufferEnd); - ByteBuffer bb = batch.columnBuffers[pc][i].getByteBufferDup(); + ByteBuffer bb = columnBuffers[i].getByteBufferDup(); bb.position(bb.position() + (int) (sliceStart - bufferStart)); bb.limit(bb.position() + (int) (sliceEnd - sliceStart)); slices.add(bb.slice()); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java index 46b3e7bc89c3..737158a8468d 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/ParquetEncodedDataConsumer.java @@ -151,14 +151,16 @@ protected void decodeBatch(ParquetEncodedColumnBatch batch, } long startTime = counters.startTimeCounter(); - try { + // NOSONAR - S2093 does not apply: codecFactory is a per-consumer field with a per-batch + // release() (not close()); try-with-resources would tie it to this scope, which is wrong. + try { // NOSONAR PageReadStore pages = new ParquetCachedPageReadStore(footer, batch, codecFactory, converter); VectorizedColumnReader[] columnReaders = new ParquetRowGroupDecoder(footer.getFileMetaData().getSchema(), initialDefaults).buildColumnReaders( pages, requestedSchema, columnTypesList, colsToInclude, readAllColumns, - skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled); + new ParquetRowGroupDecoder.TimestampConversionOptions(skipTimestampConversion, + writerTimezone, skipProlepticConversion, legacyConversionEnabled)); long rowCount = pages.getRowCount(); long rowsLeft = rowCount; @@ -190,7 +192,7 @@ protected void decodeBatch(ParquetEncodedColumnBatch batch, counters.incrCounter(LlapIOCounters.NUM_DECODED_BATCHES); } catch (IOException | RuntimeException e) { // parquet-mr reports decode failures as runtime ParquetDecodingException. - LlapIoImpl.LOG.error("Parquet decodeBatch failed for rowGroup " + batch.rowGroupIx + " of " + path, e); + LlapIoImpl.LOG.error("Parquet decodeBatch failed for rowGroup " + batch.rowGroupIx() + " of " + path, e); downstreamConsumer.setError(e); } finally { // Returns the pooled Hadoop decompressors after each row group; getDecompressor re-creates diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 8e8a917f20b9..c0551e1f8819 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -261,8 +261,8 @@ public OrcEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager buff } consumer.setUseDecimal64ColumnVectors(HiveConf.getVar(jobConf, - ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64)); + ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64)); consumer.setFileMetadata(fileMetadata); consumer.setSchemaEvolution(evolution); isReadCacheOnly = HiveConf.getBoolVar(jobConf, ConfVars.LLAP_IO_CACHE_ONLY); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java index bb950817dbcb..4438121464ed 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedColumnBatch.java @@ -27,16 +27,16 @@ * One row-group's worth of cached Parquet column-chunk buffers, handed from the reader to the * consumer. Extends {@link EncodedColumnBatch} only to satisfy the {@code EncodedDataConsumer} * generic bound; the inherited ColumnStreamData machinery is ORC-shaped and unused here. The real - * payload is in the added fields. + * payload is in the fields exposed through the accessors below. */ -public class ParquetEncodedColumnBatch extends EncodedColumnBatch { +public final class ParquetEncodedColumnBatch extends EncodedColumnBatch { - public int rowGroupIx; + private int rowGroupIx; /** This row group's chunk per projected column; the arrays below are indexed the same way. */ - public ColumnChunkMetaData[] chunks; - public MemoryBuffer[][] columnBuffers; - public long[][] bufferOffsets; - public int[][] bufferLengths; + private ColumnChunkMetaData[] chunks; + private MemoryBuffer[][] columnBuffers; + private long[][] bufferOffsets; + private int[][] bufferLengths; public ParquetEncodedColumnBatch() { // No-arg constructor for pooling / reflection-based construction; fields are populated later @@ -54,4 +54,24 @@ public void init(Object fileKey, int rowGroupIx, ColumnChunkMetaData[] chunks) { this.bufferOffsets = new long[n][]; this.bufferLengths = new int[n][]; } + + public int rowGroupIx() { + return rowGroupIx; + } + + public ColumnChunkMetaData[] chunks() { + return chunks; + } + + public MemoryBuffer[][] columnBuffers() { + return columnBuffers; + } + + public long[][] bufferOffsets() { + return bufferOffsets; + } + + public int[][] bufferLengths() { + return bufferLengths; + } } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java index bbb995c9e9f6..7231ebe1e0f7 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/ParquetEncodedDataReader.java @@ -337,7 +337,7 @@ private static long bytes(int[] projected, BlockMetaData block) { private static long bytes(Fetch fetch) { long total = 0; - for (ColumnChunkMetaData chunk : fetch.batch.chunks) { + for (ColumnChunkMetaData chunk : fetch.batch.chunks()) { total += chunk.getTotalSize(); } return total; @@ -589,8 +589,8 @@ private void putColumn(Allocator allocator, ColumnPlan column) { // raw allocations that cleanup can safely deallocate. for (Part part : parts) { MemoryBuffer fresh = part.buffer; - MemoryBuffer[] pair = new MemoryBuffer[] { fresh }; - DiskRange[] range = new DiskRange[] { part.range }; + MemoryBuffer[] pair = new MemoryBuffer[] {fresh}; + DiskRange[] range = new DiskRange[] {part.range}; lowLevelCache.putFileData(fileKey, range, pair, 0, Priority.NORMAL, counters, cacheTag); if (pair[0] != fresh) { // The cache kept its own buffer (locked for us) and unlocked ours without freeing it. @@ -604,20 +604,23 @@ private void putColumn(Allocator allocator, ColumnPlan column) { private static void assemble(ParquetEncodedColumnBatch batch, int pc, List parts) { int n = parts.size(); - batch.columnBuffers[pc] = new MemoryBuffer[n]; - batch.bufferOffsets[pc] = new long[n]; - batch.bufferLengths[pc] = new int[n]; + MemoryBuffer[] columnBuffers = new MemoryBuffer[n]; + long[] bufferOffsets = new long[n]; + int[] bufferLengths = new int[n]; + batch.columnBuffers()[pc] = columnBuffers; + batch.bufferOffsets()[pc] = bufferOffsets; + batch.bufferLengths()[pc] = bufferLengths; for (int i = 0; i < n; ++i) { Part part = parts.get(i); - batch.columnBuffers[pc][i] = part.buffer; - batch.bufferOffsets[pc][i] = part.range.getOffset(); - batch.bufferLengths[pc][i] = part.range.getLength(); + columnBuffers[i] = part.buffer; + bufferOffsets[i] = part.range.getOffset(); + bufferLengths[i] = part.range.getLength(); } } @Override public void returnData(ParquetEncodedColumnBatch batch) { - for (MemoryBuffer[] column : batch.columnBuffers) { + for (MemoryBuffer[] column : batch.columnBuffers()) { for (MemoryBuffer buffer : column) { bufferManager.decRefBuffer(buffer); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java index 0b454d002584..16b2228c6b89 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java @@ -232,8 +232,8 @@ public MemoryBuffer create() { this.reporter = reporter; this.jobConf = jobConf; final boolean useDecimal64ColumnVectors = HiveConf.getVar(jobConf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); consumer.setUseDecimal64ColumnVectors(useDecimal64ColumnVectors); this.schema = schema; this.writerIncludes = OrcInputFormat.genIncludedColumns(schema, columnIds); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java index 4181118daca0..60c5a72dbe30 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java @@ -235,8 +235,8 @@ public void startAsync(AsyncCallback callback) { private static VectorizedRowBatchCtx createVrbCtx(StructObjectInspector oi, final Properties tblProps, final Configuration conf) throws IOException { final boolean useDecimal64ColumnVectors = HiveConf.getVar(conf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); final String serde = tblProps.getProperty(serdeConstants.SERIALIZATION_LIB); final String inputFormat = tblProps.getProperty(hive_metastoreConstants.FILE_INPUT_FORMAT); final boolean isTextFormat = TextInputFormat.class.getName().equals(inputFormat) diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java index bc9c9eb708e2..83d59b9fb602 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestParquetEncodedDataReader.java @@ -628,7 +628,7 @@ public void testProjectedLeavesSkipsPrecedingNestedGroup() { int[] leaves = ParquetEncodedDataReader.projectedLeaves(requested, fileSchema); // x sits at leaf index 2 (nested.a=0, nested.b=1, x=2), not at its top-level ordinal 1. - assertArrayEquals(new int[] { 2 }, leaves); + assertArrayEquals(new int[] {2}, leaves); } @Test @@ -645,7 +645,7 @@ public void testProjectedLeavesMissingFieldSkipped() { .named("requested"); // Only a is present in the file schema (leaf 0); "missing" contributes nothing. - assertArrayEquals(new int[] { 0 }, + assertArrayEquals(new int[] {0}, ParquetEncodedDataReader.projectedLeaves(requested, fileSchema)); } @@ -766,7 +766,7 @@ private Run read(JobConf job, FileSplit split, Configuration readerDaemonConf, @Override protected void decodeBatch(ParquetEncodedColumnBatch batch, Consumer consumer) throws InterruptedException { - for (MemoryBuffer[] col : batch.columnBuffers) { + for (MemoryBuffer[] col : batch.columnBuffers()) { Collections.addAll(buffers, col); } if (failDecode) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 25d00b315c16..97f3037e1c8d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -68,8 +68,8 @@ protected RecordReaderImpl(ReaderImpl fileReader, Reader.Options options, final Configuration conf) throws IOException { super(fileReader, options); final boolean useDecimal64ColumnVectors = conf != null && HiveConf.getVar(conf, - HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); + HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors){ batch = this.schema.createRowBatchV2(); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index 20d2c07e3df4..970947ea58af 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -212,8 +212,8 @@ public float getProgress() throws IOException { } }; final boolean useDecimal64ColumnVectors = HiveConf - .getVar(conf, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); + .getVar(conf, ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVectors) { this.vectorizedRowBatchBase = ((RecordReaderImpl) innerReader).createRowBatch(true); } else { @@ -1515,8 +1515,8 @@ static class DeleteReaderValue { this.bucketForSplit = bucket; final boolean useDecimal64ColumnVector = HiveConf.getVar(conf, ConfVars - .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) - .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); + .HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED) + .equalsIgnoreCase(HiveConf.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_DECIMAL_64); if (useDecimal64ColumnVector) { this.batch = acidEmptyStructSchema.createRowBatchV2(); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java index b62f8592b25f..2f85094769bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetRowGroupDecoder.java @@ -27,6 +27,7 @@ import java.util.Optional; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.parquet.ParquetRuntimeException; @@ -52,6 +53,17 @@ public class ParquetRowGroupDecoder { private static final int MAP_DEFINITION_LEVEL_MAX = 3; + /** + * Bag of writer-timezone / proleptic / legacy conversion flags forwarded to every primitive + * column reader. Grouped so the reader-construction entry point stays under the parameter cap. + */ + public record TimestampConversionOptions( + boolean skipTimestampConversion, + ZoneId writerTimezone, + boolean skipProlepticConversion, + boolean legacyConversionEnabled) { + } + private final MessageType fileSchema; private final Map initialDefaults; @@ -63,18 +75,15 @@ public ParquetRowGroupDecoder(MessageType fileSchema, Map initia /** * Builds the per-(requested-)column {@link VectorizedColumnReader} array for a row group. * - * @param pages the row group's page store (from {@code reader.readRowGroup(..)} - * / {@code readNextRowGroup()}) - * @param requestedSchema the projected Parquet schema being read - * @param columnTypesList the Hive type infos for ALL table columns (indexed by table col id) - * @param colsToInclude the table column ids being read, in requested-schema field order; - * may be empty (e.g. {@code count(*)}), in which case all readers are - * null - * @param readAllColumns whether projection is "read all columns" - * @param skipTimestampConversion see {@code VectorizedParquetRecordReader} - * @param writerTimezone see {@code VectorizedParquetRecordReader} - * @param skipProlepticConversion see {@code VectorizedParquetRecordReader} - * @param legacyConversionEnabled see {@code VectorizedParquetRecordReader} + * @param pages the row group's page store (from {@code reader.readRowGroup(..)} + * / {@code readNextRowGroup()}) + * @param requestedSchema the projected Parquet schema being read + * @param columnTypesList the Hive type infos for ALL table columns (indexed by table col id) + * @param colsToInclude the table column ids being read, in requested-schema field order; + * may be empty (e.g. {@code count(*)}), in which case all readers are null + * @param readAllColumns whether projection is "read all columns" + * @param options writer-timezone / proleptic / legacy conversion flags, see + * {@link TimestampConversionOptions} * @return one reader per requested-schema field (null entries where no reader is needed) */ public VectorizedColumnReader[] buildColumnReaders( @@ -83,10 +92,7 @@ public VectorizedColumnReader[] buildColumnReaders( List columnTypesList, List colsToInclude, boolean readAllColumns, - boolean skipTimestampConversion, - ZoneId writerTimezone, - boolean skipProlepticConversion, - boolean legacyConversionEnabled) throws IOException { + TimestampConversionOptions options) throws IOException { List types = requestedSchema.getFields(); VectorizedColumnReader[] columnReaders = new VectorizedColumnReader[types.size()]; @@ -99,15 +105,13 @@ public VectorizedColumnReader[] buildColumnReaders( for (int i = 0; i < types.size(); ++i) { columnReaders[i] = buildVectorizedParquetReader( columnTypesList.get(colsToInclude.get(i)), types.get(i), pages, - requestedSchema.getColumns(), skipTimestampConversion, writerTimezone, - skipProlepticConversion, legacyConversionEnabled, 0, 0); + requestedSchema.getColumns(), options, 0, 0); } } } else { for (int i = 0; i < types.size(); ++i) { columnReaders[i] = buildVectorizedParquetReader(columnTypesList.get(i), - types.get(i), pages, requestedSchema.getColumns(), skipTimestampConversion, - writerTimezone, skipProlepticConversion, legacyConversionEnabled, 0, 0); + types.get(i), pages, requestedSchema.getColumns(), options, 0, 0); } } return columnReaders; @@ -129,13 +133,14 @@ private static List getAllColumnDescriptorByType( return res; } - // TODO support only non nested case + // Nested types are unsupported here on purpose; callers detect that in advance + // (ParquetEncodedDataReader.projectsNestedTypes) and route the query to the non-native reader. private static PrimitiveType getElementType(Type type) { if (type.isPrimitive()) { return type.asPrimitiveType(); } if (type.asGroupType().getFields().size() > 1) { - throw new RuntimeException( + throw new UnsupportedOperationException( "Current Parquet Vectorization reader doesn't support nested type"); } @@ -155,10 +160,7 @@ private VectorizedColumnReader buildVectorizedParquetReader( Type type, PageReadStore pages, List columnDescriptors, - boolean skipTimestampConversion, - ZoneId writerTimezone, - boolean skipProlepticConversion, - boolean legacyConversionEnabled, + TimestampConversionOptions options, int depth, int currentDefLevel) throws IOException { int typeDefLevel = currentDefLevel; if (type.isRepetition(Type.Repetition.OPTIONAL) || type.isRepetition(Type.Repetition.REPEATED)) { @@ -177,12 +179,13 @@ private VectorizedColumnReader buildVectorizedParquetReader( switch (typeInfo.getCategory()) { case PRIMITIVE: if (columnDescriptors == null || columnDescriptors.isEmpty()) { - throw new RuntimeException( + throw new InvalidSchemaException( "Failed to find related Parquet column descriptor with type " + type); } return new VectorizedPrimitiveColumnReader(descriptors.get(0), - pages.getPageReader(descriptors.get(0)), skipTimestampConversion, writerTimezone, - skipProlepticConversion, legacyConversionEnabled, type, typeInfo); + pages.getPageReader(descriptors.get(0)), options.skipTimestampConversion(), + options.writerTimezone(), options.skipProlepticConversion(), + options.legacyConversionEnabled(), type, typeInfo); case STRUCT: StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List fieldReaders = new ArrayList<>(); @@ -191,12 +194,11 @@ private VectorizedColumnReader buildVectorizedParquetReader( for (int i = 0; i < fieldTypes.size(); i++) { VectorizedColumnReader r = buildVectorizedParquetReader(fieldTypes.get(i), types.get(i), pages, - descriptors, skipTimestampConversion, writerTimezone, skipProlepticConversion, - legacyConversionEnabled, depth + 1, typeDefLevel); + descriptors, options, depth + 1, typeDefLevel); if (r != null) { fieldReaders.add(r); } else { - throw new RuntimeException( + throw new IllegalStateException( "Fail to build Parquet vectorized reader based on Hive type " + fieldTypes.get(i) .getTypeName() + " and Parquet type" + types.get(i).toString()); } @@ -205,52 +207,49 @@ private VectorizedColumnReader buildVectorizedParquetReader( case LIST: checkListColumnSupport(((ListTypeInfo) typeInfo).getListElementTypeInfo()); if (columnDescriptors == null || columnDescriptors.isEmpty()) { - throw new RuntimeException( + throw new InvalidSchemaException( "Failed to find related Parquet column descriptor with type " + type); } return new VectorizedListColumnReader(descriptors.get(0), - pages.getPageReader(descriptors.get(0)), skipTimestampConversion, writerTimezone, - skipProlepticConversion, legacyConversionEnabled, getElementType(type), typeInfo); + pages.getPageReader(descriptors.get(0)), options.skipTimestampConversion(), + options.writerTimezone(), options.skipProlepticConversion(), + options.legacyConversionEnabled(), getElementType(type), typeInfo); case MAP: if (columnDescriptors == null || columnDescriptors.isEmpty()) { - throw new RuntimeException( + throw new InvalidSchemaException( "Failed to find related Parquet column descriptor with type " + type); } - // to handle the different Map definition in Parquet, eg: - // definition has 1 group: - // repeated group map (MAP_KEY_VALUE) - // {required binary key (UTF8); optional binary value (UTF8);} - // definition has 2 groups: - // optional group m1 (MAP) { - // repeated group map (MAP_KEY_VALUE) - // {required binary key (UTF8); optional binary value (UTF8);} - // } + // Parquet has more than one on-disk shape for MAP; walk down until we find the + // {key, value} group, tolerating up to MAP_DEFINITION_LEVEL_MAX wrapping groups. + // See parquet-format's LogicalTypes spec for the MAP annotation for the shapes. int nestGroup = 0; GroupType groupType = type.asGroupType(); // if FieldCount == 2, get types for key & value, // otherwise, continue to get the group type until MAP_DEFINITION_LEVEL_MAX. while (groupType.getFieldCount() < 2) { if (nestGroup > MAP_DEFINITION_LEVEL_MAX) { - throw new RuntimeException( - "More than " + MAP_DEFINITION_LEVEL_MAX + " level is found in Map definition, " + - "Failed to get the field types for Map with type " + type); + throw new InvalidSchemaException( + "More than " + MAP_DEFINITION_LEVEL_MAX + " level is found in Map definition, " + + "Failed to get the field types for Map with type " + type); } groupType = groupType.getFields().get(0).asGroupType(); nestGroup++; } List kvTypes = groupType.getFields(); VectorizedListColumnReader keyListColumnReader = new VectorizedListColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0)), skipTimestampConversion, - writerTimezone, skipProlepticConversion, legacyConversionEnabled, kvTypes.get(0), typeInfo); + descriptors.get(0), pages.getPageReader(descriptors.get(0)), options.skipTimestampConversion(), + options.writerTimezone(), options.skipProlepticConversion(), + options.legacyConversionEnabled(), kvTypes.get(0), typeInfo); VectorizedListColumnReader valueListColumnReader = new VectorizedListColumnReader( - descriptors.get(1), pages.getPageReader(descriptors.get(1)), skipTimestampConversion, - writerTimezone, skipProlepticConversion, legacyConversionEnabled, kvTypes.get(1), typeInfo); + descriptors.get(1), pages.getPageReader(descriptors.get(1)), options.skipTimestampConversion(), + options.writerTimezone(), options.skipProlepticConversion(), + options.legacyConversionEnabled(), kvTypes.get(1), typeInfo); return new VectorizedMapColumnReader(keyListColumnReader, valueListColumnReader); case UNION: default: - throw new RuntimeException("Unsupported category " + typeInfo.getCategory().name()); + throw new UnsupportedOperationException("Unsupported category " + typeInfo.getCategory().name()); } } @@ -260,17 +259,15 @@ private VectorizedColumnReader buildVectorizedParquetReader( * STRING, CHAR, VARCHAR, FLOAT, DECIMAL */ private static void checkListColumnSupport(TypeInfo elementType) { - if (elementType instanceof org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) { - switch (((org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) elementType) - .getPrimitiveCategory()) { - case INTERVAL_DAY_TIME: - case TIMESTAMP: - throw new RuntimeException("Unsupported primitive type used in list:: " + elementType); + if (elementType instanceof PrimitiveTypeInfo primitiveTypeInfo) { + switch (primitiveTypeInfo.getPrimitiveCategory()) { + case INTERVAL_DAY_TIME, TIMESTAMP: + throw new UnsupportedOperationException("Unsupported primitive type used in list: " + elementType); default: // supported } } else { - throw new RuntimeException("Unsupported type used in list:" + elementType); + throw new UnsupportedOperationException("Unsupported type used in list: " + elementType); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java index d6bdc3826695..c7f6041123e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java @@ -443,8 +443,9 @@ private void checkEndOfRowGroup() throws IOException { // LLAP cache-backed consumer can reuse the exact same logic. Behavior is unchanged. columnReaders = new ParquetRowGroupDecoder(fileSchema, initialDefaults).buildColumnReaders( pages, requestedSchema, columnTypesList, colsToInclude, - ColumnProjectionUtils.isReadAllColumns(jobConf), skipTimestampConversion, writerTimezone, - skipProlepticConversion, legacyConversionEnabled); + ColumnProjectionUtils.isReadAllColumns(jobConf), + new ParquetRowGroupDecoder.TimestampConversionOptions(skipTimestampConversion, writerTimezone, + skipProlepticConversion, legacyConversionEnabled)); currentRowNumInRowGroup = 0; currentRowGroupIndex++; diff --git a/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java index 76a289d556a0..60098661f01e 100644 --- a/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java +++ b/ql/src/test/org/apache/hadoop/hive/llap/TestParquetRangeBuffers.java @@ -23,7 +23,6 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; -import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -36,7 +35,7 @@ public class TestParquetRangeBuffers { @Test - public void anExclusiveStreamsBuffersAreReusedAndLimitedToTheRange() throws IOException { + public void anExclusiveStreamsBuffersAreReusedAndLimitedToTheRange() { ParquetRangeBuffers buffers = ParquetRangeBuffers.forStream(stream(false)); ByteBuffer first = buffers.allocate(100); assertEquals(100, first.limit()); @@ -50,7 +49,7 @@ public void anExclusiveStreamsBuffersAreReusedAndLimitedToTheRange() throws IOEx } @Test - public void aSlicingStreamsBuffersAreNeverPooled() throws IOException { + public void aSlicingStreamsBuffersAreNeverPooled() { ParquetRangeBuffers buffers = ParquetRangeBuffers.forStream(stream(true)); ByteBuffer first = buffers.allocate(100); buffers.release(first);