From a4c43b3cd3ffe448a914e1f90d47d151473fe0aa Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Fri, 2 Oct 2026 15:46:57 +0800 Subject: [PATCH 01/16] Add opt-in physical plan cache with HBase and Iceberg support --- PLAN_CACHE_DESIGN.md | 104 +++ PLAN_CACHE_PLUGIN_GUIDE.md | 121 ++++ .../exec/store/iceberg/IcebergGroupScan.java | 5 + .../iceberg/format/IcebergFormatPlugin.java | 32 + .../plan/DrillExprToIcebergTranslator.java | 50 +- .../plan/IcebergPluginImplementor.java | 7 + .../exec/store/hbase/HBaseGroupScan.java | 65 +- .../store/hbase/HBasePushFilterIntoScan.java | 3 +- .../exec/store/hbase/HBaseStoragePlugin.java | 31 + .../apache/drill/exec/ops/QueryContext.java | 20 + .../drill/exec/physical/base/GroupScan.java | 15 + .../physical/config/HashToMergeExchange.java | 10 +- .../physical/config/OrderedMuxExchange.java | 7 + .../exec/planner/PhysicalPlanReader.java | 4 + .../drill/exec/planner/index/ExprToRex.java | 11 +- .../exec/planner/logical/DrillOptiq.java | 11 + .../exec/planner/physical/DirectScanPrel.java | 2 +- .../planner/physical/PlannerSettings.java | 3 + .../drill/exec/planner/physical/ScanPrel.java | 2 +- .../exec/planner/sql/DrillSqlWorker.java | 115 ++- .../drill/exec/planner/sql/PlanCache.java | 623 ++++++++++++++++ .../planner/sql/PlanCacheEligibility.java | 136 ++++ .../planner/sql/PlanCacheParameterizer.java | 197 ++++++ .../conversion/DrillSqlToRelConverter.java | 61 ++ .../sql/conversion/DrillSqlValidator.java | 68 ++ .../sql/conversion/RexBoundDynamicParam.java | 36 + .../sql/conversion/SqlBoundDynamicParam.java | 41 ++ .../planner/sql/conversion/SqlConverter.java | 7 +- .../drill/exec/rpc/user/UserSession.java | 5 + .../drill/exec/server/DrillbitContext.java | 8 + .../server/options/SystemOptionManager.java | 1 + .../server/rest/profile/ProfileWrapper.java | 7 + .../drill/exec/store/PlanCacheTable.java | 39 + .../drill/exec/store/StoragePlugin.java | 25 + .../exec/store/dfs/FileSystemPlugin.java | 52 ++ .../drill/exec/store/dfs/FormatPlugin.java | 15 + .../drill/exec/store/plan/rel/PluginPrel.java | 2 +- .../drill/exec/work/foreman/Foreman.java | 5 + .../drill/exec/work/foreman/QueryManager.java | 2 + .../src/main/resources/drill-module.conf | 1 + .../main/resources/rest/profile/profile.ftl | 2 + .../common/expression/parser/ExprLexer.g4 | 2 + .../common/expression/parser/ExprParser.g4 | 14 +- .../expression/ExpressionStringBuilder.java | 125 +++- .../common/expression/LiteralExpression.java | 41 ++ .../common/expression/TypedNullConstant.java | 2 +- .../common/expression/ValueExpressions.java | 73 +- .../drill/exec/proto/SchemaUserBitShared.java | 7 + .../drill/exec/proto/UserBitShared.java | 666 ++++++++++-------- .../src/main/protobuf/UserBitShared.proto | 1 + 50 files changed, 2543 insertions(+), 339 deletions(-) create mode 100644 PLAN_CACHE_DESIGN.md create mode 100644 PLAN_CACHE_PLUGIN_GUIDE.md create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheEligibility.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlToRelConverter.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlValidator.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/RexBoundDynamicParam.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlBoundDynamicParam.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/PlanCacheTable.java create mode 100644 logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java diff --git a/PLAN_CACHE_DESIGN.md b/PLAN_CACHE_DESIGN.md new file mode 100644 index 00000000000..c5b3462feea --- /dev/null +++ b/PLAN_CACHE_DESIGN.md @@ -0,0 +1,104 @@ + + +# Physical plan cache + +Drill normally validates and optimizes every SQL query, even when the same query +shape has just been executed. The plan cache reuses that work for repeated reads. +It caches a physical plan, not query results: every execution reads the current +source data. + +The feature is opt-in and disabled by default: + +```sql +ALTER SESSION SET planner.enable_plan_cache = true; +``` + +Use `ALTER SYSTEM` instead of `ALTER SESSION` to set the default for new queries. + +## The basic idea + +These queries can share a plan when the scan plugin supports it: + +```sql +SELECT row_key FROM hbase.usertable WHERE row_key = 'user00000101'; +SELECT row_key FROM hbase.usertable WHERE row_key = 'user00000250'; +``` + +Drill replaces eligible data literals with typed slots when building the cache +key. Each slot carries its current value during the first planning pass. The +resulting physical expressions retain the slot number, so a later query can +replace the value without running the optimizer again. Literal type, numeric +precision and scale, and string length are part of the template. + +```mermaid +flowchart LR + SQL[Parse SQL and build template] --> Lookup{Compatible cached plan?} + Lookup -->|Yes| Bind[Bind current values and rebuild scans] + Lookup -->|No| Plan[Validate and optimize normally] + Bind --> Execute[Execute against current data] + Plan --> Execute + Execute -->|Successful first execution| Save[Publish immutable plan snapshot] +``` + +The cache belongs to one Drillbit and can be shared across connections. Its key +includes the query user, default schema and SQL template. Each entry also records +effective planner options, plugin configurations and the compatibility version +of every referenced table, including tables inside joins, CTEs and subqueries. +A mismatch or failure falls back to ordinary planning. + +The cached representation is immutable physical-plan JSON. A hit creates a new +plan object graph and binds the new values throughout its expressions. Each +participating scan must also rebuild any state derived from those values. +HBase rebuilds row-key bounds and remote filters, then discovers current regions. +Iceberg reloads table metadata and plans tasks for the current snapshot and +predicate. Reusing the first execution's filters or file list would be incorrect. + +## What can be reused + +Read queries can include joins, aggregates, subqueries, CTEs, ordering and limits. +Every `GroupScan` in a plan must explicitly support caching. This change opts in +HBase and Iceberg; other plugins retain the default of no support. + +Constants that determine SQL structure stay in the key: group/order expressions, +window definitions, limit/offset, field selectors, and function configuration +arguments such as `DATE_PART`'s unit or `CONVERT_FROM`'s encoding. DATE literals +also stay in the key because DATE parameter binding is not supported. Other +eligible literals in the same query can still be rebound. Queries with no slots +can reuse their complete template. + +Writes, DDL, existing dynamic parameters, volatile/query-context functions and +sessions containing temporary tables or mutable aliases bypass the cache. +Unsupported scans, views without a compatible table identity, Iceberg metadata tables and explicit +snapshot selections use normal planning. + +## Lifetime and observation + +An entry is published only after the first query succeeds. A bounded background +queue performs serialization and read-back validation; a full queue simply skips +publication. The cache is bounded by serialized-plan weight and entries expire +after ten minutes. Cached row-count estimates come from the first optimization; +new values do not trigger a new cost-based plan choice. + +The query profile's **Plan Cache Hit** field, or JSON `planCacheHit`, identifies a +successful hit. Explain shows slot names and this execution's parameter values; +those alone do not prove a hit. + +See the [plugin developer guide](PLAN_CACHE_PLUGIN_GUIDE.md) for the integration +contract. The accompanying pull request description contains validation results +and benchmark methodology. diff --git a/PLAN_CACHE_PLUGIN_GUIDE.md b/PLAN_CACHE_PLUGIN_GUIDE.md new file mode 100644 index 00000000000..42528923066 --- /dev/null +++ b/PLAN_CACHE_PLUGIN_GUIDE.md @@ -0,0 +1,121 @@ + + +# Adding plan-cache support to a plugin + +Plan caching is disabled by default. A plugin is eligible only when both its +metadata interface and every participating `GroupScan` opt in. The engine binds +SQL literals in serialized Drill expressions; the plugin remains responsible for +rebuilding scan state derived from those expressions. + +## Storage-plugin API + +Implement these methods in [StoragePlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java): + +```java +boolean supportPlanCache(); +PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException; +String planCacheTableVersion(String identifier) throws IOException; +``` + +`planCacheTable` is called during a miss. Return a stable, opaque physical table +identifier and a compatibility version. Return `null` for unsupported selections. +`planCacheTableVersion` is called on a hit with that same identifier; it should +read the current version without rebuilding the SQL schema. Return `null` if the +table is missing or unsupported. Throwing also causes a fallback to normal +planning. The engine separately compares the storage-plugin configuration. + +The version must change whenever the cached plan's table definition becomes +incompatible. Do not use a data snapshot number if compatible data updates should +keep hitting. A path alone is insufficient when replacement can change the +schema or table identity. + +For a DFS format, implement `supportPlanCache()` and both +`planCacheTableVersion(FileSelection)` and `planCacheTableVersion(Path)` in +[FormatPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java). +[FileSystemPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java) +provides the storage-plugin bridge. The selection overload validates the original +selection; the path overload validates a hit without rediscovering the schema. + +## Scan API and JSON reconstruction + +Override `GroupScan.supportPlanCache()`; its default is `false`. The engine rejects +a plan containing even one unsupported scan. + +Keep the bound `LogicalExpression` in the scan's JSON. Converting a predicate to +a native filter or file list and dropping the source expression loses the +information needed to bind the next query. On deserialization: + +1. Resolve the plugin and reload current table metadata. +2. Translate the newly bound expression into the plugin's native predicate. +3. Recompute value-dependent row bounds, filters, files/splits, region locations + and assignments. Preserve residual filtering for unsupported predicates. +4. Reject reconstruction if the original pushdown guarantees cannot be met. + +Serialize stable inputs rather than live connections, execution resources or old +endpoint assignments. Validate the round trip with Drill's `PhysicalPlanReader`. +If a parameter changes projection, schema, limit or another derived field that +cannot be rebuilt, keep that argument structural or leave the scan ineligible. + +For explain output containing expressions, use +`ExpressionStringBuilder.toExplainString()` via `getExplainDigest()` to display +`?index`. Continue using the regular expression serialization for JSON so slot +numbers survive the round trip. + +## Reference implementations + +| Plugin | Compatibility version | Reconstruction | +| --- | --- | --- | +| [HBaseStoragePlugin](contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java) / [HBaseGroupScan](contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java) | Table descriptor fingerprint | Retains the selection before pushdown and the original predicate; rebuilds native row-key bounds/filters and discovers current regions. | +| [IcebergFormatPlugin](contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java) / [IcebergGroupScan](contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java) | Table UUID and schema ID | Reloads the table and runs `planTasks()` for the current predicate and snapshot. Metadata tables and explicit snapshot selection return no support. | + +HBase descriptor equality means definition compatibility; a replacement with an +identical descriptor can still be read safely because regions and filters are +rebuilt. Iceberg uses a UUID to distinguish tables recreated at the same path. + +## Required correctness checks + +Compare complete results with caching disabled, after the first planning pass, +and after a hit with different literals. Exercise cross-connection reuse, +pushdown and residual filtering, and an empty scan after rebinding. Verify that +compatible data updates are visible on hits, while incompatible schema changes, +deletions, plugin configuration changes and failed metadata reads reject the old +entry. Include joins or subqueries so every table dependency is checked, and +confirm that a mixed plan containing an unsupported scan is rejected. + +Check `planCacheHit` in the query profile. In isolated tests, the Drillbit's hit +counter can also establish reuse: + +```java +PlanCache cache = cluster.drillbit().getContext().getPlanCache(); +first.queryBuilder().sql(firstSql).run(); +cache.awaitWrites(); // Test synchronization for asynchronous publication. +long before = cache.getHitCount(); +second.queryBuilder().sql(sameTemplateWithNewValues).run(); +assertEquals(before + 1, cache.getHitCount()); +``` + +The counter is shared across the Drillbit; use per-query profiles for concurrent +workloads. `awaitWrites()` is a test helper, not a SQL operation. + +## Scope of the contract + +The cache validates saved physical identifiers. Sessions with temporary tables +or mutable user/public aliases bypass caching, because those mappings can redirect +a SQL name without changing the old physical table. Statistics changes alone do +not trigger re-optimization. diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java index 4beedd4fd0d..b67e2a3b92f 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java @@ -184,6 +184,11 @@ public IcebergGroupScan clone(List columns) { } } + @Override + public boolean supportPlanCache() { + return formatPlugin.supportPlanCache() && path.indexOf('#') < 0; + } + @Override public IcebergGroupScan applyLimit(int maxRecords) { IcebergGroupScan clone = new IcebergGroupScan(this); diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java index 36c51c09d53..e7c872452bd 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java @@ -41,6 +41,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.hadoop.HadoopTables; import java.io.IOException; import java.util.Collections; @@ -104,6 +108,34 @@ public boolean supportsRead() { return true; } + @Override + public boolean supportPlanCache() { + return config.getSnapshot() == null; + } + + @Override + public String planCacheTableVersion(FileSelection selection) throws IOException { + if (!supportPlanCache() || selection instanceof IcebergMetadataFileSelection + || selection.getSelectionRoot() == null) { + return null; + } + return planCacheTableVersion(selection.getSelectionRoot()); + } + + @Override + public String planCacheTableVersion(Path tablePath) throws IOException { + if (!supportPlanCache()) { + return null; + } + Table table = new HadoopTables(fsConf).load(tablePath.toString()); + if (!(table instanceof HasTableOperations)) { + return null; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + return metadata == null || metadata.uuid() == null ? null + : metadata.uuid() + ":" + metadata.currentSchemaId(); + } + @Override public boolean supportsWrite() { return false; diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java index 3449935e4a1..4eeb8a9502d 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/DrillExprToIcebergTranslator.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.store.iceberg.plan; import org.apache.drill.common.FunctionNames; +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; @@ -32,6 +34,46 @@ public class DrillExprToIcebergTranslator extends AbstractExprVisitor INSTANCE = new DrillExprToIcebergTranslator(); + @Override + public Expression visitBooleanOperator(BooleanOperator op, Void value) { + Expression result = null; + for (LogicalExpression argument : op.args()) { + Expression next = argument.accept(this, value); + if (next == null) { + return null; + } + result = result == null ? next : FunctionNames.AND.equals(op.getName()) + ? Expressions.and(result, next) : Expressions.or(result, next); + } + return result; + } + + @Override + public Expression visitCastExpression(CastExpression cast, Void value) { + Expression input = cast.getInput().accept(this, value); + if (!(input instanceof ConstantExpression)) { + return null; + } + Object literal = ((ConstantExpression) input).getValue(); + if (!(literal instanceof Integer) && !(literal instanceof Long)) { + return null; + } + long millis = ((Number) literal).longValue(); + // Drill serializes temporal constants as casts of epoch milliseconds. + // Iceberg's DATE uses days; TIME and TIMESTAMP use microseconds. + switch (cast.getMajorType().getMinorType()) { + case BIGINT: + return new ConstantExpression<>(millis); + case DATE: + return new ConstantExpression<>(Math.toIntExact(Math.floorDiv(millis, 86_400_000L))); + case TIME: + case TIMESTAMP: + return new ConstantExpression<>(Math.multiplyExact(millis, 1_000L)); + default: + return null; + } + } + @Override public Expression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { switch (call.getName()) { @@ -103,7 +145,7 @@ public Expression visitFunctionCall(FunctionCall call, Void value) throws Runtim } case FunctionNames.GE: { LogicalExpression nameRef = call.args().get(0); - Expression expression = call.args().get(0).accept(this, null); + Expression expression = call.args().get(1).accept(this, null); if (nameRef instanceof SchemaPath && expression instanceof ConstantExpression) { String name = IcebergGroupScan.getPath((SchemaPath) nameRef); return Expressions.greaterThanOrEqual(name, ((ConstantExpression) expression).getValue()); @@ -174,17 +216,17 @@ public Expression visitVarDecimalConstant(ValueExpressions.VarDecimalExpression @Override public Expression visitDateConstant(ValueExpressions.DateExpression dateExpr, Void value) throws RuntimeException { - return new ConstantExpression<>(dateExpr.getDate()); + return new ConstantExpression<>(Math.toIntExact(Math.floorDiv(dateExpr.getDate(), 86_400_000L))); } @Override public Expression visitTimeConstant(ValueExpressions.TimeExpression timeExpr, Void value) throws RuntimeException { - return new ConstantExpression<>(timeExpr.getTime()); + return new ConstantExpression<>((long) timeExpr.getTime() * 1_000L); } @Override public Expression visitTimeStampConstant(ValueExpressions.TimeStampExpression timestampExpr, Void value) throws RuntimeException { - return new ConstantExpression<>(timestampExpr.getTimeStamp()); + return new ConstantExpression<>(Math.multiplyExact(timestampExpr.getTimeStamp(), 1_000L)); } @Override diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java index a4863273d89..2606d44b42f 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/plan/IcebergPluginImplementor.java @@ -25,6 +25,8 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; +import org.apache.drill.common.FunctionNames; +import org.apache.drill.common.expression.FunctionCallFactory; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.base.GroupScan; @@ -46,6 +48,7 @@ import java.io.IOException; import java.math.BigDecimal; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -67,6 +70,10 @@ public void implement(PluginFilterRel filter) throws IOException { new DrillParseContext(PrelUtil.getPlannerSettings(filter.getCluster().getPlanner())), filter.getInput(), condition); + if (groupScan.getCondition() != null) { + expression = FunctionCallFactory.createBooleanOperator(FunctionNames.AND, + Arrays.asList(groupScan.getCondition(), expression)); + } groupScan = groupScan.toBuilder().condition(expression).build(); } diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java index d4867da24ca..99ac66a49dd 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java @@ -39,6 +39,9 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.parser.LogicalExpressionParser; +import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.exec.physical.EndpointAffinity; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.GroupScan; @@ -84,6 +87,11 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst private HBaseScanSpec hbaseScanSpec; + /** Selection before predicate pushdown, needed to rebuild value-dependent HBase filters. */ + private HBaseScanSpec baseScanSpec; + private LogicalExpression pushedFilter; + private boolean pushedFilterFullyConverted; + private HBaseStoragePlugin storagePlugin; private final Stopwatch watch = Stopwatch.createUnstarted(); @@ -108,18 +116,44 @@ public HBaseGroupScan(@JsonProperty("userName") String userName, @JsonProperty("storage") HBaseStoragePluginConfig storagePluginConfig, @JsonProperty("columns") List columns, @JsonProperty("maxRecords") int maxRecords, + @JsonProperty("baseScanSpec") HBaseScanSpec baseScanSpec, + @JsonProperty("pushedFilter") String pushedFilter, + @JsonProperty("pushedFilterFullyConverted") boolean pushedFilterFullyConverted, @JacksonInject StoragePluginRegistry pluginRegistry) throws IOException, ExecutionSetupException { - this (userName, pluginRegistry.resolve(storagePluginConfig, HBaseStoragePlugin.class), hbaseScanSpec, columns, maxRecords); + this(userName, pluginRegistry.resolve(storagePluginConfig, HBaseStoragePlugin.class), + hbaseScanSpec, columns, maxRecords, baseScanSpec, + pushedFilter == null ? null : LogicalExpressionParser.parse(pushedFilter), + pushedFilterFullyConverted, true); } public HBaseGroupScan(String userName, HBaseStoragePlugin storagePlugin, HBaseScanSpec scanSpec, List columns, int maxRecords) { + this(userName, storagePlugin, scanSpec, columns, maxRecords, null, null, false, false); + } + + HBaseGroupScan(String userName, HBaseStoragePlugin storagePlugin, HBaseScanSpec scanSpec, + List columns, int maxRecords, HBaseScanSpec baseScanSpec, + LogicalExpression pushedFilter, boolean pushedFilterFullyConverted, boolean rebuild) { super(userName); this.storagePlugin = storagePlugin; this.storagePluginConfig = storagePlugin.getConfig(); - this.hbaseScanSpec = scanSpec; + this.baseScanSpec = baseScanSpec; + this.pushedFilter = pushedFilter; + this.pushedFilterFullyConverted = pushedFilterFullyConverted; + this.hbaseScanSpec = rebuild && pushedFilter != null ? baseScanSpec : scanSpec; this.columns = columns == null ? ALL_COLUMNS : columns; this.maxRecords = maxRecords; + if (rebuild && pushedFilter != null) { + if (baseScanSpec == null) { + throw new IllegalArgumentException("Cached HBase filter has no base scan specification"); + } + HBaseFilterBuilder builder = new HBaseFilterBuilder(this, pushedFilter); + HBaseScanSpec rebuilt = builder.parseTree(); + if (rebuilt == null || (pushedFilterFullyConverted && !builder.isAllExpressionsConverted())) { + throw new IllegalArgumentException("Cached HBase filter cannot be rebuilt safely"); + } + this.hbaseScanSpec = rebuilt; + } init(); } @@ -131,6 +165,9 @@ private HBaseGroupScan(HBaseGroupScan that) { super(that); this.columns = that.columns == null ? ALL_COLUMNS : that.columns; this.hbaseScanSpec = that.hbaseScanSpec; + this.baseScanSpec = that.baseScanSpec; + this.pushedFilter = that.pushedFilter; + this.pushedFilterFullyConverted = that.pushedFilterFullyConverted; this.endpointFragmentMapping = that.endpointFragmentMapping; this.regionsToScan = that.regionsToScan; this.storagePlugin = that.storagePlugin; @@ -146,6 +183,9 @@ private HBaseGroupScan(HBaseGroupScan that, int maxRecords) { super(that); this.columns = that.columns == null ? ALL_COLUMNS : that.columns; this.hbaseScanSpec = that.hbaseScanSpec; + this.baseScanSpec = that.baseScanSpec; + this.pushedFilter = that.pushedFilter; + this.pushedFilterFullyConverted = that.pushedFilterFullyConverted; this.endpointFragmentMapping = that.endpointFragmentMapping; this.regionsToScan = that.regionsToScan; this.storagePlugin = that.storagePlugin; @@ -449,6 +489,27 @@ public HBaseScanSpec getHBaseScanSpec() { return hbaseScanSpec; } + @JsonProperty("baseScanSpec") + public HBaseScanSpec getBaseScanSpec() { + return baseScanSpec; + } + + @JsonProperty("pushedFilter") + public String getPushedFilter() { + return pushedFilter == null ? null : ExpressionStringBuilder.toString(pushedFilter); + } + + @JsonProperty("pushedFilterFullyConverted") + public boolean isPushedFilterFullyConverted() { + return pushedFilterFullyConverted; + } + + @Override + @JsonIgnore + public boolean supportPlanCache() { + return storagePlugin.supportPlanCache() && (pushedFilter == null || baseScanSpec != null); + } + @Override @JsonIgnore public boolean canPushdownProjects(List columns) { diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java index 835d36a8a11..a07126b87e8 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java @@ -122,7 +122,8 @@ protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel fi } final HBaseGroupScan newGroupsScan = new HBaseGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), - newScanSpec, groupScan.getColumns(), groupScan.getMaxRecords()); + newScanSpec, groupScan.getColumns(), groupScan.getMaxRecords(), groupScan.getHBaseScanSpec(), + conditionExp, hbaseFilterBuilder.isAllExpressionsConverted(), false); newGroupsScan.setFilterPushedDown(true); final ScanPrel newScanPrel = new ScanPrel(scan.getCluster(), filter.getTraitSet(), newGroupsScan, scan.getRowType(), scan.getTable()); diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java index 48c4fe4d6d5..d6e2eed1b0a 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java @@ -25,12 +25,17 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.planner.PlannerPhase; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; +import org.apache.drill.exec.store.PlanCacheTable; import org.apache.drill.exec.store.SchemaConfig; import org.apache.drill.exec.store.StoragePluginOptimizerRule; import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hashing; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import com.fasterxml.jackson.core.type.TypeReference; @@ -58,6 +63,32 @@ public boolean supportsRead() { return true; } + @Override + public boolean supportPlanCache() { + return true; + } + + @Override + public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { + if (!(selection instanceof HBaseScanSpec)) { + return null; + } + String tableName = ((HBaseScanSpec) selection).getTableName(); + String version = planCacheTableVersion(tableName); + return version == null ? null : new PlanCacheTable(tableName, version); + } + + @Override + public String planCacheTableVersion(String identifier) throws IOException { + try (Admin admin = getConnection().getAdmin()) { + TableName tableName = TableName.valueOf(identifier); + if (!admin.tableExists(tableName)) { + return null; + } + return Hashing.sha256().hashBytes(admin.getTableDescriptor(tableName).toByteArray()).toString(); + } + } + @Override public HBaseGroupScan getPhysicalScan(String userName, JSONOptions selection) throws IOException { HBaseScanSpec scanSpec = selection.getListWith(new TypeReference() {}); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index 90cb45e4138..d40abd0eaba 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -86,6 +86,8 @@ public enum SqlStatementType {OTHER, ANALYZE, CTAS, EXPLAIN, DESCRIBE_TABLE, DES /** Stores constants and their holders by type */ private final Map> constantValueHolderCache; private SqlStatementType stmtType; + private Runnable pendingPlanCacheInsert; + private boolean planCacheHit; /* * Flag to indicate if close has been called, after calling close the first @@ -381,6 +383,24 @@ public void close() throws Exception { } } + public void setPendingPlanCacheInsert(Runnable insert) { + pendingPlanCacheInsert = insert; + } + + public void setPlanCacheHit() { + planCacheHit = true; + } + + public boolean isPlanCacheHit() { + return planCacheHit; + } + + public Runnable takePendingPlanCacheInsert() { + Runnable insert = pendingPlanCacheInsert; + pendingPlanCacheInsert = null; + return insert; + } + /** * @param stmtType : Sets the type {@link SqlStatementType} of the statement e.g. CTAS, ANALYZE */ diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index bfaa4cee19e..48df200a8dd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -43,6 +43,15 @@ */ public interface GroupScan extends Scan, HasAffinity { + /** + * The scan implementation must reconstruct value-dependent work from rebound + * expressions during physical-plan deserialization, including pruning and + * assignments that depend on the original predicate. + */ + default boolean supportPlanCache() { + return false; + } + /** * columns list in GroupScan : 1) empty_column is for skipAll query. * 2) NULL is interpreted as ALL_COLUMNS. @@ -87,6 +96,12 @@ public interface GroupScan extends Scan, HasAffinity { @JsonIgnore String getDigest(); + /** Text shown for this scan in an explain plan; may hide bound literal values. */ + @JsonIgnore + default String getExplainDigest() { + return getDigest(); + } + @JsonIgnore ScanStats getScanStats(PlannerSettings settings); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java index 0828bea676c..6097d22c4aa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashToMergeExchange.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.base.Sender; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; @@ -40,7 +41,7 @@ public class HashToMergeExchange extends AbstractExchange { @JsonCreator public HashToMergeExchange(@JsonProperty("child") PhysicalOperator child, @JsonProperty("expr") LogicalExpression expr, - @JsonProperty("orderings") List orderExprs) { + @JsonProperty("orderings") @JsonAlias("orderExpr") List orderExprs) { super(child); this.distExpr = expr; this.orderExprs = orderExprs; @@ -62,7 +63,12 @@ protected PhysicalOperator getNewWithChild(PhysicalOperator child) { return new HashToMergeExchange(child, distExpr, orderExprs); } - @JsonProperty("orderExpr") + @JsonProperty("expr") + public LogicalExpression getExpression() { + return distExpr; + } + + @JsonProperty("orderings") public List getOrderExpressions(){ return orderExprs; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java index 7808c67e42b..9ef3e303a12 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/OrderedMuxExchange.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.physical.base.Receiver; import org.apache.drill.common.logical.data.Order.Ordering; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonTypeName; /** @@ -34,6 +35,7 @@ public class OrderedMuxExchange extends AbstractMuxExchange { private final List orderings; + @JsonCreator public OrderedMuxExchange(@JsonProperty("child") PhysicalOperator child, @JsonProperty("orderings")List orderings) { super(child); this.orderings = orderings; @@ -48,4 +50,9 @@ public Receiver getReceiver(int minorFragmentId) { protected PhysicalOperator getNewWithChild(PhysicalOperator child) { return new OrderedMuxExchange(child, orderings); } + + @JsonProperty("orderings") + public List getOrderings() { + return orderings; + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java index 96c29162fd8..f527e651715 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java @@ -93,6 +93,10 @@ public String writeJson(PhysicalOperator op) throws JsonProcessingException{ return mapper.writeValueAsString(op); } + public String writeJson(PhysicalPlan plan) throws JsonProcessingException { + return mapper.writeValueAsString(plan); + } + public PhysicalPlan readPhysicalPlan(String json) throws IOException { logger.debug("Reading physical plan {}", json); return physicalPlanReader.readValue(json); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java index bed9a2716fe..cd41c307abf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/ExprToRex.java @@ -51,8 +51,17 @@ public ExprToRex(RelNode inputRel, RelDataType newRowType, RexBuilder builder) { } public static RelDataTypeField findField(String fieldName, RelDataType rowType) { + // A projected wildcard can be named "*" in a plugin row type. It is a + // literal field name here, not an expression to parse. + for (RelDataTypeField f : rowType.getFieldList()) { + if (fieldName.equalsIgnoreCase(f.getName())) { + return f; + } + } + if ("*".equals(fieldName)) { + return null; + } final String rootPart = SchemaPath.parseFromString(fieldName).getRootSegmentPath(); - for (RelDataTypeField f : rowType.getFieldList()) { if (rootPart.equalsIgnoreCase(f.getName())) { return f; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java index 349ba2a02f9..128e08f113a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java @@ -469,6 +469,17 @@ public LogicalExpression visitCorrelVariable(RexCorrelVariable correlVariable) { @Override public LogicalExpression visitDynamicParam(RexDynamicParam dynamicParam) { + if (dynamicParam instanceof org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam) { + org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam bound = + (org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam) dynamicParam; + LogicalExpression expression = bound.getLiteral().accept(this); + if (!(expression instanceof org.apache.drill.common.expression.LiteralExpression)) { + return doUnknown(dynamicParam); + } + ((org.apache.drill.common.expression.LiteralExpression) expression) + .setDynamicParamIndex(bound.getIndex()); + return expression; + } return doUnknown(dynamicParam); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java index dac84082e8e..d7b90992b14 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java @@ -128,7 +128,7 @@ public RelDataType deriveRowType() { @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("groupscan", this.getGroupScan().getDigest()); + return super.explainTerms(pw).item("groupscan", this.getGroupScan().getExplainDigest()); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java index 4b5202d8366..deb8cdd1924 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java @@ -58,6 +58,9 @@ public class PlannerSettings implements Context{ public static final OptionValidator CONSTANT_FOLDING = new BooleanValidator("planner.enable_constant_folding", new OptionDescription("If one side of a filter condition is a constant expression, constant folding evaluates the expression in the planning phase and replaces the expression with the constant value. For example, Drill can rewrite WHERE age + 5 < 42 as WHERE age < 37.")); + public static final String ENABLE_PLAN_CACHE_OPTION = "planner.enable_plan_cache"; + public static final BooleanValidator PLAN_CACHE = new BooleanValidator(ENABLE_PLAN_CACHE_OPTION, + new OptionDescription("Enables reuse of compatible physical plans for parameterized SQL queries.")); public static final String DISABLE_EXCHANGE_OPTION = "planner.disable_exchanges"; public static final OptionValidator EXCHANGE = new BooleanValidator(DISABLE_EXCHANGE_OPTION, new OptionDescription("Toggles the state of hashing to a random exchange.")); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java index 1e0bdf428a3..84aeb63efa3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java @@ -87,7 +87,7 @@ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("groupscan", this.getGroupScan().getDigest()); + return super.explainTerms(pw).item("groupscan", this.getGroupScan().getExplainDigest()); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index c706f8f3733..40fd162662b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -18,6 +18,9 @@ package org.apache.drill.exec.planner.sql; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.TreeMap; import org.apache.calcite.sql.SqlDescribeSchema; import org.apache.calcite.sql.SqlKind; @@ -31,10 +34,13 @@ import org.apache.calcite.tools.ValidationException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.alias.AliasRegistry; import org.apache.drill.exec.exception.MetadataException; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.ops.QueryContext.SqlStatementType; +import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler; import org.apache.drill.exec.planner.sql.handlers.AnalyzeTableHandler; import org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler; @@ -60,6 +66,7 @@ import org.apache.drill.exec.work.foreman.ForemanSetupException; import org.apache.drill.exec.work.foreman.SqlUnsupportedException; import com.google.common.base.Throwables; +import com.google.common.hash.Hashing; import org.apache.hadoop.security.AccessControlException; public class DrillSqlWorker { @@ -218,7 +225,6 @@ private static PhysicalPlan getPhysicalPlan(QueryContext context, String sql, Po */ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Pointer textPlan) throws ForemanSetupException, RelConversionException, IOException, ValidationException { - final SqlConverter parser = new SqlConverter(context); injector.injectChecked(context.getExecutionControls(), "sql-parsing", ForemanSetupException.class); final SqlNode sqlNode = checkAndApplyAutoLimit(parser, context, sql); @@ -295,9 +301,116 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point context.getOptions().setLocalOption(ExecConstants.RETURN_RESULT_SET_FOR_DDL, true); } + if (context.getOptions().getOption(PlannerSettings.PLAN_CACHE) + && !context.getSession().hasTemporaryTables() + && !hasMutableAliases(context) + && PlanCacheEligibility.isSafeToCache(sqlNode, context.getFunctionRegistry())) { + try { + PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(sqlNode); + PlanCache cache = context.getDrillbitContext().getPlanCache(); + String key = context.getQueryUserName() + '\n' + + context.getSession().getDefaultSchemaPath() + '\n' + + candidate.template; + PlanCache.Entry entry = cache.get(key); + String optionsFingerprint = null; + if (entry != null) { + try { + optionsFingerprint = optionFingerprint(context); + if (entry.matchesContext(optionsFingerprint, context.getStorage())) { + PhysicalPlan bound = entry.bind(candidate.literals, + context.getDrillbitContext().getPlanReader()); + if (textPlan != null) { + textPlan.value = PlanCache.textWithBindings(entry.getTextPlan(), candidate.literals); + } + cache.recordBind(); + context.setPlanCacheHit(); + return bound; + } + } catch (RuntimeException | IOException e) { + logger.debug("Cached plan could not be rebound", e); + } + cache.invalidate(key); + } + // Capture the context before planning. Changes during planning invalidate + // this entry on its first hit. + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve( + parser.getDefaultSchema(), sqlNode, + context.getStorage()); + if (snapshot != null) { + if (optionsFingerprint == null) { + optionsFingerprint = optionFingerprint(context); + } + snapshot = snapshot.withOptionsFingerprint(optionsFingerprint); + } + PhysicalPlan planned = handler.getPlan(candidate.sql); + final String templateTextPlan = textPlan == null ? null : textPlan.value; + if (textPlan != null) { + textPlan.value = PlanCache.textWithBindings(templateTextPlan, candidate.literals); + } + if (snapshot != null && cache.canCache(planned)) { + final org.apache.drill.exec.planner.PhysicalPlanReader cacheReader = + context.getDrillbitContext().getPlanReader(); + final PlanCache.ContextSnapshot cacheContext = snapshot; + context.setPendingPlanCacheInsert(() -> cache.writeAfterSuccess(key, planned, + templateTextPlan, cacheReader, cacheContext)); + } + return planned; + } catch (Exception e) { + logger.debug("Plan cache attempt fell back to ordinary planning", e); + } + } + return handler.getPlan(sqlNode); } + private static String optionFingerprint(QueryContext context) { + // OptionManager iteration walks system defaults, system overrides, session + // overrides, and query overrides in precedence order. getOptionList() only + // exposes local session/query values and misses system-level changes. + Map options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (OptionValue option : context.getOptions()) { + options.put(option.name, option); + } + StringBuilder fingerprint = new StringBuilder(); + for (OptionValue option : options.values()) { + fingerprint.append(option.name).append(':').append(option.kind).append('='); + switch (option.kind) { + case BOOLEAN: + fingerprint.append(option.bool_val); + break; + case LONG: + fingerprint.append(option.num_val); + break; + case DOUBLE: + fingerprint.append(option.float_val); + break; + case STRING: + fingerprint.append(option.string_val.length()).append(':').append(option.string_val); + break; + default: + throw new IllegalArgumentException("Unsupported option kind"); + } + fingerprint.append('\n'); + } + // Keep the cache key bounded even when the system has many registered options. + return Hashing.sha256().hashString(fingerprint, StandardCharsets.UTF_8).toString(); + } + + private static boolean hasMutableAliases(QueryContext context) { + // Table-version checks cannot detect an alias redirected to another table. + AliasRegistry[] registries = { + context.getAliasRegistryProvider().getStorageAliasesRegistry(), + context.getAliasRegistryProvider().getTableAliasesRegistry() + }; + for (AliasRegistry registry : registries) { + if (registry.getPublicAliases().getAllAliases().hasNext() + || registry.getUserAliases(context.getQueryUserName()).getAllAliases().hasNext()) { + return true; + } + } + return false; + } + private static boolean isAutoLimitShouldBeApplied(SqlNode sqlNode, int queryMaxRows) { return (queryMaxRows > 0) && sqlNode.getKind().belongsTo(SqlKind.QUERY) && (sqlNode.getKind() != SqlKind.ORDER_BY || isAutoLimitLessThanOrderByFetch((SqlOrderBy) sqlNode, queryMaxRows)); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java new file mode 100644 index 00000000000..0dfda14891a --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -0,0 +1,623 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.io.IOException; +import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.drill.common.expression.ExpressionStringBuilder; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.LiteralExpression; +import org.apache.drill.common.expression.ValueExpressions; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.parser.LogicalExpressionParser; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; +import org.apache.drill.exec.rpc.NamedThreadFactory; +import org.apache.drill.exec.store.StoragePlugin; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.StoragePluginRegistry.PluginException; +import org.apache.drill.exec.store.PlanCacheTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Objects; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.hash.Hashing; + +/** Drillbit-scoped, immutable JSON snapshots of query plans. */ +public final class PlanCache implements AutoCloseable { + private static final Logger logger = LoggerFactory.getLogger(PlanCache.class); + private static final ObjectMapper JSON = new ObjectMapper(); + private static final String MARKER = ExpressionStringBuilder.BOUND_DYNAMIC_PARAM + "("; + private final Cache entries = CacheBuilder.newBuilder() + .maximumWeight(32 * 1024 * 1024) + .weigher((String key, Entry value) -> value.json.length()) + .expireAfterWrite(10, TimeUnit.MINUTES) + .recordStats() + .build(); + private final AtomicLong successfulBinds = new AtomicLong(); + private final ThreadPoolExecutor writer = new ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(64), + new NamedThreadFactory("plan-cache-writer-")); + + public long getHitCount() { + return successfulBinds.get(); + } + + public void recordBind() { + successfulBinds.incrementAndGet(); + } + + /** Appends values from this query without changing the cached explain template. */ + static String textWithBindings(String template, List literals) { + if (template == null || literals.isEmpty()) { + return template; + } + StringBuilder text = new StringBuilder(template); + if (text.length() > 0 && text.charAt(text.length() - 1) != '\n') { + text.append('\n'); + } + text.append("Parameters: "); + for (int i = 0; i < literals.size(); i++) { + if (i > 0) { + text.append(", "); + } + text.append('?').append(i).append(" = ") + .append(literals.get(i).toString().replace("\r", "\\r").replace("\n", "\\n")); + } + return text.toString(); + } + + public Entry get(String key) { + return entries.getIfPresent(key); + } + + /** Waits for writes already submitted by completed queries; intended for tests. */ + public void awaitWrites() throws InterruptedException, ExecutionException, TimeoutException { + writer.submit(() -> { }).get(10, TimeUnit.SECONDS); + } + + public boolean canCache(PhysicalPlan plan) { + return plan.getSortedOperators().stream() + .noneMatch(op -> op instanceof GroupScan && !((GroupScan) op).supportPlanCache()); + } + + public void writeAfterSuccess(String key, PhysicalPlan plan, String textPlan, + PhysicalPlanReader reader, ContextSnapshot context) { + try { + writer.execute(() -> { + try { + put(key, plan, textPlan, reader, context); + } catch (RuntimeException | IOException e) { + logger.debug("Plan cache entry could not be published", e); + } + }); + } catch (RejectedExecutionException e) { + logger.debug("Plan cache writer is busy; skipping cache entry", e); + } + } + + public void invalidate(String key) { + entries.invalidate(key); + } + + public boolean put(String key, PhysicalPlan plan, String textPlan, + PhysicalPlanReader reader, ContextSnapshot context) throws IOException { + // The engine checks generic context and expression compatibility. Each + // opted-in scan is responsible for rebuilding its value-dependent state. + if (context == null || context.optionsFingerprint == null || !canCache(plan)) { + return false; + } + String json = reader.writeJson(plan); + // Confirm that standard PhysicalPlan JSON can be read back before publishing. + reader.readPhysicalPlan(json); + entries.put(key, new Entry(json, textPlan, context)); + return true; + } + + @Override + public void close() { + writer.shutdownNow(); + } + + public static final class Entry { + private final String json; + private final String textPlan; + private final ContextSnapshot context; + + private Entry(String json, String textPlan, ContextSnapshot context) { + this.json = json; + this.textPlan = textPlan; + this.context = context; + } + + public boolean matchesContext(String optionsFingerprint, StoragePluginRegistry plugins) { + return context.isCurrent(optionsFingerprint, plugins); + } + + public String getTextPlan() { + return textPlan; + } + + public PhysicalPlan bind(List literals, PhysicalPlanReader reader) + throws IOException { + JsonNode tree = JSON.readTree(json); + walk(tree, literals); + return reader.readPhysicalPlan(JSON.writeValueAsString(tree)); + } + } + + /** Options, plugin configurations, and table versions checked after lookup. */ + public static final class ContextSnapshot { + private final String optionsFingerprint; + private final Map tableVersions; + private final Map pluginConfigs; + + private ContextSnapshot(String optionsFingerprint, Map tableVersions, + Map pluginConfigs) { + this.optionsFingerprint = optionsFingerprint; + this.tableVersions = java.util.Collections.unmodifiableMap(tableVersions); + this.pluginConfigs = java.util.Collections.unmodifiableMap(pluginConfigs); + } + + public ContextSnapshot withOptionsFingerprint(String fingerprint) { + return new ContextSnapshot(Objects.requireNonNull(fingerprint), tableVersions, pluginConfigs); + } + + private static final class TableIdentifier { + private final String storageName; + private final String tableId; + + private TableIdentifier(String storageName, String tableId) { + this.storageName = storageName; + this.tableId = tableId; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof TableIdentifier)) { + return false; + } + TableIdentifier that = (TableIdentifier) other; + return Objects.equals(storageName, that.storageName) + && Objects.equals(tableId, that.tableId); + } + + @Override + public int hashCode() { + return Objects.hash(storageName, tableId); + } + } + + private static final class ResolvedTable { + private final List tableNames; + private final TableIdentifier identifier; + private final String version; + private final StoragePluginConfig pluginConfig; + + private ResolvedTable(List tableNames, TableIdentifier identifier, String version, + StoragePluginConfig pluginConfig) { + this.tableNames = tableNames; + this.identifier = identifier; + this.version = version; + this.pluginConfig = pluginConfig; + } + } + + /** Resolves every physical table in a query, including joins and subqueries. */ + public static ContextSnapshot resolve(SchemaPlus defaultSchema, SqlNode query, + StoragePluginRegistry plugins) { + List dependencies = new ArrayList<>(); + try { + if (!collectQuery(defaultSchema, query, new HashSet<>(), dependencies)) { + return null; + } + } catch (RuntimeException e) { + logger.debug("Query sources could not be validated for the plan cache", e); + return null; + } + Map versions = new LinkedHashMap<>(); + Map configs = new LinkedHashMap<>(); + for (ResolvedTable dependency : dependencies) { + String previous = versions.putIfAbsent(dependency.identifier, dependency.version); + if (previous != null && !previous.equals(dependency.version)) { + return null; + } + try { + String fingerprint = configFingerprint(plugins, dependency.pluginConfig); + String previousConfig = configs.putIfAbsent(dependency.identifier.storageName, + fingerprint); + if (previousConfig != null && !previousConfig.equals(fingerprint)) { + return null; + } + } catch (RuntimeException e) { + logger.debug("Plugin configuration could not be fingerprinted for the plan cache", e); + return null; + } + } + return new ContextSnapshot(null, versions, configs); + } + + private static boolean collectQuery(SchemaPlus schema, SqlNode node, + Set ctes, List dependencies) { + if (node == null) { + return true; + } + if (node instanceof SqlNodeList) { + for (SqlNode child : (SqlNodeList) node) { + if (!collectQuery(schema, child, ctes, dependencies)) { + return false; + } + } + return true; + } + if (node instanceof SqlWith) { + SqlWith with = (SqlWith) node; + Set local = new HashSet<>(ctes); + for (SqlNode itemNode : with.withList) { + SqlWithItem item = (SqlWithItem) itemNode; + if (!collectQuery(schema, item.query, local, dependencies)) { + return false; + } + local.add(item.name.getSimple()); + } + return collectQuery(schema, with.body, local, dependencies); + } + if (node instanceof SqlSelect) { + SqlSelect select = (SqlSelect) node; + if (!collectFrom(schema, select.getFrom(), ctes, dependencies)) { + return false; + } + for (SqlNode operand : select.getOperandList()) { + if (operand != select.getFrom() + && !collectQuery(schema, operand, ctes, dependencies)) { + return false; + } + } + return true; + } + if (node instanceof SqlCall) { + for (SqlNode operand : ((SqlCall) node).getOperandList()) { + if (!collectQuery(schema, operand, ctes, dependencies)) { + return false; + } + } + } + return true; + } + + private static boolean collectFrom(SchemaPlus schema, SqlNode from, + Set ctes, List dependencies) { + if (from == null || from.getKind() == SqlKind.VALUES) { + return true; + } + if (from instanceof SqlIdentifier) { + List names = ((SqlIdentifier) from).names; + if (names.size() == 1 && ctes.contains(names.get(0))) { + return true; + } + for (ResolvedTable dependency : dependencies) { + if (dependency.tableNames.equals(names)) { + return true; + } + } + ResolvedTable source = resolveTable(schema, names); + if (source == null) { + return false; + } + dependencies.add(source); + return true; + } + if (from instanceof SqlCall) { + SqlCall call = (SqlCall) from; + if (call.getKind() == SqlKind.AS || call.getKind() == SqlKind.LATERAL) { + return collectFrom(schema, call.operand(0), ctes, dependencies); + } + if (call.getKind() == SqlKind.JOIN) { + return collectFrom(schema, call.operand(0), ctes, dependencies) + && collectFrom(schema, call.operand(3), ctes, dependencies) + && collectQuery(schema, call.operand(5), ctes, dependencies); + } + if (call.getKind().belongsTo(SqlKind.QUERY)) { + return collectQuery(schema, call, ctes, dependencies); + } + } + return false; + } + + private static ResolvedTable resolveTable(SchemaPlus defaultSchema, List names) { + if (names.isEmpty() || names.get(names.size() - 1).contains("*") + || names.get(names.size() - 1).contains("?")) { + return null; + } + SchemaPlus schema = SchemaUtilities.findSchema(defaultSchema, + names.subList(0, names.size() - 1)); + if (schema == null) { + return null; + } + Table table = schema.getTable(names.get(names.size() - 1)); + if (!(table instanceof DrillTable)) { + return null; + } + DrillTable drillTable = (DrillTable) table; + Object selection = drillTable.getSelection(); + if (!(selection instanceof DrillTableSelection)) { + return null; + } + StoragePlugin storagePlugin = drillTable.getPlugin(); + if (!storagePlugin.supportPlanCache()) { + return null; + } + try { + PlanCacheTable source = storagePlugin.planCacheTable((DrillTableSelection) selection); + if (source == null) { + return null; + } + return new ResolvedTable( + java.util.Collections.unmodifiableList(new java.util.ArrayList<>(names)), + new TableIdentifier(drillTable.getStorageEngineName(), source.getIdentifier()), + source.getVersion(), storagePlugin.getConfig()); + } catch (IOException e) { + logger.debug("Table version could not be read for the plan cache", e); + return null; + } + } + + public boolean isCurrent(String currentOptionsFingerprint, StoragePluginRegistry plugins) { + if (!Objects.equals(optionsFingerprint, currentOptionsFingerprint)) { + return false; + } + Map currentPlugins = new HashMap<>(); + for (Map.Entry entry : pluginConfigs.entrySet()) { + try { + StoragePlugin plugin = plugins.getPlugin(entry.getKey()); + if (plugin == null || !plugin.supportPlanCache() + || !entry.getValue().equals(configFingerprint(plugins, plugin.getConfig()))) { + return false; + } + currentPlugins.put(entry.getKey(), plugin); + } catch (PluginException | RuntimeException e) { + logger.debug("Plugin configuration could not be checked for the plan cache", e); + return false; + } + } + for (Map.Entry entry : tableVersions.entrySet()) { + TableIdentifier identifier = entry.getKey(); + try { + StoragePlugin storagePlugin = currentPlugins.get(identifier.storageName); + if (!Objects.equals(entry.getValue(), + storagePlugin.planCacheTableVersion(identifier.tableId))) { + return false; + } + } catch (IOException e) { + logger.debug("Table version could not be read for the plan cache", e); + return false; + } + } + return true; + } + + private static String configFingerprint(StoragePluginRegistry plugins, + StoragePluginConfig config) { + if (config == null) { + throw new IllegalArgumentException("Missing plugin configuration"); + } + String encoded = plugins.encode(config); + if (encoded == null) { + throw new IllegalArgumentException("Plugin configuration could not be encoded"); + } + return Hashing.sha256().hashString(encoded, StandardCharsets.UTF_8).toString(); + } + } + + private static String serialize(SqlLiteral literal, MajorType type, int index) { + Object value = literal.getValue(); + LogicalExpression expression; + if (value instanceof java.math.BigDecimal) { + java.math.BigDecimal number = (java.math.BigDecimal) value; + if (type.getMinorType() == MinorType.VARDECIMAL) { + java.math.BigDecimal decimal = number.setScale(type.getScale(), RoundingMode.UNNECESSARY); + if (decimal.precision() > type.getPrecision()) { + throw new IllegalArgumentException("Decimal parameter exceeds cached precision"); + } + expression = ValueExpressions.getVarDecimal(decimal, + type.getPrecision(), type.getScale()); + } else if (type.getMinorType() == MinorType.FLOAT8) { + expression = ValueExpressions.getFloat8(number.doubleValue()); + } else if (type.getMinorType() == MinorType.FLOAT4) { + expression = ValueExpressions.getFloat4(number.floatValue()); + } else if (type.getMinorType() == MinorType.BIGINT) { + expression = ValueExpressions.getBigInt(number.longValueExact()); + } else if (type.getMinorType() == MinorType.INT) { + expression = ValueExpressions.getInt(number.intValueExact()); + } else { + throw new IllegalArgumentException("Unsupported numeric parameter type"); + } + } else if (value instanceof Boolean) { + expression = ValueExpressions.getBit((Boolean) value); + } else if (value instanceof org.apache.calcite.util.NlsString) { + String string = ((org.apache.calcite.util.NlsString) value).getValue(); + expression = ValueExpressions.getChar(string, type.getPrecision()); + } else { + throw new IllegalArgumentException("Unsupported parameter type"); + } + ((LiteralExpression) expression).setDynamicParamIndex(index); + return ExpressionStringBuilder.toString(expression); + } + + /** Visits every serialized field, including nested plugin-specific fields. */ + private static void walk(JsonNode node, List replacements) { + if (node.isObject()) { + ObjectNode object = (ObjectNode) node; + object.fields().forEachRemaining(field -> { + JsonNode value = field.getValue(); + if (value.isTextual() && value.asText().contains(MARKER)) { + object.put(field.getKey(), rewrite(value.asText(), replacements)); + } else { + walk(value, replacements); + } + }); + } else if (node.isArray()) { + ArrayNode array = (ArrayNode) node; + for (int i = 0; i < array.size(); i++) { + JsonNode value = array.get(i); + if (value.isTextual() && value.asText().contains(MARKER)) { + array.set(i, JSON.getNodeFactory().textNode(rewrite(value.asText(), replacements))); + } else { + walk(value, replacements); + } + } + } + } + + private static String rewrite(String input, List replacements) { + // Parsing first ensures that a marker belongs to a Drill expression field. + LogicalExpression original = LogicalExpressionParser.parse(input); + Map originalTypes = new HashMap<>(); + collectTypes(original, originalTypes); + StringBuilder output = new StringBuilder(); + int offset = 0; + while (offset < input.length()) { + int start = findMarker(input, offset); + if (start < 0) { + output.append(input, offset, input.length()); + break; + } + output.append(input, offset, start); + int indexEnd = input.indexOf(',', start + MARKER.length()); + if (indexEnd < 0) { + throw new IllegalArgumentException("Malformed cached parameter"); + } + int index = Integer.parseInt(input.substring(start + MARKER.length(), indexEnd).trim()); + if (!originalTypes.containsKey(index)) { + throw new IllegalArgumentException("Cached marker is not an expression parameter"); + } + int end = matchingClose(input, indexEnd + 1); + if (index < 0 || index >= replacements.size()) { + throw new IllegalArgumentException("Unexpected cached parameter index"); + } + // Keep the slot marker in the execution plan for round-trip checks. + output.append(serialize(replacements.get(index), originalTypes.get(index), index)); + offset = end + 1; + } + String result = output.toString(); + LogicalExpression parsed = LogicalExpressionParser.parse(result); + Map newTypes = new HashMap<>(); + collectTypes(parsed, newTypes); + if (!originalTypes.equals(newTypes)) { + throw new IllegalArgumentException("Cached parameter type changed"); + } + return result; + } + + private static int findMarker(String input, int offset) { + boolean quoted = false; + boolean escaped = false; + for (int i = 0; i < input.length(); i++) { + char c = input.charAt(i); + if (escaped) { + escaped = false; + } else if (c == '\\' && quoted) { + escaped = true; + } else if (c == '\'') { + quoted = !quoted; + } else if (!quoted && i >= offset && input.startsWith(MARKER, i)) { + return i; + } + } + return -1; + } + + private static void collectTypes(LogicalExpression expression, + Map types) { + if (expression instanceof LiteralExpression) { + int index = ((LiteralExpression) expression).getDynamicParamIndex(); + if (index >= 0) { + MajorType type = expression.getMajorType(); + MajorType previous = types.putIfAbsent(index, type); + if (previous != null && !previous.equals(type)) { + throw new IllegalArgumentException("Conflicting cached parameter types"); + } + } + } + for (LogicalExpression child : expression) { + collectTypes(child, types); + } + } + + private static int matchingClose(String input, int start) { + boolean quoted = false; + boolean escaped = false; + int depth = 1; + for (int i = start; i < input.length(); i++) { + char c = input.charAt(i); + if (escaped) { + escaped = false; + } else if (c == '\\' && quoted) { + escaped = true; + } else if (c == '\'') { + quoted = !quoted; + } else if (!quoted && c == '(') { + depth++; + } else if (!quoted && c == ')' && --depth == 0) { + return i; + } + } + throw new IllegalArgumentException("Unclosed cached parameter"); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheEligibility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheEligibility.java new file mode 100644 index 00000000000..d9b70dc60bb --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheEligibility.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.util.Locale; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.drill.exec.expr.fn.DrillFuncHolder; +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; + +/** Checks whether a parsed query is safe to reuse without SQL validation. */ +final class PlanCacheEligibility { + private PlanCacheEligibility() { } + + static boolean isSafeToCache(SqlNode parsed, FunctionImplementationRegistry functions) { + // SqlKind.OTHER_FUNCTION is a classification, not evidence of volatility. + if (!parsed.getKind().belongsTo(SqlKind.QUERY)) { + return false; + } + try { + parsed.accept(new SqlShuttle() { + @Override + public SqlNode visit(SqlCall call) { + if (!call.getOperator().isDeterministic() + || call.getOperator().isDynamicFunction() + || isQueryContextFunction(call) + || (call.getKind() == SqlKind.OTHER_FUNCTION + && hasVolatileOverload(call, functions))) { + throw UnsafeSql.INSTANCE; + } + return super.visit(call); + } + + @Override + public SqlNode visit(SqlDynamicParam param) { + throw UnsafeSql.INSTANCE; + } + + @Override + public SqlNode visit(SqlIdentifier id) { + // Niladic functions can be parsed as identifiers rather than calls. + if (id.isSimple() && isNiladicQueryContextName(id.getSimple())) { + throw UnsafeSql.INSTANCE; + } + return super.visit(id); + } + }); + return true; + } catch (UnsafeSql ignored) { + return false; + } + } + + private static boolean isQueryContextFunction(SqlCall call) { + String name = call.getOperator().getName().toUpperCase(Locale.ROOT); + switch (name) { + case "NOW": + case "CURRENT_TIMESTAMP": + case "LOCALTIMESTAMP": + case "STATEMENT_TIMESTAMP": + case "TRANSACTION_TIMESTAMP": + case "CURRENT_DATE": + case "CURRENT_TIME": + case "LOCALTIME": + case "SESSION_ID": + case "CURRENT_SCHEMA": + case "USER": + case "SESSION_USER": + case "SYSTEM_USER": + return true; + case "UNIX_TIMESTAMP": + return call.getOperandList().isEmpty(); + default: + return false; + } + } + + private static boolean isNiladicQueryContextName(String name) { + switch (name.toUpperCase(Locale.ROOT)) { + case "CURRENT_TIMESTAMP": + case "LOCALTIMESTAMP": + case "CURRENT_DATE": + case "CURRENT_TIME": + case "LOCALTIME": + case "SESSION_ID": + case "CURRENT_SCHEMA": + case "USER": + case "SESSION_USER": + case "SYSTEM_USER": + return true; + default: + return false; + } + } + + private static boolean hasVolatileOverload(SqlCall call, + FunctionImplementationRegistry functions) { + // Before validation, Drill functions may still be unresolved Calcite + // functions whose determinism flag is always true. + for (DrillFuncHolder holder : functions.getLocalFunctionRegistry() + .getMethods(call.getOperator().getName())) { + if (!holder.isDeterministic()) { + return true; + } + } + return false; + } + + private static final class UnsafeSql extends RuntimeException { + private static final UnsafeSql INSTANCE = new UnsafeSql(); + + private UnsafeSql() { + super(null, null, false, false); + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java new file mode 100644 index 00000000000..34bd405f4ba --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.drill.exec.planner.sql.conversion.SqlBoundDynamicParam; + +/** Extracts SQL literals into typed slots independently of cache eligibility. */ +final class PlanCacheParameterizer extends SqlShuttle { + private final List literals = new ArrayList<>(); + private final List types = new ArrayList<>(); + private boolean preserveProjection; + + static Candidate parameterize(SqlNode parsed) { + Objects.requireNonNull(parsed, "parsed"); + PlanCacheParameterizer visitor = new PlanCacheParameterizer(); + SqlNode parameterized = parsed.accept(visitor); + if (visitor.literals.isEmpty()) { + parameterized = parsed; + } + return new Candidate(parameterized, + Collections.unmodifiableList(visitor.literals), + parameterized.toString() + visitor.types); + } + + @Override + public SqlNode visit(SqlCall call) { + int configurationOperand = configurationOperand(call); + if (configurationOperand >= 0 && configurationOperand < call.operandCount()) { + // These operands become function names, encodings or resolved plugin + // references during conversion. They are not bindable expressions in + // the physical plan, so keep their entire subtree in the template key. + SqlCall parameterized = (SqlCall) call.clone(call.getParserPosition()); + for (int i = 0; i < call.operandCount(); i++) { + if (i != configurationOperand) { + parameterized.setOperand(i, visitNullable(call.operand(i))); + } + } + return parameterized; + } + if ("COALESCE".equalsIgnoreCase(call.getOperator().getName()) + && call.getOperandList().stream() + .anyMatch(operand -> operand instanceof SqlLiteral + && ((SqlLiteral) operand).getTypeName() == SqlTypeName.NULL)) { + // Calcite cannot infer a dynamic parameter's type alongside an untyped + // NULL. Keep this call in the key; explicit casts still allow slots. + return call; + } + if (call instanceof SqlJoin) { + SqlJoin join = (SqlJoin) call; + return new SqlJoin(join.getParserPosition(), visitNullable(join.getLeft()), + join.isNaturalNode(), join.getJoinTypeNode(), visitNullable(join.getRight()), + join.getConditionTypeNode(), visitNullable(join.getCondition())); + } + if (call.getKind() == SqlKind.ITEM && call.operand(1) instanceof SqlLiteral) { + // A map/array key identifies a field in the input schema. Changing it can + // change the scan projection and the HBase column used by pushdown. + return call.getOperator().createCall(call.getParserPosition(), + visitNullable(call.operand(0)), call.operand(1)); + } + // These literals control rows, ordinals, window frames or operator + // configuration. Keep them verbatim in the template key. + if (call.getKind() == SqlKind.VALUES || call.getKind() == SqlKind.ORDER_BY + || call.getKind() == SqlKind.OVER) { + if (call instanceof SqlOrderBy) { + SqlOrderBy orderBy = (SqlOrderBy) call; + boolean previous = preserveProjection; + preserveProjection = true; + SqlNode query; + try { + query = orderBy.query.accept(this); + } finally { + preserveProjection = previous; + } + return new SqlOrderBy(call.getParserPosition(), query, orderBy.orderList, + orderBy.offset, orderBy.fetch); + } + return call; + } + if (call instanceof SqlSelect) { + SqlSelect select = (SqlSelect) call; + // Keep structural SQL operands verbatim while visiting expressions. + boolean structuralProjection = preserveProjection + || (select.getGroup() != null && !select.getGroup().isEmpty()) + || (select.getOrderList() != null && !select.getOrderList().isEmpty()) + || (select.getWindowList() != null && !select.getWindowList().isEmpty()); + return new SqlSelect(select.getParserPosition(), + (SqlNodeList) select.getOperandList().get(0), + structuralProjection ? select.getSelectList() + : (SqlNodeList) visitNullable(select.getSelectList()), + visitNullable(select.getFrom()), visitNullable(select.getWhere()), + select.getGroup(), visitNullable(select.getHaving()), + select.getWindowList(), visitNullable(select.getQualify()), + select.getOrderList(), select.getOffset(), select.getFetch(), + select.getHints()); + } + return super.visit(call); + } + + private static int configurationOperand(SqlCall call) { + // Keep this list aligned with the value-dependent rewrites in DrillOptiq + // and PreProcessLogicalRel; ordinary data operands still get slots. + switch (call.getOperator().getName().toUpperCase(Locale.ROOT)) { + case "DATE_PART": + case "DATE_TRUNC": + case "EXTRACT": + case "TIMESTAMPDIFF": + case "TRIM": + case "HTTPREQUEST": + case "HTTP_REQUEST": + return 0; + case "CONVERT_FROM": + case "CONVERT_TO": + return 1; + case "LENGTH": + return call.operandCount() == 2 ? 1 : -1; + default: + return -1; + } + } + + private SqlNode visitNullable(SqlNode node) { + return node == null ? null : node.accept(this); + } + + @Override + public SqlNode visit(SqlLiteral literal) { + if (literal.getValue() == null) { + // A NULL without an inferred type is part of the template, not a slot. + return literal; + } + Object value = literal.getValue(); + if (!(value instanceof BigDecimal) && !(value instanceof Boolean) + && !(value instanceof org.apache.calcite.util.NlsString)) { + // Temporal literals, including unresolved DATE literals, and structural + // literals such as EXTRACT's time unit stay in the template key. + return literal; + } + int index = literals.size(); + literals.add(literal); + // Precision and scale affect inferred types and must be part of the key. + String shape = literal.getTypeName().name(); + if (value instanceof BigDecimal) { + BigDecimal number = (BigDecimal) value; + shape += ":" + number.precision() + ":" + number.scale(); + } else if (value instanceof org.apache.calcite.util.NlsString) { + org.apache.calcite.util.NlsString string = (org.apache.calcite.util.NlsString) value; + shape += ":" + string.getValue().length() + ":" + string.getCharsetName() + + ":" + string.getCollation(); + } + types.add(shape); + return new SqlBoundDynamicParam(index, literal.getParserPosition(), literal); + } + + static final class Candidate { + final SqlNode sql; + final List literals; + final String template; + + Candidate(SqlNode sql, List literals, String template) { + this.sql = sql; + this.literals = literals; + this.template = template; + } + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlToRelConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlToRelConverter.java new file mode 100644 index 00000000000..9545b21a783 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlToRelConverter.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql2rel.SqlRexConvertletTable; +import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.calcite.util.NlsString; + +/** Converts Drill SQL to relational expressions, preserving bound cache parameters. */ +public final class DrillSqlToRelConverter extends SqlToRelConverter { + public DrillSqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, + DrillCalciteCatalogReader catalog, RelOptCluster cluster, + SqlRexConvertletTable convertletTable, Config config) { + super(viewExpander, validator, catalog, cluster, convertletTable, config); + } + + @Override + public RexDynamicParam convertDynamicParam(SqlDynamicParam param) { + if (!(param instanceof SqlBoundDynamicParam)) { + return super.convertDynamicParam(param); + } + // Register the parameter with Calcite before reading its inferred type. + RexDynamicParam converted = super.convertDynamicParam(param); + SqlLiteral source = ((SqlBoundDynamicParam) param).getLiteral(); + Object literalValue = source.getValue(); + if (literalValue instanceof NlsString) { + literalValue = ((NlsString) literalValue).getValue(); + } + // Calcite may leave a dynamic parameter as ANY. RexBuilder then guesses a + // type from the Java value and can turn an exact decimal into BIGINT. + // Preserve the SQL literal's own precision and scale in that case. + RelDataType valueType = converted.getType().getSqlTypeName() == SqlTypeName.ANY + ? source.createSqlType(rexBuilder.getTypeFactory()) : converted.getType(); + RexLiteral value = rexBuilder.makeLiteral(literalValue, valueType); + return new RexBoundDynamicParam(converted.getType(), param.getIndex(), value); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlValidator.java new file mode 100644 index 00000000000..b8f6566f910 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillSqlValidator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; +import org.apache.calcite.sql.validate.SqlValidatorImpl; +import org.apache.calcite.sql.validate.SqlValidatorScope; + +/** Gives bound plan-cache parameters a type during SQL validation. */ +final class DrillSqlValidator extends SqlValidatorImpl { + DrillSqlValidator(SqlOperatorTable operators, SqlValidatorCatalogReader catalog, + RelDataTypeFactory types, Config config) { + super(operators, catalog, types, config); + } + + private RelDataType boundType(SqlNode node) { + SqlBoundDynamicParam param = (SqlBoundDynamicParam) node; + return param.getLiteral().createSqlType(typeFactory); + } + + @Override + public RelDataType getValidatedNodeTypeIfKnown(SqlNode node) { + RelDataType type = super.getValidatedNodeTypeIfKnown(node); + if (type == null && node instanceof SqlBoundDynamicParam) { + type = boundType(node); + setValidatedNodeType(node, type); + } + return type; + } + + @Override + public RelDataType deriveType(SqlValidatorScope scope, SqlNode node) { + if (node instanceof SqlBoundDynamicParam) { + return getValidatedNodeType(node); + } + return super.deriveType(scope, node); + } + + @Override + protected void inferUnknownTypes(RelDataType inferredType, SqlValidatorScope scope, SqlNode node) { + if (node instanceof SqlBoundDynamicParam) { + // Unlike an unbound JDBC parameter, this slot already has a concrete + // non-null SQL literal. Do not reject it when the parent has no type. + getValidatedNodeType(node); + return; + } + super.inferUnknownTypes(inferredType, scope, node); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/RexBoundDynamicParam.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/RexBoundDynamicParam.java new file mode 100644 index 00000000000..e19d2411235 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/RexBoundDynamicParam.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexLiteral; + +/** A SQL parameter whose value is available during the first plan build. */ +public final class RexBoundDynamicParam extends RexDynamicParam { + private final RexLiteral literal; + + public RexBoundDynamicParam(RelDataType type, int index, RexLiteral literal) { + super(type, index); + this.literal = literal; + } + + public RexLiteral getLiteral() { + return literal; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlBoundDynamicParam.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlBoundDynamicParam.java new file mode 100644 index 00000000000..b1de3cbae37 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlBoundDynamicParam.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.conversion; + +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.parser.SqlParserPos; + +/** SQL parameter that retains its value for the first physical plan build. */ +public final class SqlBoundDynamicParam extends SqlDynamicParam { + private final SqlLiteral literal; + + public SqlBoundDynamicParam(int index, SqlParserPos pos, SqlLiteral literal) { + super(index, pos); + this.literal = literal; + } + + public SqlLiteral getLiteral() { + return literal; + } + + @Override + public SqlBoundDynamicParam clone(SqlParserPos pos) { + return new SqlBoundDynamicParam(getIndex(), pos, literal); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java index 25ed545c687..93ee87271e0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java @@ -42,7 +42,6 @@ import org.apache.calcite.sql.util.ChainedSqlOperatorTable; import org.apache.calcite.sql.validate.SqlConformance; import org.apache.calcite.sql.validate.SqlValidator; -import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; @@ -152,7 +151,7 @@ public SqlConverter(QueryContext context) { ); this.opTab = new ChainedSqlOperatorTable(Arrays.asList(context.getDrillOperatorTable(), catalog)); this.costFactory = (settings.useDefaultCosting()) ? null : new DrillCostBase.DrillCostFactory(); - this.validator = SqlValidatorUtil.newValidator(opTab, catalog, typeFactory, + this.validator = new DrillSqlValidator(opTab, catalog, typeFactory, SqlValidator.Config.DEFAULT.withConformance(parserConfig.conformance()) .withTypeCoercionEnabled(true) .withIdentifierExpansion(true)); @@ -176,7 +175,7 @@ public SqlConverter(QueryContext context) { this.catalog = catalog; this.opTab = parent.opTab; this.planner = parent.planner; - this.validator = SqlValidatorUtil.newValidator(opTab, catalog, typeFactory, + this.validator = new DrillSqlValidator(opTab, catalog, typeFactory, SqlValidator.Config.DEFAULT.withConformance(parserConfig.conformance()) .withTypeCoercionEnabled(true) .withIdentifierExpansion(true)); @@ -225,7 +224,7 @@ public RelRoot toRel(final SqlNode validatedNode) { initCluster(initPlanner()); DrillViewExpander viewExpander = new DrillViewExpander(this); util.getViewExpansionContext().setViewExpander(viewExpander); - final SqlToRelConverter sqlToRelConverter = new SqlToRelConverter( + final SqlToRelConverter sqlToRelConverter = new DrillSqlToRelConverter( viewExpander, validator, catalog, cluster, DrillConvertletTable.INSTANCE, sqlToRelConverterConfig); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java index fc632df06d9..b1a3fc52888 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserSession.java @@ -309,6 +309,11 @@ public String resolveTemporaryTableName(String tableName) { return temporaryTables.get(tableName.toLowerCase()); } + /** Temporary tables can shadow names in plans shared across sessions. */ + public boolean hasTemporaryTables() { + return !temporaryTables.isEmpty(); + } + public String getOriginalTableNameFromTemporaryTable(String tableName) { for (String originalTableName : temporaryTables.keySet()) { if (temporaryTables.get(originalTableName).equals(tableName)) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java index 3fa9bcede8c..1dee205ec34 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java @@ -33,6 +33,7 @@ import org.apache.drill.exec.oauth.OAuthTokenProvider; import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.sql.PlanCache; import org.apache.drill.exec.planner.sql.DrillOperatorTable; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.rpc.control.Controller; @@ -63,6 +64,7 @@ public class DrillbitContext implements AutoCloseable { private final BootStrapContext context; private final PhysicalPlanReader reader; + private final PlanCache planCache; private final ClusterCoordinator coord; private final DataConnectionCreator connectionsPool; private final DrillbitEndpoint endpoint; @@ -122,6 +124,7 @@ public DrillbitContext( ExecConstants.STORAGE_PLUGIN_REGISTRY_IMPL, StoragePluginRegistry.class, this); reader = new PhysicalPlanReader(config, classpathScan, lpPersistence, endpoint, storagePlugins); + planCache = new PlanCache(); operatorCreatorRegistry = new OperatorCreatorRegistry(classpathScan); systemOptions = new SystemOptionManager(lpPersistence, provider, config, context.getDefinitions()); functionRegistry = new FunctionImplementationRegistry(config, classpathScan, systemOptions); @@ -252,6 +255,10 @@ public PhysicalPlanReader getPlanReader() { return reader; } + public PlanCache getPlanCache() { + return planCache; + } + public PersistentStoreProvider getStoreProvider() { return provider; } @@ -325,6 +332,7 @@ public Set> getUserConnect @Override public void close() throws Exception { + planCache.close(); getOptionManager().close(); getFunctionImplementationRegistry().close(); getRemoteFunctionRegistry().close(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 6cda5c872c0..d4502cee418 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -83,6 +83,7 @@ public static CaseInsensitiveMap createDefaultOptionDefinition @SuppressWarnings("deprecation") final OptionDefinition[] definitions = new OptionDefinition[]{ new OptionDefinition(PlannerSettings.CONSTANT_FOLDING), + new OptionDefinition(PlannerSettings.PLAN_CACHE), new OptionDefinition(PlannerSettings.EXCHANGE), new OptionDefinition(PlannerSettings.HASHAGG), new OptionDefinition(PlannerSettings.STREAMAGG), diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java index 02b0890cc99..472c6f94f77 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileWrapper.java @@ -179,6 +179,10 @@ public QueryProfile getProfile() { return profile; } + public String getPlanCacheStatus() { + return profile.hasPlanCacheHit() ? (profile.getPlanCacheHit() ? "Yes" : "No") : "Unknown"; + } + public String getProfileDuration() { return (new SimpleDurationFormat(profile.getStart(), profile.getEnd())).verbose(); } @@ -377,6 +381,9 @@ private void generateOpMap(String plan) { //[e.g ] operatorLine = "01-03 Flatten(flattenField=[$1]) : rowType = RecordType(ANY rfsSpecCode, ..." String[] operatorLine = plan.split("\\n"); for (String line : operatorLine) { + if (line.startsWith("Parameters: ")) { + continue; + } String[] lineToken = line.split("\\s+", 3); if (lineToken.length < 2) { continue; //Skip due to possible invalid entry diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/PlanCacheTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/PlanCacheTable.java new file mode 100644 index 00000000000..89572f31c33 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/PlanCacheTable.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store; + +import java.util.Objects; + +/** Immutable plan-cache identity and compatibility version of one resolved table. */ +public final class PlanCacheTable { + private final String identifier; + private final String version; + + public PlanCacheTable(String identifier, String version) { + this.identifier = Objects.requireNonNull(identifier, "identifier"); + this.version = Objects.requireNonNull(version, "version"); + } + + public String getIdentifier() { + return identifier; + } + + public String getVersion() { + return version; + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java index b3bf92813c9..f61f4dab4ef 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -28,6 +28,7 @@ import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.planner.PlannerPhase; import org.apache.drill.exec.metastore.MetadataProviderManager; import org.apache.drill.exec.server.options.SessionOptionManager; @@ -71,6 +72,30 @@ public interface StoragePlugin extends SchemaFactory, AutoCloseable { */ boolean supportsWrite(); + /** + * Opts this plugin into plan caching. The engine validates query context and + * rebinds expression parameters; the plugin must provide a plan-compatible + * table version and rebuild value-dependent scan state for each execution. + */ + default boolean supportPlanCache() { + return false; + } + + /** + * Returns a stable table identifier and a version that changes whenever the + * cached operator structure is no longer compatible. Data-only changes may + * keep the version only if the scan reads current data on deserialization. + * Return null when that contract cannot be met. + */ + default PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { + return null; + } + + /** Reads the current version by a previously resolved identifier, without schema traversal. */ + default String planCacheTableVersion(String identifier) throws IOException { + return null; + } + /** * Indicates if Drill can insert to a table to this plugin. */ diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 5665ae72a4a..e4f99b026b3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -51,17 +51,20 @@ import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.planner.PlannerPhase; +import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.SessionOptionManager; import org.apache.drill.exec.store.AbstractStoragePlugin; import org.apache.drill.exec.store.ClassPathFileSystem; import org.apache.drill.exec.store.LocalSyncableFileSystem; +import org.apache.drill.exec.store.PlanCacheTable; import org.apache.drill.exec.store.SchemaConfig; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.sftp.SFTPFileSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -330,6 +333,55 @@ public FormatPlugin getFormatPlugin(String name) { return formatCreator.getFormatPluginByName(name); } + @Override + public boolean supportPlanCache() { + return true; + } + + @Override + public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { + if (!(selection instanceof FormatSelection)) { + return null; + } + FormatSelection formatSelection = (FormatSelection) selection; + FileSelection files = formatSelection.getSelection(); + FormatPlugin format = getFormatPlugin(formatSelection.getFormat()); + if (files == null || files.getSelectionRoot() == null + || format == null || !format.supportPlanCache()) { + return null; + } + String version = format.planCacheTableVersion(files); + if (version == null) { + return null; + } + String formatName = format.getName(); + if (formatName == null || formatName.isEmpty()) { + return null; + } + String identifier = formatName.length() + ":" + formatName + files.getSelectionRoot(); + return new PlanCacheTable(identifier, version); + } + + @Override + public String planCacheTableVersion(String identifier) throws IOException { + int separator = identifier.indexOf(':'); + if (separator < 1) { + return null; + } + try { + int nameLength = Integer.parseInt(identifier.substring(0, separator)); + if (nameLength < 1 || nameLength >= identifier.length() - separator - 1) { + return null; + } + int nameEnd = separator + 1 + nameLength; + FormatPlugin format = getFormatPlugin(identifier.substring(separator + 1, nameEnd)); + return format == null || !format.supportPlanCache() ? null + : format.planCacheTableVersion(new Path(identifier.substring(nameEnd))); + } catch (NumberFormatException e) { + return null; + } + } + /** * If format plugin configuration is for named format plugin, will return * format plugin from pre-loaded list by name. For other cases will try to diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java index 364af39bc6c..ba88f33d0dc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java @@ -48,6 +48,21 @@ public interface FormatPlugin { boolean supportsWrite(); + /** Whether scans of this format can be rebuilt for a cached plan. */ + default boolean supportPlanCache() { + return false; + } + + /** Version of the table definition that determines physical-plan compatibility. */ + default String planCacheTableVersion(FileSelection selection) throws IOException { + return null; + } + + /** Reads the current version of a previously resolved table without rebuilding its schema. */ + default String planCacheTableVersion(Path tablePath) throws IOException { + return null; + } + /** * Indicates whether this FormatPlugin supports auto-partitioning for CTAS statements * @return true if auto-partitioning is supported diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java index 1d4fff8df14..70743476e0d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java @@ -75,7 +75,7 @@ public boolean needsFinalColumnReordering() { @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("groupScan", groupScan); + return super.explainTerms(pw).item("groupScan", groupScan.getExplainDigest()); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index a099b96b123..7fb7d764135 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -759,6 +759,7 @@ public void close() { drillbitContext.getWorkBus().removeFragmentStatusListener(queryId); drillbitContext.getClusterCoordinator().removeDrillbitStatusListener(queryManager.getDrillbitStatusListener()); + final Runnable pendingPlanCacheInsert = queryContext.takePendingPlanCacheInsert(); suppressingClose(queryContext); /* @@ -800,6 +801,10 @@ public void close() { queryManager.writeFinalProfile(uex); } + if (resultState == QueryState.COMPLETED && pendingPlanCacheInsert != null) { + pendingPlanCacheInsert.run(); + } + /* * If sending the result fails, we don't really have any way to modify the result we tried to send; * it is possible it got sent but the result came from a later part of the code path. It is also diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java index a919ed90059..10147955dd2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java @@ -372,6 +372,8 @@ private QueryProfile getQueryProfile(UserException ex) { .setQueueName(queueName == null ? "-" : queueName) .setOptionsJson(getQueryOptionsAsJson()); + profileBuilder.setPlanCacheHit(queryCtx.isPlanCacheHit()); + if (ex != null) { profileBuilder.setError(ex.getMessage(false)); profileBuilder.setVerboseError(ex.getVerboseMessage(false)); diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 887fa15bff4..6176a873053 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -663,6 +663,7 @@ drill.exec.options: { planner.disable_exchanges: false, planner.enable_broadcast_join: true, planner.enable_constant_folding: true, + planner.enable_plan_cache: false, planner.enable_decimal_data_type: true, planner.enable_demux_exchange: false, planner.enable_file_listing_limit0_optimization: false, diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl index 513e22cf592..129d79f0b02 100644 --- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl +++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl @@ -313,6 +313,7 @@ State Foreman Total Fragments + Plan Cache Hit <#if queued> Total Cost Queue @@ -324,6 +325,7 @@ ${model.getQueryStateDisplayName()} ${model.getProfile().getForeman().getAddress()} ${model.getProfile().getTotalFragments()} + ${model.getPlanCacheStatus()} <#if queued> ${model.getProfile().getTotalCost()} ${queueName} diff --git a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 index 53b39bd98a7..5288057d62a 100644 --- a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 +++ b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprLexer.g4 @@ -94,6 +94,8 @@ Bool | 'false' ; +BoundDynamicParam : 'bound_dynamic_param'; + Number : Int ('.' Digit*)? (('e' | 'E') ('+' | '-')? Digit*)? ; diff --git a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 index d868cff3e7c..5d2dcae9288 100644 --- a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 +++ b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 @@ -324,7 +324,11 @@ arraySegment returns [PathSegment seg] ; lookup returns [LogicalExpression e] - : functionCall {$e = $functionCall.e ;} + : BoundDynamicParam OParen Number Comma dataType Comma boundLiteralValue CParen { + $e = ValueExpressions.getBoundDynamicParam(Integer.parseInt($Number.text), + $dataType.type, $boundLiteralValue.value); + } + | functionCall {$e = $functionCall.e ;} | convertCall {$e = $convertCall.e; } | anyValueCall {$e = $anyValueCall.e; } | castCall {$e = $castCall.e; } @@ -333,6 +337,14 @@ lookup returns [LogicalExpression e] | OParen expression CParen {$e = $expression.e; } | SingleQuote Identifier SingleQuote {$e = new SchemaPath($Identifier.text, pos($Identifier) ); } ; + +boundLiteralValue returns [String value] + : s=String { $value = $s.text; } + | b=Bool { $value = $b.text; } + | sign=(Plus|Minus)? n=Number { + $value = ($sign == null ? "" : $sign.text) + $n.text; + } + ; diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 4133f174a9d..838f0cda283 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -45,7 +45,50 @@ public class ExpressionStringBuilder extends AbstractExprVisitor{ + public static final String BOUND_DYNAMIC_PARAM = "bound_dynamic_param"; + static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder(); + private final boolean explainParameters; + + public ExpressionStringBuilder() { + this(false); + } + + private ExpressionStringBuilder(boolean explainParameters) { + this.explainParameters = explainParameters; + } + + private boolean startBoundLiteral(LiteralExpression expression, StringBuilder sb) { + int index = expression.getDynamicParamIndex(); + if (index >= 0) { + if (explainParameters) { + sb.append('?').append(index); + return true; + } + MajorType type = expression.getMajorType(); + sb.append(BOUND_DYNAMIC_PARAM).append('(').append(index).append(", ") + .append(type.getMinorType().name()); + switch (type.getMinorType()) { + case VARCHAR: + sb.append('(').append(type.getPrecision()).append(')'); + break; + case VARDECIMAL: + sb.append('(').append(type.getPrecision()).append(", ") + .append(type.getScale()).append(')'); + break; + default: + break; + } + sb.append(", "); + } + return false; + } + + private static void endBoundLiteral(LiteralExpression expression, StringBuilder sb) { + if (expression.getDynamicParamIndex() >= 0) { + sb.append(')'); + } + } public static String toString(LogicalExpression expr) { StringBuilder sb = new StringBuilder(); @@ -53,6 +96,13 @@ public static String toString(LogicalExpression expr) { return sb.toString(); } + /** Renders parameter slots without exposing the literal stored in a cached plan. */ + public static String toExplainString(LogicalExpression expr) { + StringBuilder sb = new StringBuilder(); + expr.accept(new ExpressionStringBuilder(true), sb); + return sb.toString(); + } + public static void toString(LogicalExpression expr, StringBuilder sb) { expr.accept(INSTANCE, sb); } @@ -147,99 +197,160 @@ public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeExc @Override public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws RuntimeException { - sb.append(lExpr.getLong()); + if (startBoundLiteral(lExpr, sb)) { + return null; + } + if (lExpr.getDynamicParamIndex() < 0) { + // Small BIGINT values otherwise parse back as INT during a plan round trip. + sb.append("cast(").append(lExpr.getLong()).append(" as BIGINT)"); + } else { + sb.append(lExpr.getLong()); + } + endBoundLiteral(lExpr, sb); return null; } @Override public Void visitDateConstant(DateExpression lExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(lExpr, sb)) { + return null; + } sb.append("cast( "); sb.append(lExpr.getDate()); sb.append(" as DATE)"); + endBoundLiteral(lExpr, sb); return null; } @Override public Void visitTimeConstant(TimeExpression lExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(lExpr, sb)) { + return null; + } sb.append("cast( "); sb.append(lExpr.getTime()); sb.append(" as TIME)"); + endBoundLiteral(lExpr, sb); return null; } @Override public Void visitTimeStampConstant(TimeStampExpression lExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(lExpr, sb)) { + return null; + } sb.append("cast( "); sb.append(lExpr.getTimeStamp()); sb.append(" as TIMESTAMP)"); + endBoundLiteral(lExpr, sb); return null; } @Override public Void visitIntervalYearConstant(IntervalYearExpression lExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(lExpr, sb)) { + return null; + } sb.append("cast( '"); sb.append(Period.months(lExpr.getIntervalYear()).toString()); sb.append("' as INTERVALYEAR)"); + endBoundLiteral(lExpr, sb); return null; } @Override public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(lExpr, sb)) { + return null; + } sb.append("cast( '"); sb.append(Period.days(lExpr.getIntervalDay()).plusMillis(lExpr.getIntervalMillis()).toString()); sb.append("' as INTERVALDAY)"); + endBoundLiteral(lExpr, sb); return null; } @Override public Void visitDecimal9Constant(Decimal9Expression decExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(decExpr, sb)) { + return null; + } BigDecimal value = new BigDecimal(decExpr.getIntFromDecimal()); sb.append((value.setScale(decExpr.getScale())).toString()); + endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDecimal18Constant(Decimal18Expression decExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(decExpr, sb)) { + return null; + } BigDecimal value = new BigDecimal(decExpr.getLongFromDecimal()); sb.append((value.setScale(decExpr.getScale())).toString()); + endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDecimal28Constant(Decimal28Expression decExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(decExpr, sb)) { + return null; + } sb.append(decExpr.toString()); + endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDecimal38Constant(Decimal38Expression decExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(decExpr, sb)) { + return null; + } sb.append(decExpr.getBigDecimal().toString()); + endBoundLiteral(decExpr, sb); return null; } @Override public Void visitVarDecimalConstant(VarDecimalExpression decExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(decExpr, sb)) { + return null; + } sb.append(decExpr.getBigDecimal().toString()); + endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDoubleConstant(DoubleExpression dExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(dExpr, sb)) { + return null; + } sb.append(dExpr.getDouble()); + endBoundLiteral(dExpr, sb); return null; } @Override public Void visitBooleanConstant(BooleanExpression e, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(e, sb)) { + return null; + } sb.append(e.getBoolean()); + endBoundLiteral(e, sb); return null; } @Override public Void visitQuotedStringConstant(QuotedString e, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(e, sb)) { + return null; + } sb.append("'"); sb.append(escapeSingleQuote(e.value)); sb.append("'"); + endBoundLiteral(e, sb); return null; } @@ -324,19 +435,31 @@ public Void visitCastExpression(CastExpression e, StringBuilder sb) throws Runti @Override public Void visitFloatConstant(FloatExpression fExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(fExpr, sb)) { + return null; + } sb.append(fExpr.getFloat()); + endBoundLiteral(fExpr, sb); return null; } @Override public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(intExpr, sb)) { + return null; + } sb.append(intExpr.getInt()); + endBoundLiteral(intExpr, sb); return null; } @Override public Void visitNullConstant(TypedNullConstant e, StringBuilder sb) throws RuntimeException { + if (startBoundLiteral(e, sb)) { + return null; + } sb.append("NULL"); + endBoundLiteral(e, sb); return null; } diff --git a/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java b/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java new file mode 100644 index 00000000000..73258f32588 --- /dev/null +++ b/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.expression; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +/** A literal that may carry a bound SQL parameter slot in a cached plan. */ +public abstract class LiteralExpression extends LogicalExpressionBase { + private int dynamicParamIndex = -1; + + protected LiteralExpression(ExpressionPosition position) { + super(position); + } + + @JsonIgnore + public int getDynamicParamIndex() { + return dynamicParamIndex; + } + + public void setDynamicParamIndex(int dynamicParamIndex) { + if (dynamicParamIndex < 0) { + throw new IllegalArgumentException("Dynamic parameter index must be non-negative"); + } + this.dynamicParamIndex = dynamicParamIndex; + } +} diff --git a/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java b/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java index 3e773ffbe65..415a6ca2331 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java +++ b/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java @@ -25,7 +25,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; -public class TypedNullConstant extends LogicalExpressionBase { +public class TypedNullConstant extends LiteralExpression { private final MajorType type; diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java index 2f04a588ccd..01597dc59bb 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java @@ -95,6 +95,49 @@ public static LogicalExpression getVarDecimal(BigDecimal input, int precision, i return new VarDecimalExpression(input, precision, scale, ExpressionPosition.UNKNOWN); } + /** Restores the exact literal type carried by a serialized bound parameter. */ + public static LiteralExpression getBoundDynamicParam(int index, MajorType type, String value) { + LiteralExpression expression; + switch (type.getMinorType()) { + case INT: + expression = (LiteralExpression) getInt(Integer.parseInt(value)); + break; + case BIGINT: + expression = (LiteralExpression) getBigInt(Long.parseLong(value)); + break; + case FLOAT4: + expression = (LiteralExpression) getFloat4(Float.parseFloat(value)); + break; + case FLOAT8: + expression = (LiteralExpression) getFloat8(Double.parseDouble(value)); + break; + case BIT: + if (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value)) { + throw new IllegalArgumentException("Invalid boolean parameter value"); + } + expression = (LiteralExpression) getBit(Boolean.parseBoolean(value)); + break; + case VARCHAR: + expression = (LiteralExpression) getChar(value, type.getPrecision()); + break; + case VARDECIMAL: + BigDecimal decimal = new BigDecimal(value); + if (decimal.scale() != type.getScale() || decimal.precision() > type.getPrecision()) { + throw new IllegalArgumentException("Invalid decimal parameter value"); + } + expression = (LiteralExpression) getVarDecimal(decimal, + type.getPrecision(), type.getScale()); + break; + default: + throw new IllegalArgumentException("Unsupported bound parameter type: " + type.getMinorType()); + } + if (!expression.getMajorType().equals(type)) { + throw new IllegalArgumentException("Bound parameter type changed"); + } + expression.setDynamicParamIndex(index); + return expression; + } + public static LogicalExpression getNumericExpression(String sign, String s, ExpressionPosition ep) { String numStr = (sign == null) ? s : sign+s; try { @@ -120,7 +163,7 @@ public static LogicalExpression getParameterExpression(String name, MajorType ty return new ParameterExpression(name, type, ExpressionPosition.UNKNOWN); } - protected static abstract class ValueExpression extends LogicalExpressionBase { + protected static abstract class ValueExpression extends LiteralExpression { public final V value; protected ValueExpression(String value, ExpressionPosition pos) { @@ -165,7 +208,7 @@ public boolean getBoolean() { } } - public static class FloatExpression extends LogicalExpressionBase { + public static class FloatExpression extends LiteralExpression { private final float f; private static final MajorType FLOAT_CONSTANT = Types.required(MinorType.FLOAT4); @@ -195,7 +238,7 @@ public Iterator iterator() { } } - public static class IntExpression extends LogicalExpressionBase { + public static class IntExpression extends LiteralExpression { private static final MajorType INT_CONSTANT = Types.required(MinorType.INT); @@ -226,7 +269,7 @@ public Iterator iterator() { } } - public static class Decimal9Expression extends LogicalExpressionBase { + public static class Decimal9Expression extends LiteralExpression { private final int decimal; private final int scale; @@ -267,7 +310,7 @@ public Iterator iterator() { } } - public static class Decimal18Expression extends LogicalExpressionBase { + public static class Decimal18Expression extends LiteralExpression { private final long decimal; private final int scale; @@ -308,7 +351,7 @@ public Iterator iterator() { } } - public static class Decimal28Expression extends LogicalExpressionBase { + public static class Decimal28Expression extends LiteralExpression { private final BigDecimal bigDecimal; @@ -337,7 +380,7 @@ public Iterator iterator() { } } - public static class Decimal38Expression extends LogicalExpressionBase { + public static class Decimal38Expression extends LiteralExpression { private final BigDecimal bigDecimal; @@ -366,7 +409,7 @@ public Iterator iterator() { } } - public static class VarDecimalExpression extends LogicalExpressionBase { + public static class VarDecimalExpression extends LiteralExpression { private final BigDecimal bigDecimal; private final int precision; @@ -405,7 +448,7 @@ public Iterator iterator() { } } - public static class DoubleExpression extends LogicalExpressionBase { + public static class DoubleExpression extends LiteralExpression { private final double d; private static final MajorType DOUBLE_CONSTANT = Types.required(MinorType.FLOAT8); @@ -435,7 +478,7 @@ public Iterator iterator() { } } - public static class LongExpression extends LogicalExpressionBase { + public static class LongExpression extends LiteralExpression { private static final MajorType LONG_CONSTANT = Types.required(MinorType.BIGINT); @@ -470,7 +513,7 @@ public Iterator iterator() { } } - public static class DateExpression extends LogicalExpressionBase { + public static class DateExpression extends LiteralExpression { private static final MajorType DATE_CONSTANT = Types.required(MinorType.DATE); @@ -505,7 +548,7 @@ public Iterator iterator() { } } - public static class TimeExpression extends LogicalExpressionBase { + public static class TimeExpression extends LiteralExpression { private static final MajorType TIME_CONSTANT = Types.required(MinorType.TIME); @@ -540,7 +583,7 @@ public Iterator iterator() { } } - public static class TimeStampExpression extends LogicalExpressionBase { + public static class TimeStampExpression extends LiteralExpression { private static final MajorType TIMESTAMP_CONSTANT = Types.required(MinorType.TIMESTAMP); @@ -575,7 +618,7 @@ public Iterator iterator() { } } - public static class IntervalYearExpression extends LogicalExpressionBase { + public static class IntervalYearExpression extends LiteralExpression { private static final MajorType INTERVALYEAR_CONSTANT = Types.required(MinorType.INTERVALYEAR); @@ -610,7 +653,7 @@ public Iterator iterator() { } } - public static class IntervalDayExpression extends LogicalExpressionBase { + public static class IntervalDayExpression extends LiteralExpression { private static final MajorType INTERVALDAY_CONSTANT = Types.required(MinorType.INTERVALDAY); private static final long MILLIS_IN_DAY = 1000 * 60 * 60 * 24; diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java index 296fc710cd2..45d0ff063a0 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/SchemaUserBitShared.java @@ -1832,6 +1832,8 @@ public void writeTo(io.protostuff.Output output, org.apache.drill.exec.proto.Use output.writeInt32(23, message.getAutoLimit(), false); for(String scannedPlugins : message.getScannedPluginsList()) output.writeString(24, scannedPlugins, true); + if(message.hasPlanCacheHit()) + output.writeBool(25, message.getPlanCacheHit(), false); } public boolean isInitialized(org.apache.drill.exec.proto.UserBitShared.QueryProfile message) { @@ -1946,6 +1948,9 @@ public void mergeFrom(io.protostuff.Input input, org.apache.drill.exec.proto.Use case 24: builder.addScannedPlugins(input.readString()); break; + case 25: + builder.setPlanCacheHit(input.readBool()); + break; default: input.handleUnknownField(number, this); } @@ -2010,6 +2015,7 @@ public static java.lang.String getFieldName(int number) case 22: return "queryId"; case 23: return "autoLimit"; case 24: return "scannedPlugins"; + case 25: return "planCacheHit"; default: return null; } } @@ -2045,6 +2051,7 @@ public static int getFieldNumber(java.lang.String name) fieldMap.put("queryId", 22); fieldMap.put("autoLimit", 23); fieldMap.put("scannedPlugins", 24); + fieldMap.put("planCacheHit", 25); } } diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java index 622490b67f1..85cbdec9b13 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java @@ -15955,6 +15955,17 @@ org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfileOrBuilder getFragm */ com.google.protobuf.ByteString getScannedPluginsBytes(int index); + + /** + * optional bool plan_cache_hit = 25; + * @return Whether the planCacheHit field is set. + */ + boolean hasPlanCacheHit(); + /** + * optional bool plan_cache_hit = 25; + * @return The planCacheHit. + */ + boolean getPlanCacheHit(); } /** * Protobuf type {@code exec.shared.QueryProfile} @@ -15982,7 +15993,8 @@ private QueryProfile() { optionsJson_ = ""; queueName_ = "-"; queryId_ = ""; - scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; + scannedPlugins_ = + com.google.protobuf.LazyStringArrayList.emptyList(); } @java.lang.Override @@ -15992,11 +16004,6 @@ protected java.lang.Object newInstance( return new QueryProfile(); } - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { return org.apache.drill.exec.proto.UserBitShared.internal_static_exec_shared_QueryProfile_descriptor; @@ -16038,7 +16045,7 @@ public org.apache.drill.exec.proto.UserBitShared.QueryIdOrBuilder getIdOrBuilder } public static final int TYPE_FIELD_NUMBER = 2; - private int type_; + private int type_ = 1; /** * optional .exec.shared.QueryType type = 2; * @return Whether the type field is set. @@ -16051,13 +16058,12 @@ public org.apache.drill.exec.proto.UserBitShared.QueryIdOrBuilder getIdOrBuilder * @return The type. */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryType getType() { - @SuppressWarnings("deprecation") - org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.valueOf(type_); + org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.forNumber(type_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryType.SQL : result; } public static final int START_FIELD_NUMBER = 3; - private long start_; + private long start_ = 0L; /** * optional int64 start = 3; * @return Whether the start field is set. @@ -16076,7 +16082,7 @@ public long getStart() { } public static final int END_FIELD_NUMBER = 4; - private long end_; + private long end_ = 0L; /** * optional int64 end = 4; * @return Whether the end field is set. @@ -16095,7 +16101,8 @@ public long getEnd() { } public static final int QUERY_FIELD_NUMBER = 5; - private volatile java.lang.Object query_; + @SuppressWarnings("serial") + private volatile java.lang.Object query_ = ""; /** * optional string query = 5; * @return Whether the query field is set. @@ -16143,7 +16150,8 @@ public java.lang.String getQuery() { } public static final int PLAN_FIELD_NUMBER = 6; - private volatile java.lang.Object plan_; + @SuppressWarnings("serial") + private volatile java.lang.Object plan_ = ""; /** * optional string plan = 6; * @return Whether the plan field is set. @@ -16217,7 +16225,7 @@ public org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpointOrBuilder } public static final int STATE_FIELD_NUMBER = 8; - private int state_; + private int state_ = 0; /** * optional .exec.shared.QueryResult.QueryState state = 8; * @return Whether the state field is set. @@ -16230,13 +16238,12 @@ public org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpointOrBuilder * @return The state. */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState getState() { - @SuppressWarnings("deprecation") - org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.valueOf(state_); + org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.forNumber(state_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.STARTING : result; } public static final int TOTAL_FRAGMENTS_FIELD_NUMBER = 9; - private int totalFragments_; + private int totalFragments_ = 0; /** * optional int32 total_fragments = 9; * @return Whether the totalFragments field is set. @@ -16255,7 +16262,7 @@ public int getTotalFragments() { } public static final int FINISHED_FRAGMENTS_FIELD_NUMBER = 10; - private int finishedFragments_; + private int finishedFragments_ = 0; /** * optional int32 finished_fragments = 10; * @return Whether the finishedFragments field is set. @@ -16274,6 +16281,7 @@ public int getFinishedFragments() { } public static final int FRAGMENT_PROFILE_FIELD_NUMBER = 11; + @SuppressWarnings("serial") private java.util.List fragmentProfile_; /** * repeated .exec.shared.MajorFragmentProfile fragment_profile = 11; @@ -16314,7 +16322,8 @@ public org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfileOrBuilder g } public static final int USER_FIELD_NUMBER = 12; - private volatile java.lang.Object user_; + @SuppressWarnings("serial") + private volatile java.lang.Object user_ = "-"; /** * optional string user = 12 [default = "-"]; * @return Whether the user field is set. @@ -16362,7 +16371,8 @@ public java.lang.String getUser() { } public static final int ERROR_FIELD_NUMBER = 13; - private volatile java.lang.Object error_; + @SuppressWarnings("serial") + private volatile java.lang.Object error_ = ""; /** * optional string error = 13; * @return Whether the error field is set. @@ -16410,7 +16420,8 @@ public java.lang.String getError() { } public static final int VERBOSEERROR_FIELD_NUMBER = 14; - private volatile java.lang.Object verboseError_; + @SuppressWarnings("serial") + private volatile java.lang.Object verboseError_ = ""; /** * optional string verboseError = 14; * @return Whether the verboseError field is set. @@ -16458,7 +16469,8 @@ public java.lang.String getVerboseError() { } public static final int ERROR_ID_FIELD_NUMBER = 15; - private volatile java.lang.Object errorId_; + @SuppressWarnings("serial") + private volatile java.lang.Object errorId_ = ""; /** * optional string error_id = 15; * @return Whether the errorId field is set. @@ -16506,7 +16518,8 @@ public java.lang.String getErrorId() { } public static final int ERROR_NODE_FIELD_NUMBER = 16; - private volatile java.lang.Object errorNode_; + @SuppressWarnings("serial") + private volatile java.lang.Object errorNode_ = ""; /** * optional string error_node = 16; * @return Whether the errorNode field is set. @@ -16554,7 +16567,8 @@ public java.lang.String getErrorNode() { } public static final int OPTIONS_JSON_FIELD_NUMBER = 17; - private volatile java.lang.Object optionsJson_; + @SuppressWarnings("serial") + private volatile java.lang.Object optionsJson_ = ""; /** * optional string options_json = 17; * @return Whether the optionsJson field is set. @@ -16602,7 +16616,7 @@ public java.lang.String getOptionsJson() { } public static final int PLANEND_FIELD_NUMBER = 18; - private long planEnd_; + private long planEnd_ = 0L; /** * optional int64 planEnd = 18; * @return Whether the planEnd field is set. @@ -16621,7 +16635,7 @@ public long getPlanEnd() { } public static final int QUEUEWAITEND_FIELD_NUMBER = 19; - private long queueWaitEnd_; + private long queueWaitEnd_ = 0L; /** * optional int64 queueWaitEnd = 19; * @return Whether the queueWaitEnd field is set. @@ -16640,7 +16654,7 @@ public long getQueueWaitEnd() { } public static final int TOTAL_COST_FIELD_NUMBER = 20; - private double totalCost_; + private double totalCost_ = 0D; /** * optional double total_cost = 20; * @return Whether the totalCost field is set. @@ -16659,7 +16673,8 @@ public double getTotalCost() { } public static final int QUEUE_NAME_FIELD_NUMBER = 21; - private volatile java.lang.Object queueName_; + @SuppressWarnings("serial") + private volatile java.lang.Object queueName_ = "-"; /** * optional string queue_name = 21 [default = "-"]; * @return Whether the queueName field is set. @@ -16707,7 +16722,8 @@ public java.lang.String getQueueName() { } public static final int QUERYID_FIELD_NUMBER = 22; - private volatile java.lang.Object queryId_; + @SuppressWarnings("serial") + private volatile java.lang.Object queryId_ = ""; /** * optional string queryId = 22; * @return Whether the queryId field is set. @@ -16755,7 +16771,7 @@ public java.lang.String getQueryId() { } public static final int AUTOLIMIT_FIELD_NUMBER = 23; - private int autoLimit_; + private int autoLimit_ = 0; /** * optional int32 autoLimit = 23; * @return Whether the autoLimit field is set. @@ -16774,7 +16790,9 @@ public int getAutoLimit() { } public static final int SCANNED_PLUGINS_FIELD_NUMBER = 24; - private com.google.protobuf.LazyStringList scannedPlugins_; + @SuppressWarnings("serial") + private com.google.protobuf.LazyStringArrayList scannedPlugins_ = + com.google.protobuf.LazyStringArrayList.emptyList(); /** * repeated string scanned_plugins = 24; * @return A list containing the scannedPlugins. @@ -16808,6 +16826,25 @@ public java.lang.String getScannedPlugins(int index) { return scannedPlugins_.getByteString(index); } + public static final int PLAN_CACHE_HIT_FIELD_NUMBER = 25; + private boolean planCacheHit_ = false; + /** + * optional bool plan_cache_hit = 25; + * @return Whether the planCacheHit field is set. + */ + @java.lang.Override + public boolean hasPlanCacheHit() { + return ((bitField0_ & 0x00400000) != 0); + } + /** + * optional bool plan_cache_hit = 25; + * @return The planCacheHit. + */ + @java.lang.Override + public boolean getPlanCacheHit() { + return planCacheHit_; + } + private byte memoizedIsInitialized = -1; @java.lang.Override public final boolean isInitialized() { @@ -16894,6 +16931,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) for (int i = 0; i < scannedPlugins_.size(); i++) { com.google.protobuf.GeneratedMessageV3.writeString(output, 24, scannedPlugins_.getRaw(i)); } + if (((bitField0_ & 0x00400000) != 0)) { + output.writeBool(25, planCacheHit_); + } getUnknownFields().writeTo(output); } @@ -16993,6 +17033,10 @@ public int getSerializedSize() { size += dataSize; size += 2 * getScannedPluginsList().size(); } + if (((bitField0_ & 0x00400000) != 0)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(25, planCacheHit_); + } size += getUnknownFields().getSerializedSize(); memoizedSize = size; return size; @@ -17121,6 +17165,11 @@ public boolean equals(final java.lang.Object obj) { } if (!getScannedPluginsList() .equals(other.getScannedPluginsList())) return false; + if (hasPlanCacheHit() != other.hasPlanCacheHit()) return false; + if (hasPlanCacheHit()) { + if (getPlanCacheHit() + != other.getPlanCacheHit()) return false; + } if (!getUnknownFields().equals(other.getUnknownFields())) return false; return true; } @@ -17233,6 +17282,11 @@ public int hashCode() { hash = (37 * hash) + SCANNED_PLUGINS_FIELD_NUMBER; hash = (53 * hash) + getScannedPluginsList().hashCode(); } + if (hasPlanCacheHit()) { + hash = (37 * hash) + PLAN_CACHE_HIT_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean( + getPlanCacheHit()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -17282,11 +17336,13 @@ public static org.apache.drill.exec.proto.UserBitShared.QueryProfile parseFrom( return com.google.protobuf.GeneratedMessageV3 .parseWithIOException(PARSER, input, extensionRegistry); } + public static org.apache.drill.exec.proto.UserBitShared.QueryProfile parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException { return com.google.protobuf.GeneratedMessageV3 .parseDelimitedWithIOException(PARSER, input); } + public static org.apache.drill.exec.proto.UserBitShared.QueryProfile parseDelimitedFrom( java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -17369,34 +17425,25 @@ private void maybeForceBuilderInitialization() { @java.lang.Override public Builder clear() { super.clear(); - if (idBuilder_ == null) { - id_ = null; - } else { - idBuilder_.clear(); + bitField0_ = 0; + id_ = null; + if (idBuilder_ != null) { + idBuilder_.dispose(); + idBuilder_ = null; } - bitField0_ = (bitField0_ & ~0x00000001); type_ = 1; - bitField0_ = (bitField0_ & ~0x00000002); start_ = 0L; - bitField0_ = (bitField0_ & ~0x00000004); end_ = 0L; - bitField0_ = (bitField0_ & ~0x00000008); query_ = ""; - bitField0_ = (bitField0_ & ~0x00000010); plan_ = ""; - bitField0_ = (bitField0_ & ~0x00000020); - if (foremanBuilder_ == null) { - foreman_ = null; - } else { - foremanBuilder_.clear(); + foreman_ = null; + if (foremanBuilder_ != null) { + foremanBuilder_.dispose(); + foremanBuilder_ = null; } - bitField0_ = (bitField0_ & ~0x00000040); state_ = 0; - bitField0_ = (bitField0_ & ~0x00000080); totalFragments_ = 0; - bitField0_ = (bitField0_ & ~0x00000100); finishedFragments_ = 0; - bitField0_ = (bitField0_ & ~0x00000200); if (fragmentProfileBuilder_ == null) { fragmentProfile_ = java.util.Collections.emptyList(); } else { @@ -17405,31 +17452,20 @@ public Builder clear() { } bitField0_ = (bitField0_ & ~0x00000400); user_ = "-"; - bitField0_ = (bitField0_ & ~0x00000800); error_ = ""; - bitField0_ = (bitField0_ & ~0x00001000); verboseError_ = ""; - bitField0_ = (bitField0_ & ~0x00002000); errorId_ = ""; - bitField0_ = (bitField0_ & ~0x00004000); errorNode_ = ""; - bitField0_ = (bitField0_ & ~0x00008000); optionsJson_ = ""; - bitField0_ = (bitField0_ & ~0x00010000); planEnd_ = 0L; - bitField0_ = (bitField0_ & ~0x00020000); queueWaitEnd_ = 0L; - bitField0_ = (bitField0_ & ~0x00040000); totalCost_ = 0D; - bitField0_ = (bitField0_ & ~0x00080000); queueName_ = "-"; - bitField0_ = (bitField0_ & ~0x00100000); queryId_ = ""; - bitField0_ = (bitField0_ & ~0x00200000); autoLimit_ = 0; - bitField0_ = (bitField0_ & ~0x00400000); - scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00800000); + scannedPlugins_ = + com.google.protobuf.LazyStringArrayList.emptyList(); + planCacheHit_ = false; return this; } @@ -17456,20 +17492,37 @@ public org.apache.drill.exec.proto.UserBitShared.QueryProfile build() { @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryProfile buildPartial() { org.apache.drill.exec.proto.UserBitShared.QueryProfile result = new org.apache.drill.exec.proto.UserBitShared.QueryProfile(this); + buildPartialRepeatedFields(result); + if (bitField0_ != 0) { buildPartial0(result); } + onBuilt(); + return result; + } + + private void buildPartialRepeatedFields(org.apache.drill.exec.proto.UserBitShared.QueryProfile result) { + if (fragmentProfileBuilder_ == null) { + if (((bitField0_ & 0x00000400) != 0)) { + fragmentProfile_ = java.util.Collections.unmodifiableList(fragmentProfile_); + bitField0_ = (bitField0_ & ~0x00000400); + } + result.fragmentProfile_ = fragmentProfile_; + } else { + result.fragmentProfile_ = fragmentProfileBuilder_.build(); + } + } + + private void buildPartial0(org.apache.drill.exec.proto.UserBitShared.QueryProfile result) { int from_bitField0_ = bitField0_; int to_bitField0_ = 0; if (((from_bitField0_ & 0x00000001) != 0)) { - if (idBuilder_ == null) { - result.id_ = id_; - } else { - result.id_ = idBuilder_.build(); - } + result.id_ = idBuilder_ == null + ? id_ + : idBuilder_.build(); to_bitField0_ |= 0x00000001; } if (((from_bitField0_ & 0x00000002) != 0)) { + result.type_ = type_; to_bitField0_ |= 0x00000002; } - result.type_ = type_; if (((from_bitField0_ & 0x00000004) != 0)) { result.start_ = start_; to_bitField0_ |= 0x00000004; @@ -17479,25 +17532,23 @@ public org.apache.drill.exec.proto.UserBitShared.QueryProfile buildPartial() { to_bitField0_ |= 0x00000008; } if (((from_bitField0_ & 0x00000010) != 0)) { + result.query_ = query_; to_bitField0_ |= 0x00000010; } - result.query_ = query_; if (((from_bitField0_ & 0x00000020) != 0)) { + result.plan_ = plan_; to_bitField0_ |= 0x00000020; } - result.plan_ = plan_; if (((from_bitField0_ & 0x00000040) != 0)) { - if (foremanBuilder_ == null) { - result.foreman_ = foreman_; - } else { - result.foreman_ = foremanBuilder_.build(); - } + result.foreman_ = foremanBuilder_ == null + ? foreman_ + : foremanBuilder_.build(); to_bitField0_ |= 0x00000040; } if (((from_bitField0_ & 0x00000080) != 0)) { + result.state_ = state_; to_bitField0_ |= 0x00000080; } - result.state_ = state_; if (((from_bitField0_ & 0x00000100) != 0)) { result.totalFragments_ = totalFragments_; to_bitField0_ |= 0x00000100; @@ -17506,39 +17557,30 @@ public org.apache.drill.exec.proto.UserBitShared.QueryProfile buildPartial() { result.finishedFragments_ = finishedFragments_; to_bitField0_ |= 0x00000200; } - if (fragmentProfileBuilder_ == null) { - if (((bitField0_ & 0x00000400) != 0)) { - fragmentProfile_ = java.util.Collections.unmodifiableList(fragmentProfile_); - bitField0_ = (bitField0_ & ~0x00000400); - } - result.fragmentProfile_ = fragmentProfile_; - } else { - result.fragmentProfile_ = fragmentProfileBuilder_.build(); - } if (((from_bitField0_ & 0x00000800) != 0)) { + result.user_ = user_; to_bitField0_ |= 0x00000400; } - result.user_ = user_; if (((from_bitField0_ & 0x00001000) != 0)) { + result.error_ = error_; to_bitField0_ |= 0x00000800; } - result.error_ = error_; if (((from_bitField0_ & 0x00002000) != 0)) { + result.verboseError_ = verboseError_; to_bitField0_ |= 0x00001000; } - result.verboseError_ = verboseError_; if (((from_bitField0_ & 0x00004000) != 0)) { + result.errorId_ = errorId_; to_bitField0_ |= 0x00002000; } - result.errorId_ = errorId_; if (((from_bitField0_ & 0x00008000) != 0)) { + result.errorNode_ = errorNode_; to_bitField0_ |= 0x00004000; } - result.errorNode_ = errorNode_; if (((from_bitField0_ & 0x00010000) != 0)) { + result.optionsJson_ = optionsJson_; to_bitField0_ |= 0x00008000; } - result.optionsJson_ = optionsJson_; if (((from_bitField0_ & 0x00020000) != 0)) { result.planEnd_ = planEnd_; to_bitField0_ |= 0x00010000; @@ -17552,25 +17594,26 @@ public org.apache.drill.exec.proto.UserBitShared.QueryProfile buildPartial() { to_bitField0_ |= 0x00040000; } if (((from_bitField0_ & 0x00100000) != 0)) { + result.queueName_ = queueName_; to_bitField0_ |= 0x00080000; } - result.queueName_ = queueName_; if (((from_bitField0_ & 0x00200000) != 0)) { + result.queryId_ = queryId_; to_bitField0_ |= 0x00100000; } - result.queryId_ = queryId_; if (((from_bitField0_ & 0x00400000) != 0)) { result.autoLimit_ = autoLimit_; to_bitField0_ |= 0x00200000; } - if (((bitField0_ & 0x00800000) != 0)) { - scannedPlugins_ = scannedPlugins_.getUnmodifiableView(); - bitField0_ = (bitField0_ & ~0x00800000); + if (((from_bitField0_ & 0x00800000) != 0)) { + scannedPlugins_.makeImmutable(); + result.scannedPlugins_ = scannedPlugins_; } - result.scannedPlugins_ = scannedPlugins_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; + if (((from_bitField0_ & 0x01000000) != 0)) { + result.planCacheHit_ = planCacheHit_; + to_bitField0_ |= 0x00400000; + } + result.bitField0_ |= to_bitField0_; } @java.lang.Override @@ -17630,13 +17673,13 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile setEnd(other.getEnd()); } if (other.hasQuery()) { - bitField0_ |= 0x00000010; query_ = other.query_; + bitField0_ |= 0x00000010; onChanged(); } if (other.hasPlan()) { - bitField0_ |= 0x00000020; plan_ = other.plan_; + bitField0_ |= 0x00000020; onChanged(); } if (other.hasForeman()) { @@ -17678,33 +17721,33 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile } } if (other.hasUser()) { - bitField0_ |= 0x00000800; user_ = other.user_; + bitField0_ |= 0x00000800; onChanged(); } if (other.hasError()) { - bitField0_ |= 0x00001000; error_ = other.error_; + bitField0_ |= 0x00001000; onChanged(); } if (other.hasVerboseError()) { - bitField0_ |= 0x00002000; verboseError_ = other.verboseError_; + bitField0_ |= 0x00002000; onChanged(); } if (other.hasErrorId()) { - bitField0_ |= 0x00004000; errorId_ = other.errorId_; + bitField0_ |= 0x00004000; onChanged(); } if (other.hasErrorNode()) { - bitField0_ |= 0x00008000; errorNode_ = other.errorNode_; + bitField0_ |= 0x00008000; onChanged(); } if (other.hasOptionsJson()) { - bitField0_ |= 0x00010000; optionsJson_ = other.optionsJson_; + bitField0_ |= 0x00010000; onChanged(); } if (other.hasPlanEnd()) { @@ -17717,13 +17760,13 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile setTotalCost(other.getTotalCost()); } if (other.hasQueueName()) { - bitField0_ |= 0x00100000; queueName_ = other.queueName_; + bitField0_ |= 0x00100000; onChanged(); } if (other.hasQueryId()) { - bitField0_ |= 0x00200000; queryId_ = other.queryId_; + bitField0_ |= 0x00200000; onChanged(); } if (other.hasAutoLimit()) { @@ -17732,13 +17775,16 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile if (!other.scannedPlugins_.isEmpty()) { if (scannedPlugins_.isEmpty()) { scannedPlugins_ = other.scannedPlugins_; - bitField0_ = (bitField0_ & ~0x00800000); + bitField0_ |= 0x00800000; } else { ensureScannedPluginsIsMutable(); scannedPlugins_.addAll(other.scannedPlugins_); } onChanged(); } + if (other.hasPlanCacheHit()) { + setPlanCacheHit(other.getPlanCacheHit()); + } this.mergeUnknownFields(other.getUnknownFields()); onChanged(); return this; @@ -17912,6 +17958,11 @@ public Builder mergeFrom( scannedPlugins_.add(bs); break; } // case 194 + case 200: { + planCacheHit_ = input.readBool(); + bitField0_ |= 0x01000000; + break; + } // case 200 default: { if (!super.parseUnknownField(input, extensionRegistry, tag)) { done = true; // was an endgroup tag @@ -17959,11 +18010,11 @@ public Builder setId(org.apache.drill.exec.proto.UserBitShared.QueryId value) { throw new NullPointerException(); } id_ = value; - onChanged(); } else { idBuilder_.setMessage(value); } bitField0_ |= 0x00000001; + onChanged(); return this; } /** @@ -17973,11 +18024,11 @@ public Builder setId( org.apache.drill.exec.proto.UserBitShared.QueryId.Builder builderForValue) { if (idBuilder_ == null) { id_ = builderForValue.build(); - onChanged(); } else { idBuilder_.setMessage(builderForValue.build()); } bitField0_ |= 0x00000001; + onChanged(); return this; } /** @@ -17986,31 +18037,32 @@ public Builder setId( public Builder mergeId(org.apache.drill.exec.proto.UserBitShared.QueryId value) { if (idBuilder_ == null) { if (((bitField0_ & 0x00000001) != 0) && - id_ != null && - id_ != org.apache.drill.exec.proto.UserBitShared.QueryId.getDefaultInstance()) { - id_ = - org.apache.drill.exec.proto.UserBitShared.QueryId.newBuilder(id_).mergeFrom(value).buildPartial(); + id_ != null && + id_ != org.apache.drill.exec.proto.UserBitShared.QueryId.getDefaultInstance()) { + getIdBuilder().mergeFrom(value); } else { id_ = value; } - onChanged(); } else { idBuilder_.mergeFrom(value); } - bitField0_ |= 0x00000001; + if (id_ != null) { + bitField0_ |= 0x00000001; + onChanged(); + } return this; } /** * optional .exec.shared.QueryId id = 1; */ public Builder clearId() { - if (idBuilder_ == null) { - id_ = null; - onChanged(); - } else { - idBuilder_.clear(); - } bitField0_ = (bitField0_ & ~0x00000001); + id_ = null; + if (idBuilder_ != null) { + idBuilder_.dispose(); + idBuilder_ = null; + } + onChanged(); return this; } /** @@ -18063,8 +18115,7 @@ public org.apache.drill.exec.proto.UserBitShared.QueryIdOrBuilder getIdOrBuilder */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryType getType() { - @SuppressWarnings("deprecation") - org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.valueOf(type_); + org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.forNumber(type_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryType.SQL : result; } /** @@ -18115,8 +18166,9 @@ public long getStart() { * @return This builder for chaining. */ public Builder setStart(long value) { - bitField0_ |= 0x00000004; + start_ = value; + bitField0_ |= 0x00000004; onChanged(); return this; } @@ -18154,8 +18206,9 @@ public long getEnd() { * @return This builder for chaining. */ public Builder setEnd(long value) { - bitField0_ |= 0x00000008; + end_ = value; + bitField0_ |= 0x00000008; onChanged(); return this; } @@ -18220,11 +18273,9 @@ public java.lang.String getQuery() { */ public Builder setQuery( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000010; + if (value == null) { throw new NullPointerException(); } query_ = value; + bitField0_ |= 0x00000010; onChanged(); return this; } @@ -18233,8 +18284,8 @@ public Builder setQuery( * @return This builder for chaining. */ public Builder clearQuery() { - bitField0_ = (bitField0_ & ~0x00000010); query_ = getDefaultInstance().getQuery(); + bitField0_ = (bitField0_ & ~0x00000010); onChanged(); return this; } @@ -18245,11 +18296,9 @@ public Builder clearQuery() { */ public Builder setQueryBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000010; + if (value == null) { throw new NullPointerException(); } query_ = value; + bitField0_ |= 0x00000010; onChanged(); return this; } @@ -18304,11 +18353,9 @@ public java.lang.String getPlan() { */ public Builder setPlan( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000020; + if (value == null) { throw new NullPointerException(); } plan_ = value; + bitField0_ |= 0x00000020; onChanged(); return this; } @@ -18317,8 +18364,8 @@ public Builder setPlan( * @return This builder for chaining. */ public Builder clearPlan() { - bitField0_ = (bitField0_ & ~0x00000020); plan_ = getDefaultInstance().getPlan(); + bitField0_ = (bitField0_ & ~0x00000020); onChanged(); return this; } @@ -18329,11 +18376,9 @@ public Builder clearPlan() { */ public Builder setPlanBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000020; + if (value == null) { throw new NullPointerException(); } plan_ = value; + bitField0_ |= 0x00000020; onChanged(); return this; } @@ -18368,11 +18413,11 @@ public Builder setForeman(org.apache.drill.exec.proto.CoordinationProtos.Drillbi throw new NullPointerException(); } foreman_ = value; - onChanged(); } else { foremanBuilder_.setMessage(value); } bitField0_ |= 0x00000040; + onChanged(); return this; } /** @@ -18382,11 +18427,11 @@ public Builder setForeman( org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.Builder builderForValue) { if (foremanBuilder_ == null) { foreman_ = builderForValue.build(); - onChanged(); } else { foremanBuilder_.setMessage(builderForValue.build()); } bitField0_ |= 0x00000040; + onChanged(); return this; } /** @@ -18395,31 +18440,32 @@ public Builder setForeman( public Builder mergeForeman(org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint value) { if (foremanBuilder_ == null) { if (((bitField0_ & 0x00000040) != 0) && - foreman_ != null && - foreman_ != org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.getDefaultInstance()) { - foreman_ = - org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.newBuilder(foreman_).mergeFrom(value).buildPartial(); + foreman_ != null && + foreman_ != org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.getDefaultInstance()) { + getForemanBuilder().mergeFrom(value); } else { foreman_ = value; } - onChanged(); } else { foremanBuilder_.mergeFrom(value); } - bitField0_ |= 0x00000040; + if (foreman_ != null) { + bitField0_ |= 0x00000040; + onChanged(); + } return this; } /** * optional .exec.DrillbitEndpoint foreman = 7; */ public Builder clearForeman() { - if (foremanBuilder_ == null) { - foreman_ = null; - onChanged(); - } else { - foremanBuilder_.clear(); - } bitField0_ = (bitField0_ & ~0x00000040); + foreman_ = null; + if (foremanBuilder_ != null) { + foremanBuilder_.dispose(); + foremanBuilder_ = null; + } + onChanged(); return this; } /** @@ -18472,8 +18518,7 @@ public org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpointOrBuilder */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState getState() { - @SuppressWarnings("deprecation") - org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.valueOf(state_); + org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.forNumber(state_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.STARTING : result; } /** @@ -18524,8 +18569,9 @@ public int getTotalFragments() { * @return This builder for chaining. */ public Builder setTotalFragments(int value) { - bitField0_ |= 0x00000100; + totalFragments_ = value; + bitField0_ |= 0x00000100; onChanged(); return this; } @@ -18563,8 +18609,9 @@ public int getFinishedFragments() { * @return This builder for chaining. */ public Builder setFinishedFragments(int value) { - bitField0_ |= 0x00000200; + finishedFragments_ = value; + bitField0_ |= 0x00000200; onChanged(); return this; } @@ -18869,11 +18916,9 @@ public java.lang.String getUser() { */ public Builder setUser( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000800; + if (value == null) { throw new NullPointerException(); } user_ = value; + bitField0_ |= 0x00000800; onChanged(); return this; } @@ -18882,8 +18927,8 @@ public Builder setUser( * @return This builder for chaining. */ public Builder clearUser() { - bitField0_ = (bitField0_ & ~0x00000800); user_ = getDefaultInstance().getUser(); + bitField0_ = (bitField0_ & ~0x00000800); onChanged(); return this; } @@ -18894,11 +18939,9 @@ public Builder clearUser() { */ public Builder setUserBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000800; + if (value == null) { throw new NullPointerException(); } user_ = value; + bitField0_ |= 0x00000800; onChanged(); return this; } @@ -18953,11 +18996,9 @@ public java.lang.String getError() { */ public Builder setError( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00001000; + if (value == null) { throw new NullPointerException(); } error_ = value; + bitField0_ |= 0x00001000; onChanged(); return this; } @@ -18966,8 +19007,8 @@ public Builder setError( * @return This builder for chaining. */ public Builder clearError() { - bitField0_ = (bitField0_ & ~0x00001000); error_ = getDefaultInstance().getError(); + bitField0_ = (bitField0_ & ~0x00001000); onChanged(); return this; } @@ -18978,11 +19019,9 @@ public Builder clearError() { */ public Builder setErrorBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00001000; + if (value == null) { throw new NullPointerException(); } error_ = value; + bitField0_ |= 0x00001000; onChanged(); return this; } @@ -19037,11 +19076,9 @@ public java.lang.String getVerboseError() { */ public Builder setVerboseError( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00002000; + if (value == null) { throw new NullPointerException(); } verboseError_ = value; + bitField0_ |= 0x00002000; onChanged(); return this; } @@ -19050,8 +19087,8 @@ public Builder setVerboseError( * @return This builder for chaining. */ public Builder clearVerboseError() { - bitField0_ = (bitField0_ & ~0x00002000); verboseError_ = getDefaultInstance().getVerboseError(); + bitField0_ = (bitField0_ & ~0x00002000); onChanged(); return this; } @@ -19062,11 +19099,9 @@ public Builder clearVerboseError() { */ public Builder setVerboseErrorBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00002000; + if (value == null) { throw new NullPointerException(); } verboseError_ = value; + bitField0_ |= 0x00002000; onChanged(); return this; } @@ -19121,11 +19156,9 @@ public java.lang.String getErrorId() { */ public Builder setErrorId( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00004000; + if (value == null) { throw new NullPointerException(); } errorId_ = value; + bitField0_ |= 0x00004000; onChanged(); return this; } @@ -19134,8 +19167,8 @@ public Builder setErrorId( * @return This builder for chaining. */ public Builder clearErrorId() { - bitField0_ = (bitField0_ & ~0x00004000); errorId_ = getDefaultInstance().getErrorId(); + bitField0_ = (bitField0_ & ~0x00004000); onChanged(); return this; } @@ -19146,11 +19179,9 @@ public Builder clearErrorId() { */ public Builder setErrorIdBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00004000; + if (value == null) { throw new NullPointerException(); } errorId_ = value; + bitField0_ |= 0x00004000; onChanged(); return this; } @@ -19205,11 +19236,9 @@ public java.lang.String getErrorNode() { */ public Builder setErrorNode( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00008000; + if (value == null) { throw new NullPointerException(); } errorNode_ = value; + bitField0_ |= 0x00008000; onChanged(); return this; } @@ -19218,8 +19247,8 @@ public Builder setErrorNode( * @return This builder for chaining. */ public Builder clearErrorNode() { - bitField0_ = (bitField0_ & ~0x00008000); errorNode_ = getDefaultInstance().getErrorNode(); + bitField0_ = (bitField0_ & ~0x00008000); onChanged(); return this; } @@ -19230,11 +19259,9 @@ public Builder clearErrorNode() { */ public Builder setErrorNodeBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00008000; + if (value == null) { throw new NullPointerException(); } errorNode_ = value; + bitField0_ |= 0x00008000; onChanged(); return this; } @@ -19289,11 +19316,9 @@ public java.lang.String getOptionsJson() { */ public Builder setOptionsJson( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00010000; + if (value == null) { throw new NullPointerException(); } optionsJson_ = value; + bitField0_ |= 0x00010000; onChanged(); return this; } @@ -19302,8 +19327,8 @@ public Builder setOptionsJson( * @return This builder for chaining. */ public Builder clearOptionsJson() { - bitField0_ = (bitField0_ & ~0x00010000); optionsJson_ = getDefaultInstance().getOptionsJson(); + bitField0_ = (bitField0_ & ~0x00010000); onChanged(); return this; } @@ -19314,11 +19339,9 @@ public Builder clearOptionsJson() { */ public Builder setOptionsJsonBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00010000; + if (value == null) { throw new NullPointerException(); } optionsJson_ = value; + bitField0_ |= 0x00010000; onChanged(); return this; } @@ -19346,8 +19369,9 @@ public long getPlanEnd() { * @return This builder for chaining. */ public Builder setPlanEnd(long value) { - bitField0_ |= 0x00020000; + planEnd_ = value; + bitField0_ |= 0x00020000; onChanged(); return this; } @@ -19385,8 +19409,9 @@ public long getQueueWaitEnd() { * @return This builder for chaining. */ public Builder setQueueWaitEnd(long value) { - bitField0_ |= 0x00040000; + queueWaitEnd_ = value; + bitField0_ |= 0x00040000; onChanged(); return this; } @@ -19424,8 +19449,9 @@ public double getTotalCost() { * @return This builder for chaining. */ public Builder setTotalCost(double value) { - bitField0_ |= 0x00080000; + totalCost_ = value; + bitField0_ |= 0x00080000; onChanged(); return this; } @@ -19490,11 +19516,9 @@ public java.lang.String getQueueName() { */ public Builder setQueueName( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00100000; + if (value == null) { throw new NullPointerException(); } queueName_ = value; + bitField0_ |= 0x00100000; onChanged(); return this; } @@ -19503,8 +19527,8 @@ public Builder setQueueName( * @return This builder for chaining. */ public Builder clearQueueName() { - bitField0_ = (bitField0_ & ~0x00100000); queueName_ = getDefaultInstance().getQueueName(); + bitField0_ = (bitField0_ & ~0x00100000); onChanged(); return this; } @@ -19515,11 +19539,9 @@ public Builder clearQueueName() { */ public Builder setQueueNameBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00100000; + if (value == null) { throw new NullPointerException(); } queueName_ = value; + bitField0_ |= 0x00100000; onChanged(); return this; } @@ -19574,11 +19596,9 @@ public java.lang.String getQueryId() { */ public Builder setQueryId( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00200000; + if (value == null) { throw new NullPointerException(); } queryId_ = value; + bitField0_ |= 0x00200000; onChanged(); return this; } @@ -19587,8 +19607,8 @@ public Builder setQueryId( * @return This builder for chaining. */ public Builder clearQueryId() { - bitField0_ = (bitField0_ & ~0x00200000); queryId_ = getDefaultInstance().getQueryId(); + bitField0_ = (bitField0_ & ~0x00200000); onChanged(); return this; } @@ -19599,11 +19619,9 @@ public Builder clearQueryId() { */ public Builder setQueryIdBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00200000; + if (value == null) { throw new NullPointerException(); } queryId_ = value; + bitField0_ |= 0x00200000; onChanged(); return this; } @@ -19631,8 +19649,9 @@ public int getAutoLimit() { * @return This builder for chaining. */ public Builder setAutoLimit(int value) { - bitField0_ |= 0x00400000; + autoLimit_ = value; + bitField0_ |= 0x00400000; onChanged(); return this; } @@ -19647,12 +19666,13 @@ public Builder clearAutoLimit() { return this; } - private com.google.protobuf.LazyStringList scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; + private com.google.protobuf.LazyStringArrayList scannedPlugins_ = + com.google.protobuf.LazyStringArrayList.emptyList(); private void ensureScannedPluginsIsMutable() { - if (!((bitField0_ & 0x00800000) != 0)) { + if (!scannedPlugins_.isModifiable()) { scannedPlugins_ = new com.google.protobuf.LazyStringArrayList(scannedPlugins_); - bitField0_ |= 0x00800000; - } + } + bitField0_ |= 0x00800000; } /** * repeated string scanned_plugins = 24; @@ -19660,7 +19680,8 @@ private void ensureScannedPluginsIsMutable() { */ public com.google.protobuf.ProtocolStringList getScannedPluginsList() { - return scannedPlugins_.getUnmodifiableView(); + scannedPlugins_.makeImmutable(); + return scannedPlugins_; } /** * repeated string scanned_plugins = 24; @@ -19694,11 +19715,10 @@ public java.lang.String getScannedPlugins(int index) { */ public Builder setScannedPlugins( int index, java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - ensureScannedPluginsIsMutable(); + if (value == null) { throw new NullPointerException(); } + ensureScannedPluginsIsMutable(); scannedPlugins_.set(index, value); + bitField0_ |= 0x00800000; onChanged(); return this; } @@ -19709,11 +19729,10 @@ public Builder setScannedPlugins( */ public Builder addScannedPlugins( java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - ensureScannedPluginsIsMutable(); + if (value == null) { throw new NullPointerException(); } + ensureScannedPluginsIsMutable(); scannedPlugins_.add(value); + bitField0_ |= 0x00800000; onChanged(); return this; } @@ -19727,6 +19746,7 @@ public Builder addAllScannedPlugins( ensureScannedPluginsIsMutable(); com.google.protobuf.AbstractMessageLite.Builder.addAll( values, scannedPlugins_); + bitField0_ |= 0x00800000; onChanged(); return this; } @@ -19735,8 +19755,9 @@ public Builder addAllScannedPlugins( * @return This builder for chaining. */ public Builder clearScannedPlugins() { - scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; - bitField0_ = (bitField0_ & ~0x00800000); + scannedPlugins_ = + com.google.protobuf.LazyStringArrayList.emptyList(); + bitField0_ = (bitField0_ & ~0x00800000);; onChanged(); return this; } @@ -19747,11 +19768,50 @@ public Builder clearScannedPlugins() { */ public Builder addScannedPluginsBytes( com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - ensureScannedPluginsIsMutable(); + if (value == null) { throw new NullPointerException(); } + ensureScannedPluginsIsMutable(); scannedPlugins_.add(value); + bitField0_ |= 0x00800000; + onChanged(); + return this; + } + + private boolean planCacheHit_ ; + /** + * optional bool plan_cache_hit = 25; + * @return Whether the planCacheHit field is set. + */ + @java.lang.Override + public boolean hasPlanCacheHit() { + return ((bitField0_ & 0x01000000) != 0); + } + /** + * optional bool plan_cache_hit = 25; + * @return The planCacheHit. + */ + @java.lang.Override + public boolean getPlanCacheHit() { + return planCacheHit_; + } + /** + * optional bool plan_cache_hit = 25; + * @param value The planCacheHit to set. + * @return This builder for chaining. + */ + public Builder setPlanCacheHit(boolean value) { + + planCacheHit_ = value; + bitField0_ |= 0x01000000; + onChanged(); + return this; + } + /** + * optional bool plan_cache_hit = 25; + * @return This builder for chaining. + */ + public Builder clearPlanCacheHit() { + bitField0_ = (bitField0_ & ~0x01000000); + planCacheHit_ = false; onChanged(); return this; } @@ -28607,7 +28667,7 @@ public org.apache.drill.exec.proto.UserBitShared.SaslMessage getDefaultInstanceF "red.QueryResult.QueryState\022\017\n\004user\030\004 \001(\t" + ":\001-\022\'\n\007foreman\030\005 \001(\0132\026.exec.DrillbitEndp" + "oint\022\024\n\014options_json\030\006 \001(\t\022\022\n\ntotal_cost" + - "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\337\004\n\014QueryP" + + "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\367\004\n\014QueryP" + "rofile\022 \n\002id\030\001 \001(\0132\024.exec.shared.QueryId" + "\022$\n\004type\030\002 \001(\0162\026.exec.shared.QueryType\022\r" + "\n\005start\030\003 \001(\003\022\013\n\003end\030\004 \001(\003\022\r\n\005query\030\005 \001(" + @@ -28622,46 +28682,46 @@ public org.apache.drill.exec.proto.UserBitShared.SaslMessage getDefaultInstanceF "tions_json\030\021 \001(\t\022\017\n\007planEnd\030\022 \001(\003\022\024\n\014que" + "ueWaitEnd\030\023 \001(\003\022\022\n\ntotal_cost\030\024 \001(\001\022\025\n\nq" + "ueue_name\030\025 \001(\t:\001-\022\017\n\007queryId\030\026 \001(\t\022\021\n\ta" + - "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\"" + - "t\n\024MajorFragmentProfile\022\031\n\021major_fragmen" + - "t_id\030\001 \001(\005\022A\n\026minor_fragment_profile\030\002 \003" + - "(\0132!.exec.shared.MinorFragmentProfile\"\350\002" + - "\n\024MinorFragmentProfile\022)\n\005state\030\001 \001(\0162\032." + - "exec.shared.FragmentState\022(\n\005error\030\002 \001(\013" + - "2\031.exec.shared.DrillPBError\022\031\n\021minor_fra" + - "gment_id\030\003 \001(\005\0226\n\020operator_profile\030\004 \003(\013" + - "2\034.exec.shared.OperatorProfile\022\022\n\nstart_" + - "time\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003\022\023\n\013memory_u" + - "sed\030\007 \001(\003\022\027\n\017max_memory_used\030\010 \001(\003\022(\n\010en" + - "dpoint\030\t \001(\0132\026.exec.DrillbitEndpoint\022\023\n\013" + - "last_update\030\n \001(\003\022\025\n\rlast_progress\030\013 \001(\003" + - "\"\237\002\n\017OperatorProfile\0221\n\rinput_profile\030\001 " + - "\003(\0132\032.exec.shared.StreamProfile\022\023\n\013opera" + - "tor_id\030\003 \001(\005\022\031\n\roperator_type\030\004 \001(\005B\002\030\001\022" + - "\023\n\013setup_nanos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 " + - "\001(\003\022#\n\033peak_local_memory_allocated\030\007 \001(\003" + - "\022(\n\006metric\030\010 \003(\0132\030.exec.shared.MetricVal" + - "ue\022\022\n\nwait_nanos\030\t \001(\003\022\032\n\022operator_type_" + - "name\030\n \001(\t\"B\n\rStreamProfile\022\017\n\007records\030\001" + - " \001(\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J" + - "\n\013MetricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong" + - "_value\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001\")\n\010Re" + - "gistry\022\035\n\003jar\030\001 \003(\0132\020.exec.shared.Jar\"/\n" + - "\003Jar\022\014\n\004name\030\001 \001(\t\022\032\n\022function_signature" + - "\030\002 \003(\t\"W\n\013SaslMessage\022\021\n\tmechanism\030\001 \001(\t" + - "\022\014\n\004data\030\002 \001(\014\022\'\n\006status\030\003 \001(\0162\027.exec.sh" + - "ared.SaslStatus*5\n\nRpcChannel\022\017\n\013BIT_CON" + - "TROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004USER\020\002*V\n\tQueryT" + - "ype\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022" + - "\r\n\tEXECUTION\020\004\022\026\n\022PREPARED_STATEMENT\020\005*\207" + - "\001\n\rFragmentState\022\013\n\007SENDING\020\000\022\027\n\023AWAITIN" + - "G_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020" + - "\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026CANCELLA" + - "TION_REQUESTED\020\006*g\n\nSaslStatus\022\020\n\014SASL_U" + - "NKNOWN\020\000\022\016\n\nSASL_START\020\001\022\024\n\020SASL_IN_PROG" + - "RESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020" + - "\004B.\n\033org.apache.drill.exec.protoB\rUserBi" + - "tSharedH\001" + "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\022" + + "\026\n\016plan_cache_hit\030\031 \001(\010\"t\n\024MajorFragment" + + "Profile\022\031\n\021major_fragment_id\030\001 \001(\005\022A\n\026mi" + + "nor_fragment_profile\030\002 \003(\0132!.exec.shared" + + ".MinorFragmentProfile\"\350\002\n\024MinorFragmentP" + + "rofile\022)\n\005state\030\001 \001(\0162\032.exec.shared.Frag" + + "mentState\022(\n\005error\030\002 \001(\0132\031.exec.shared.D" + + "rillPBError\022\031\n\021minor_fragment_id\030\003 \001(\005\0226" + + "\n\020operator_profile\030\004 \003(\0132\034.exec.shared.O" + + "peratorProfile\022\022\n\nstart_time\030\005 \001(\003\022\020\n\010en" + + "d_time\030\006 \001(\003\022\023\n\013memory_used\030\007 \001(\003\022\027\n\017max" + + "_memory_used\030\010 \001(\003\022(\n\010endpoint\030\t \001(\0132\026.e" + + "xec.DrillbitEndpoint\022\023\n\013last_update\030\n \001(" + + "\003\022\025\n\rlast_progress\030\013 \001(\003\"\237\002\n\017OperatorPro" + + "file\0221\n\rinput_profile\030\001 \003(\0132\032.exec.share" + + "d.StreamProfile\022\023\n\013operator_id\030\003 \001(\005\022\031\n\r" + + "operator_type\030\004 \001(\005B\002\030\001\022\023\n\013setup_nanos\030\005" + + " \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022#\n\033peak_loca" + + "l_memory_allocated\030\007 \001(\003\022(\n\006metric\030\010 \003(\013" + + "2\030.exec.shared.MetricValue\022\022\n\nwait_nanos" + + "\030\t \001(\003\022\032\n\022operator_type_name\030\n \001(\t\"B\n\rSt" + + "reamProfile\022\017\n\007records\030\001 \001(\003\022\017\n\007batches\030" + + "\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013MetricValue\022\021\n" + + "\tmetric_id\030\001 \001(\005\022\022\n\nlong_value\030\002 \001(\003\022\024\n\014" + + "double_value\030\003 \001(\001\")\n\010Registry\022\035\n\003jar\030\001 " + + "\003(\0132\020.exec.shared.Jar\"/\n\003Jar\022\014\n\004name\030\001 \001" + + "(\t\022\032\n\022function_signature\030\002 \003(\t\"W\n\013SaslMe" + + "ssage\022\021\n\tmechanism\030\001 \001(\t\022\014\n\004data\030\002 \001(\014\022\'" + + "\n\006status\030\003 \001(\0162\027.exec.shared.SaslStatus*" + + "5\n\nRpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DA" + + "TA\020\001\022\010\n\004USER\020\002*V\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007" + + "LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022\r\n\tEXECUTION\020\004\022\026" + + "\n\022PREPARED_STATEMENT\020\005*\207\001\n\rFragmentState" + + "\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013" + + "\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004" + + "\022\n\n\006FAILED\020\005\022\032\n\026CANCELLATION_REQUESTED\020\006" + + "*g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL" + + "_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_S" + + "UCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org.apache." + + "drill.exec.protoB\rUserBitSharedH\001" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor .internalBuildGeneratedFileFrom(descriptorData, @@ -28753,7 +28813,7 @@ public org.apache.drill.exec.proto.UserBitShared.SaslMessage getDefaultInstanceF internal_static_exec_shared_QueryProfile_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_exec_shared_QueryProfile_descriptor, - new java.lang.String[] { "Id", "Type", "Start", "End", "Query", "Plan", "Foreman", "State", "TotalFragments", "FinishedFragments", "FragmentProfile", "User", "Error", "VerboseError", "ErrorId", "ErrorNode", "OptionsJson", "PlanEnd", "QueueWaitEnd", "TotalCost", "QueueName", "QueryId", "AutoLimit", "ScannedPlugins", }); + new java.lang.String[] { "Id", "Type", "Start", "End", "Query", "Plan", "Foreman", "State", "TotalFragments", "FinishedFragments", "FragmentProfile", "User", "Error", "VerboseError", "ErrorId", "ErrorNode", "OptionsJson", "PlanEnd", "QueueWaitEnd", "TotalCost", "QueueName", "QueryId", "AutoLimit", "ScannedPlugins", "PlanCacheHit", }); internal_static_exec_shared_MajorFragmentProfile_descriptor = getDescriptor().getMessageTypes().get(14); internal_static_exec_shared_MajorFragmentProfile_fieldAccessorTable = new diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto index dafd533cddf..dfdf6d5340b 100644 --- a/protocol/src/main/protobuf/UserBitShared.proto +++ b/protocol/src/main/protobuf/UserBitShared.proto @@ -262,6 +262,7 @@ message QueryProfile { optional string queryId = 22; optional int32 autoLimit = 23; repeated string scanned_plugins = 24; + optional bool plan_cache_hit = 25; } message MajorFragmentProfile { From 41364602d7c39e4ae2e74f681738030c80dc8338 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Fri, 2 Oct 2026 16:20:06 +0800 Subject: [PATCH 02/16] Remove unused temporal plan-cache parameter markers --- PLAN_CACHE_DESIGN.md | 8 ++++---- .../expression/ExpressionStringBuilder.java | 20 ------------------- .../common/expression/ValueExpressions.java | 10 +++++----- 3 files changed, 9 insertions(+), 29 deletions(-) diff --git a/PLAN_CACHE_DESIGN.md b/PLAN_CACHE_DESIGN.md index c5b3462feea..06fe2446b72 100644 --- a/PLAN_CACHE_DESIGN.md +++ b/PLAN_CACHE_DESIGN.md @@ -77,10 +77,10 @@ HBase and Iceberg; other plugins retain the default of no support. Constants that determine SQL structure stay in the key: group/order expressions, window definitions, limit/offset, field selectors, and function configuration -arguments such as `DATE_PART`'s unit or `CONVERT_FROM`'s encoding. DATE literals -also stay in the key because DATE parameter binding is not supported. Other -eligible literals in the same query can still be rebound. Queries with no slots -can reuse their complete template. +arguments such as `DATE_PART`'s unit or `CONVERT_FROM`'s encoding. DATE, TIME, +TIMESTAMP and INTERVAL literals also stay in the key because their parameter +binding is not supported. Other eligible literals in the same query can still +be rebound. Queries with no slots can reuse their complete template. Writes, DDL, existing dynamic parameters, volatile/query-context functions and sessions containing temporary tables or mutable aliases bypass the cache. diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 838f0cda283..682c25951bd 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -212,61 +212,41 @@ public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws Run @Override public Void visitDateConstant(DateExpression lExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(lExpr, sb)) { - return null; - } sb.append("cast( "); sb.append(lExpr.getDate()); sb.append(" as DATE)"); - endBoundLiteral(lExpr, sb); return null; } @Override public Void visitTimeConstant(TimeExpression lExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(lExpr, sb)) { - return null; - } sb.append("cast( "); sb.append(lExpr.getTime()); sb.append(" as TIME)"); - endBoundLiteral(lExpr, sb); return null; } @Override public Void visitTimeStampConstant(TimeStampExpression lExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(lExpr, sb)) { - return null; - } sb.append("cast( "); sb.append(lExpr.getTimeStamp()); sb.append(" as TIMESTAMP)"); - endBoundLiteral(lExpr, sb); return null; } @Override public Void visitIntervalYearConstant(IntervalYearExpression lExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(lExpr, sb)) { - return null; - } sb.append("cast( '"); sb.append(Period.months(lExpr.getIntervalYear()).toString()); sb.append("' as INTERVALYEAR)"); - endBoundLiteral(lExpr, sb); return null; } @Override public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(lExpr, sb)) { - return null; - } sb.append("cast( '"); sb.append(Period.days(lExpr.getIntervalDay()).plusMillis(lExpr.getIntervalMillis()).toString()); sb.append("' as INTERVALDAY)"); - endBoundLiteral(lExpr, sb); return null; } diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java index 01597dc59bb..8c96e2092d3 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java @@ -513,7 +513,7 @@ public Iterator iterator() { } } - public static class DateExpression extends LiteralExpression { + public static class DateExpression extends LogicalExpressionBase { private static final MajorType DATE_CONSTANT = Types.required(MinorType.DATE); @@ -548,7 +548,7 @@ public Iterator iterator() { } } - public static class TimeExpression extends LiteralExpression { + public static class TimeExpression extends LogicalExpressionBase { private static final MajorType TIME_CONSTANT = Types.required(MinorType.TIME); @@ -583,7 +583,7 @@ public Iterator iterator() { } } - public static class TimeStampExpression extends LiteralExpression { + public static class TimeStampExpression extends LogicalExpressionBase { private static final MajorType TIMESTAMP_CONSTANT = Types.required(MinorType.TIMESTAMP); @@ -618,7 +618,7 @@ public Iterator iterator() { } } - public static class IntervalYearExpression extends LiteralExpression { + public static class IntervalYearExpression extends LogicalExpressionBase { private static final MajorType INTERVALYEAR_CONSTANT = Types.required(MinorType.INTERVALYEAR); @@ -653,7 +653,7 @@ public Iterator iterator() { } } - public static class IntervalDayExpression extends LiteralExpression { + public static class IntervalDayExpression extends LogicalExpressionBase { private static final MajorType INTERVALDAY_CONSTANT = Types.required(MinorType.INTERVALDAY); private static final long MILLIS_IN_DAY = 1000 * 60 * 60 * 24; From b0037159919ed0d2e24fe566009c17e838add192 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Fri, 2 Oct 2026 16:27:09 +0800 Subject: [PATCH 03/16] Clarify metadata table exclusion from Iceberg plan caching --- .../apache/drill/exec/store/iceberg/IcebergGroupScan.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java index b67e2a3b92f..3af3b9885ce 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java @@ -37,6 +37,7 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.DrillFileSystem; +import org.apache.drill.exec.store.iceberg.format.IcebergFormatLocationTransformer; import org.apache.drill.exec.store.iceberg.format.IcebergFormatPlugin; import org.apache.drill.exec.store.iceberg.plan.DrillExprToIcebergTranslator; import org.apache.drill.exec.store.iceberg.snapshot.Snapshot; @@ -186,7 +187,12 @@ public IcebergGroupScan clone(List columns) { @Override public boolean supportPlanCache() { - return formatPlugin.supportPlanCache() && path.indexOf('#') < 0; + return formatPlugin.supportPlanCache() && !isMetadataTablePath(); + } + + private boolean isMetadataTablePath() { + // Metadata table paths use the form #. + return path.contains(IcebergFormatLocationTransformer.METADATA_SEPARATOR); } @Override From f5e64d59e0989fb98e5a49201159d4cf335eb4eb Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Sat, 3 Oct 2026 01:53:25 +0800 Subject: [PATCH 04/16] Regenerate plan cache profile protobufs with CI compiler --- .../client/src/protobuf/UserBitShared.pb.cc | 156 +++--- .../client/src/protobuf/UserBitShared.pb.h | 49 +- .../drill/exec/proto/UserBitShared.java | 488 ++++++++++-------- 3 files changed, 402 insertions(+), 291 deletions(-) diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc index 273b3fc328b..1c147c07dc4 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc @@ -243,6 +243,7 @@ constexpr QueryProfile::QueryProfile( , planend_(PROTOBUF_LONGLONG(0)) , queuewaitend_(PROTOBUF_LONGLONG(0)) , total_cost_(0) + , plan_cache_hit_(false) , type_(1) {} struct QueryProfileDefaultTypeInternal { @@ -583,8 +584,9 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_UserBitShared_2eproto::offsets PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, queryid_), PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, autolimit_), PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, scanned_plugins_), + PROTOBUF_FIELD_OFFSET(::exec::shared::QueryProfile, plan_cache_hit_), 10, - 21, + 22, 12, 13, 0, @@ -607,6 +609,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_UserBitShared_2eproto::offsets 9, 17, ~0u, + 21, PROTOBUF_FIELD_OFFSET(::exec::shared::MajorFragmentProfile, _has_bits_), PROTOBUF_FIELD_OFFSET(::exec::shared::MajorFragmentProfile, _internal_metadata_), ~0u, // no _extensions_ @@ -729,15 +732,15 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB { 124, 132, sizeof(::exec::shared::QueryResult)}, { 135, 144, sizeof(::exec::shared::QueryData)}, { 148, 161, sizeof(::exec::shared::QueryInfo)}, - { 169, 198, sizeof(::exec::shared::QueryProfile)}, - { 222, 229, sizeof(::exec::shared::MajorFragmentProfile)}, - { 231, 247, sizeof(::exec::shared::MinorFragmentProfile)}, - { 258, 272, sizeof(::exec::shared::OperatorProfile)}, - { 281, 289, sizeof(::exec::shared::StreamProfile)}, - { 292, 300, sizeof(::exec::shared::MetricValue)}, - { 303, -1, sizeof(::exec::shared::Registry)}, - { 309, 316, sizeof(::exec::shared::Jar)}, - { 318, 326, sizeof(::exec::shared::SaslMessage)}, + { 169, 199, sizeof(::exec::shared::QueryProfile)}, + { 224, 231, sizeof(::exec::shared::MajorFragmentProfile)}, + { 233, 249, sizeof(::exec::shared::MinorFragmentProfile)}, + { 260, 274, sizeof(::exec::shared::OperatorProfile)}, + { 283, 291, sizeof(::exec::shared::StreamProfile)}, + { 294, 302, sizeof(::exec::shared::MetricValue)}, + { 305, -1, sizeof(::exec::shared::Registry)}, + { 311, 318, sizeof(::exec::shared::Jar)}, + { 320, 328, sizeof(::exec::shared::SaslMessage)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -823,7 +826,7 @@ const char descriptor_table_protodef_UserBitShared_2eproto[] PROTOBUF_SECTION_VA "red.QueryResult.QueryState\022\017\n\004user\030\004 \001(\t" ":\001-\022\'\n\007foreman\030\005 \001(\0132\026.exec.DrillbitEndp" "oint\022\024\n\014options_json\030\006 \001(\t\022\022\n\ntotal_cost" - "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\337\004\n\014QueryP" + "\030\007 \001(\001\022\025\n\nqueue_name\030\010 \001(\t:\001-\"\367\004\n\014QueryP" "rofile\022 \n\002id\030\001 \001(\0132\024.exec.shared.QueryId" "\022$\n\004type\030\002 \001(\0162\026.exec.shared.QueryType\022\r" "\n\005start\030\003 \001(\003\022\013\n\003end\030\004 \001(\003\022\r\n\005query\030\005 \001(" @@ -838,46 +841,46 @@ const char descriptor_table_protodef_UserBitShared_2eproto[] PROTOBUF_SECTION_VA "tions_json\030\021 \001(\t\022\017\n\007planEnd\030\022 \001(\003\022\024\n\014que" "ueWaitEnd\030\023 \001(\003\022\022\n\ntotal_cost\030\024 \001(\001\022\025\n\nq" "ueue_name\030\025 \001(\t:\001-\022\017\n\007queryId\030\026 \001(\t\022\021\n\ta" - "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\"" - "t\n\024MajorFragmentProfile\022\031\n\021major_fragmen" - "t_id\030\001 \001(\005\022A\n\026minor_fragment_profile\030\002 \003" - "(\0132!.exec.shared.MinorFragmentProfile\"\350\002" - "\n\024MinorFragmentProfile\022)\n\005state\030\001 \001(\0162\032." - "exec.shared.FragmentState\022(\n\005error\030\002 \001(\013" - "2\031.exec.shared.DrillPBError\022\031\n\021minor_fra" - "gment_id\030\003 \001(\005\0226\n\020operator_profile\030\004 \003(\013" - "2\034.exec.shared.OperatorProfile\022\022\n\nstart_" - "time\030\005 \001(\003\022\020\n\010end_time\030\006 \001(\003\022\023\n\013memory_u" - "sed\030\007 \001(\003\022\027\n\017max_memory_used\030\010 \001(\003\022(\n\010en" - "dpoint\030\t \001(\0132\026.exec.DrillbitEndpoint\022\023\n\013" - "last_update\030\n \001(\003\022\025\n\rlast_progress\030\013 \001(\003" - "\"\237\002\n\017OperatorProfile\0221\n\rinput_profile\030\001 " - "\003(\0132\032.exec.shared.StreamProfile\022\023\n\013opera" - "tor_id\030\003 \001(\005\022\031\n\roperator_type\030\004 \001(\005B\002\030\001\022" - "\023\n\013setup_nanos\030\005 \001(\003\022\025\n\rprocess_nanos\030\006 " - "\001(\003\022#\n\033peak_local_memory_allocated\030\007 \001(\003" - "\022(\n\006metric\030\010 \003(\0132\030.exec.shared.MetricVal" - "ue\022\022\n\nwait_nanos\030\t \001(\003\022\032\n\022operator_type_" - "name\030\n \001(\t\"B\n\rStreamProfile\022\017\n\007records\030\001" - " \001(\003\022\017\n\007batches\030\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J" - "\n\013MetricValue\022\021\n\tmetric_id\030\001 \001(\005\022\022\n\nlong" - "_value\030\002 \001(\003\022\024\n\014double_value\030\003 \001(\001\")\n\010Re" - "gistry\022\035\n\003jar\030\001 \003(\0132\020.exec.shared.Jar\"/\n" - "\003Jar\022\014\n\004name\030\001 \001(\t\022\032\n\022function_signature" - "\030\002 \003(\t\"W\n\013SaslMessage\022\021\n\tmechanism\030\001 \001(\t" - "\022\014\n\004data\030\002 \001(\014\022\'\n\006status\030\003 \001(\0162\027.exec.sh" - "ared.SaslStatus*5\n\nRpcChannel\022\017\n\013BIT_CON" - "TROL\020\000\022\014\n\010BIT_DATA\020\001\022\010\n\004USER\020\002*V\n\tQueryT" - "ype\022\007\n\003SQL\020\001\022\013\n\007LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022" - "\r\n\tEXECUTION\020\004\022\026\n\022PREPARED_STATEMENT\020\005*\207" - "\001\n\rFragmentState\022\013\n\007SENDING\020\000\022\027\n\023AWAITIN" - "G_ALLOCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020" - "\003\022\r\n\tCANCELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026CANCELLA" - "TION_REQUESTED\020\006*g\n\nSaslStatus\022\020\n\014SASL_U" - "NKNOWN\020\000\022\016\n\nSASL_START\020\001\022\024\n\020SASL_IN_PROG" - "RESS\020\002\022\020\n\014SASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020" - "\004B.\n\033org.apache.drill.exec.protoB\rUserBi" - "tSharedH\001" + "utoLimit\030\027 \001(\005\022\027\n\017scanned_plugins\030\030 \003(\t\022" + "\026\n\016plan_cache_hit\030\031 \001(\010\"t\n\024MajorFragment" + "Profile\022\031\n\021major_fragment_id\030\001 \001(\005\022A\n\026mi" + "nor_fragment_profile\030\002 \003(\0132!.exec.shared" + ".MinorFragmentProfile\"\350\002\n\024MinorFragmentP" + "rofile\022)\n\005state\030\001 \001(\0162\032.exec.shared.Frag" + "mentState\022(\n\005error\030\002 \001(\0132\031.exec.shared.D" + "rillPBError\022\031\n\021minor_fragment_id\030\003 \001(\005\0226" + "\n\020operator_profile\030\004 \003(\0132\034.exec.shared.O" + "peratorProfile\022\022\n\nstart_time\030\005 \001(\003\022\020\n\010en" + "d_time\030\006 \001(\003\022\023\n\013memory_used\030\007 \001(\003\022\027\n\017max" + "_memory_used\030\010 \001(\003\022(\n\010endpoint\030\t \001(\0132\026.e" + "xec.DrillbitEndpoint\022\023\n\013last_update\030\n \001(" + "\003\022\025\n\rlast_progress\030\013 \001(\003\"\237\002\n\017OperatorPro" + "file\0221\n\rinput_profile\030\001 \003(\0132\032.exec.share" + "d.StreamProfile\022\023\n\013operator_id\030\003 \001(\005\022\031\n\r" + "operator_type\030\004 \001(\005B\002\030\001\022\023\n\013setup_nanos\030\005" + " \001(\003\022\025\n\rprocess_nanos\030\006 \001(\003\022#\n\033peak_loca" + "l_memory_allocated\030\007 \001(\003\022(\n\006metric\030\010 \003(\013" + "2\030.exec.shared.MetricValue\022\022\n\nwait_nanos" + "\030\t \001(\003\022\032\n\022operator_type_name\030\n \001(\t\"B\n\rSt" + "reamProfile\022\017\n\007records\030\001 \001(\003\022\017\n\007batches\030" + "\002 \001(\003\022\017\n\007schemas\030\003 \001(\003\"J\n\013MetricValue\022\021\n" + "\tmetric_id\030\001 \001(\005\022\022\n\nlong_value\030\002 \001(\003\022\024\n\014" + "double_value\030\003 \001(\001\")\n\010Registry\022\035\n\003jar\030\001 " + "\003(\0132\020.exec.shared.Jar\"/\n\003Jar\022\014\n\004name\030\001 \001" + "(\t\022\032\n\022function_signature\030\002 \003(\t\"W\n\013SaslMe" + "ssage\022\021\n\tmechanism\030\001 \001(\t\022\014\n\004data\030\002 \001(\014\022\'" + "\n\006status\030\003 \001(\0162\027.exec.shared.SaslStatus*" + "5\n\nRpcChannel\022\017\n\013BIT_CONTROL\020\000\022\014\n\010BIT_DA" + "TA\020\001\022\010\n\004USER\020\002*V\n\tQueryType\022\007\n\003SQL\020\001\022\013\n\007" + "LOGICAL\020\002\022\014\n\010PHYSICAL\020\003\022\r\n\tEXECUTION\020\004\022\026" + "\n\022PREPARED_STATEMENT\020\005*\207\001\n\rFragmentState" + "\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALLOCATION\020\001\022\013" + "\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\tCANCELLED\020\004" + "\022\n\n\006FAILED\020\005\022\032\n\026CANCELLATION_REQUESTED\020\006" + "*g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000\022\016\n\nSASL" + "_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014SASL_S" + "UCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org.apache." + "drill.exec.protoB\rUserBitSharedH\001" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_UserBitShared_2eproto_deps[3] = { &::descriptor_table_Coordination_2eproto, @@ -886,7 +889,7 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor }; static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_UserBitShared_2eproto_once; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_UserBitShared_2eproto = { - false, false, 4449, descriptor_table_protodef_UserBitShared_2eproto, "UserBitShared.proto", + false, false, 4473, descriptor_table_protodef_UserBitShared_2eproto, "UserBitShared.proto", &descriptor_table_UserBitShared_2eproto_once, descriptor_table_UserBitShared_2eproto_deps, 3, 22, schemas, file_default_instances, TableStruct_UserBitShared_2eproto::offsets, file_level_metadata_UserBitShared_2eproto, file_level_enum_descriptors_UserBitShared_2eproto, file_level_service_descriptors_UserBitShared_2eproto, @@ -5427,7 +5430,7 @@ class QueryProfile::_Internal { (*has_bits)[0] |= 1024u; } static void set_has_type(HasBits* has_bits) { - (*has_bits)[0] |= 2097152u; + (*has_bits)[0] |= 4194304u; } static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 4096u; @@ -5490,6 +5493,9 @@ class QueryProfile::_Internal { static void set_has_autolimit(HasBits* has_bits) { (*has_bits)[0] |= 131072u; } + static void set_has_plan_cache_hit(HasBits* has_bits) { + (*has_bits)[0] |= 2097152u; + } }; const ::exec::shared::QueryId& @@ -5599,8 +5605,8 @@ queue_name_.UnsafeSetDefault(nullptr); queryid_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(reinterpret_cast(this) + static_cast( reinterpret_cast(&id_) - reinterpret_cast(this)), - 0, static_cast(reinterpret_cast(&total_cost_) - - reinterpret_cast(&id_)) + sizeof(total_cost_)); + 0, static_cast(reinterpret_cast(&plan_cache_hit_) - + reinterpret_cast(&id_)) + sizeof(plan_cache_hit_)); type_ = 1; } @@ -5692,10 +5698,10 @@ void QueryProfile::Clear() { reinterpret_cast(&total_fragments_) - reinterpret_cast(&start_)) + sizeof(total_fragments_)); } - if (cached_has_bits & 0x003f0000u) { + if (cached_has_bits & 0x007f0000u) { ::memset(&finished_fragments_, 0, static_cast( - reinterpret_cast(&total_cost_) - - reinterpret_cast(&finished_fragments_)) + sizeof(total_cost_)); + reinterpret_cast(&plan_cache_hit_) - + reinterpret_cast(&finished_fragments_)) + sizeof(plan_cache_hit_)); type_ = 1; } _has_bits_.Clear(); @@ -5949,6 +5955,14 @@ const char* QueryProfile::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_I } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<194>(ptr)); } else goto handle_unusual; continue; + // optional bool plan_cache_hit = 25; + case 25: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 200)) { + _Internal::set_has_plan_cache_hit(&has_bits); + plan_cache_hit_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag == 0) || ((tag & 7) == 4)) { @@ -5989,7 +6003,7 @@ ::PROTOBUF_NAMESPACE_ID::uint8* QueryProfile::_InternalSerialize( } // optional .exec.shared.QueryType type = 2; - if (cached_has_bits & 0x00200000u) { + if (cached_has_bits & 0x00400000u) { target = stream->EnsureSpace(target); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray( 2, this->_internal_type(), target); @@ -6176,6 +6190,12 @@ ::PROTOBUF_NAMESPACE_ID::uint8* QueryProfile::_InternalSerialize( target = stream->WriteString(24, s, target); } + // optional bool plan_cache_hit = 25; + if (cached_has_bits & 0x00200000u) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(25, this->_internal_plan_cache_hit(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); @@ -6323,7 +6343,7 @@ size_t QueryProfile::ByteSizeLong() const { } } - if (cached_has_bits & 0x003f0000u) { + if (cached_has_bits & 0x007f0000u) { // optional int32 finished_fragments = 10; if (cached_has_bits & 0x00010000u) { total_size += 1 + @@ -6357,8 +6377,13 @@ size_t QueryProfile::ByteSizeLong() const { total_size += 2 + 8; } - // optional .exec.shared.QueryType type = 2; + // optional bool plan_cache_hit = 25; if (cached_has_bits & 0x00200000u) { + total_size += 2 + 1; + } + + // optional .exec.shared.QueryType type = 2; + if (cached_has_bits & 0x00400000u) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->_internal_type()); } @@ -6451,7 +6476,7 @@ void QueryProfile::MergeFrom(const QueryProfile& from) { } _has_bits_[0] |= cached_has_bits; } - if (cached_has_bits & 0x003f0000u) { + if (cached_has_bits & 0x007f0000u) { if (cached_has_bits & 0x00010000u) { finished_fragments_ = from.finished_fragments_; } @@ -6468,6 +6493,9 @@ void QueryProfile::MergeFrom(const QueryProfile& from) { total_cost_ = from.total_cost_; } if (cached_has_bits & 0x00200000u) { + plan_cache_hit_ = from.plan_cache_hit_; + } + if (cached_has_bits & 0x00400000u) { type_ = from.type_; } _has_bits_[0] |= cached_has_bits; @@ -6509,8 +6537,8 @@ void QueryProfile::InternalSwap(QueryProfile* other) { queue_name_.Swap(&other->queue_name_, nullptr, GetArena()); queryid_.Swap(&other->queryid_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena()); ::PROTOBUF_NAMESPACE_ID::internal::memswap< - PROTOBUF_FIELD_OFFSET(QueryProfile, total_cost_) - + sizeof(QueryProfile::total_cost_) + PROTOBUF_FIELD_OFFSET(QueryProfile, plan_cache_hit_) + + sizeof(QueryProfile::plan_cache_hit_) - PROTOBUF_FIELD_OFFSET(QueryProfile, id_)>( reinterpret_cast(&id_), reinterpret_cast(&other->id_)); diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h index ffbc7cffa84..b963dcd0ab5 100644 --- a/contrib/native/client/src/protobuf/UserBitShared.pb.h +++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h @@ -3202,6 +3202,7 @@ class QueryProfile PROTOBUF_FINAL : kPlanEndFieldNumber = 18, kQueueWaitEndFieldNumber = 19, kTotalCostFieldNumber = 20, + kPlanCacheHitFieldNumber = 25, kTypeFieldNumber = 2, }; // repeated .exec.shared.MajorFragmentProfile fragment_profile = 11; @@ -3579,6 +3580,19 @@ class QueryProfile PROTOBUF_FINAL : void _internal_set_total_cost(double value); public: + // optional bool plan_cache_hit = 25; + bool has_plan_cache_hit() const; + private: + bool _internal_has_plan_cache_hit() const; + public: + void clear_plan_cache_hit(); + bool plan_cache_hit() const; + void set_plan_cache_hit(bool value); + private: + bool _internal_plan_cache_hit() const; + void _internal_set_plan_cache_hit(bool value); + public: + // optional .exec.shared.QueryType type = 2; bool has_type() const; private: @@ -3626,6 +3640,7 @@ class QueryProfile PROTOBUF_FINAL : ::PROTOBUF_NAMESPACE_ID::int64 planend_; ::PROTOBUF_NAMESPACE_ID::int64 queuewaitend_; double total_cost_; + bool plan_cache_hit_; int type_; friend struct ::TableStruct_UserBitShared_2eproto; }; @@ -7808,7 +7823,7 @@ inline void QueryProfile::set_allocated_id(::exec::shared::QueryId* id) { // optional .exec.shared.QueryType type = 2; inline bool QueryProfile::_internal_has_type() const { - bool value = (_has_bits_[0] & 0x00200000u) != 0; + bool value = (_has_bits_[0] & 0x00400000u) != 0; return value; } inline bool QueryProfile::has_type() const { @@ -7816,7 +7831,7 @@ inline bool QueryProfile::has_type() const { } inline void QueryProfile::clear_type() { type_ = 1; - _has_bits_[0] &= ~0x00200000u; + _has_bits_[0] &= ~0x00400000u; } inline ::exec::shared::QueryType QueryProfile::_internal_type() const { return static_cast< ::exec::shared::QueryType >(type_); @@ -7827,7 +7842,7 @@ inline ::exec::shared::QueryType QueryProfile::type() const { } inline void QueryProfile::_internal_set_type(::exec::shared::QueryType value) { assert(::exec::shared::QueryType_IsValid(value)); - _has_bits_[0] |= 0x00200000u; + _has_bits_[0] |= 0x00400000u; type_ = value; } inline void QueryProfile::set_type(::exec::shared::QueryType value) { @@ -8852,6 +8867,34 @@ QueryProfile::mutable_scanned_plugins() { return &scanned_plugins_; } +// optional bool plan_cache_hit = 25; +inline bool QueryProfile::_internal_has_plan_cache_hit() const { + bool value = (_has_bits_[0] & 0x00200000u) != 0; + return value; +} +inline bool QueryProfile::has_plan_cache_hit() const { + return _internal_has_plan_cache_hit(); +} +inline void QueryProfile::clear_plan_cache_hit() { + plan_cache_hit_ = false; + _has_bits_[0] &= ~0x00200000u; +} +inline bool QueryProfile::_internal_plan_cache_hit() const { + return plan_cache_hit_; +} +inline bool QueryProfile::plan_cache_hit() const { + // @@protoc_insertion_point(field_get:exec.shared.QueryProfile.plan_cache_hit) + return _internal_plan_cache_hit(); +} +inline void QueryProfile::_internal_set_plan_cache_hit(bool value) { + _has_bits_[0] |= 0x00200000u; + plan_cache_hit_ = value; +} +inline void QueryProfile::set_plan_cache_hit(bool value) { + _internal_set_plan_cache_hit(value); + // @@protoc_insertion_point(field_set:exec.shared.QueryProfile.plan_cache_hit) +} + // ------------------------------------------------------------------- // MajorFragmentProfile diff --git a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java index 85cbdec9b13..b4f677535af 100644 --- a/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java +++ b/protocol/src/main/java/org/apache/drill/exec/proto/UserBitShared.java @@ -15993,8 +15993,7 @@ private QueryProfile() { optionsJson_ = ""; queueName_ = "-"; queryId_ = ""; - scannedPlugins_ = - com.google.protobuf.LazyStringArrayList.emptyList(); + scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; } @java.lang.Override @@ -16004,6 +16003,11 @@ protected java.lang.Object newInstance( return new QueryProfile(); } + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { return org.apache.drill.exec.proto.UserBitShared.internal_static_exec_shared_QueryProfile_descriptor; @@ -16045,7 +16049,7 @@ public org.apache.drill.exec.proto.UserBitShared.QueryIdOrBuilder getIdOrBuilder } public static final int TYPE_FIELD_NUMBER = 2; - private int type_ = 1; + private int type_; /** * optional .exec.shared.QueryType type = 2; * @return Whether the type field is set. @@ -16058,12 +16062,13 @@ public org.apache.drill.exec.proto.UserBitShared.QueryIdOrBuilder getIdOrBuilder * @return The type. */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryType getType() { - org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.forNumber(type_); + @SuppressWarnings("deprecation") + org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.valueOf(type_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryType.SQL : result; } public static final int START_FIELD_NUMBER = 3; - private long start_ = 0L; + private long start_; /** * optional int64 start = 3; * @return Whether the start field is set. @@ -16082,7 +16087,7 @@ public long getStart() { } public static final int END_FIELD_NUMBER = 4; - private long end_ = 0L; + private long end_; /** * optional int64 end = 4; * @return Whether the end field is set. @@ -16101,8 +16106,7 @@ public long getEnd() { } public static final int QUERY_FIELD_NUMBER = 5; - @SuppressWarnings("serial") - private volatile java.lang.Object query_ = ""; + private volatile java.lang.Object query_; /** * optional string query = 5; * @return Whether the query field is set. @@ -16150,8 +16154,7 @@ public java.lang.String getQuery() { } public static final int PLAN_FIELD_NUMBER = 6; - @SuppressWarnings("serial") - private volatile java.lang.Object plan_ = ""; + private volatile java.lang.Object plan_; /** * optional string plan = 6; * @return Whether the plan field is set. @@ -16225,7 +16228,7 @@ public org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpointOrBuilder } public static final int STATE_FIELD_NUMBER = 8; - private int state_ = 0; + private int state_; /** * optional .exec.shared.QueryResult.QueryState state = 8; * @return Whether the state field is set. @@ -16238,12 +16241,13 @@ public org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpointOrBuilder * @return The state. */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState getState() { - org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.forNumber(state_); + @SuppressWarnings("deprecation") + org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.valueOf(state_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.STARTING : result; } public static final int TOTAL_FRAGMENTS_FIELD_NUMBER = 9; - private int totalFragments_ = 0; + private int totalFragments_; /** * optional int32 total_fragments = 9; * @return Whether the totalFragments field is set. @@ -16262,7 +16266,7 @@ public int getTotalFragments() { } public static final int FINISHED_FRAGMENTS_FIELD_NUMBER = 10; - private int finishedFragments_ = 0; + private int finishedFragments_; /** * optional int32 finished_fragments = 10; * @return Whether the finishedFragments field is set. @@ -16281,7 +16285,6 @@ public int getFinishedFragments() { } public static final int FRAGMENT_PROFILE_FIELD_NUMBER = 11; - @SuppressWarnings("serial") private java.util.List fragmentProfile_; /** * repeated .exec.shared.MajorFragmentProfile fragment_profile = 11; @@ -16322,8 +16325,7 @@ public org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfileOrBuilder g } public static final int USER_FIELD_NUMBER = 12; - @SuppressWarnings("serial") - private volatile java.lang.Object user_ = "-"; + private volatile java.lang.Object user_; /** * optional string user = 12 [default = "-"]; * @return Whether the user field is set. @@ -16371,8 +16373,7 @@ public java.lang.String getUser() { } public static final int ERROR_FIELD_NUMBER = 13; - @SuppressWarnings("serial") - private volatile java.lang.Object error_ = ""; + private volatile java.lang.Object error_; /** * optional string error = 13; * @return Whether the error field is set. @@ -16420,8 +16421,7 @@ public java.lang.String getError() { } public static final int VERBOSEERROR_FIELD_NUMBER = 14; - @SuppressWarnings("serial") - private volatile java.lang.Object verboseError_ = ""; + private volatile java.lang.Object verboseError_; /** * optional string verboseError = 14; * @return Whether the verboseError field is set. @@ -16469,8 +16469,7 @@ public java.lang.String getVerboseError() { } public static final int ERROR_ID_FIELD_NUMBER = 15; - @SuppressWarnings("serial") - private volatile java.lang.Object errorId_ = ""; + private volatile java.lang.Object errorId_; /** * optional string error_id = 15; * @return Whether the errorId field is set. @@ -16518,8 +16517,7 @@ public java.lang.String getErrorId() { } public static final int ERROR_NODE_FIELD_NUMBER = 16; - @SuppressWarnings("serial") - private volatile java.lang.Object errorNode_ = ""; + private volatile java.lang.Object errorNode_; /** * optional string error_node = 16; * @return Whether the errorNode field is set. @@ -16567,8 +16565,7 @@ public java.lang.String getErrorNode() { } public static final int OPTIONS_JSON_FIELD_NUMBER = 17; - @SuppressWarnings("serial") - private volatile java.lang.Object optionsJson_ = ""; + private volatile java.lang.Object optionsJson_; /** * optional string options_json = 17; * @return Whether the optionsJson field is set. @@ -16616,7 +16613,7 @@ public java.lang.String getOptionsJson() { } public static final int PLANEND_FIELD_NUMBER = 18; - private long planEnd_ = 0L; + private long planEnd_; /** * optional int64 planEnd = 18; * @return Whether the planEnd field is set. @@ -16635,7 +16632,7 @@ public long getPlanEnd() { } public static final int QUEUEWAITEND_FIELD_NUMBER = 19; - private long queueWaitEnd_ = 0L; + private long queueWaitEnd_; /** * optional int64 queueWaitEnd = 19; * @return Whether the queueWaitEnd field is set. @@ -16654,7 +16651,7 @@ public long getQueueWaitEnd() { } public static final int TOTAL_COST_FIELD_NUMBER = 20; - private double totalCost_ = 0D; + private double totalCost_; /** * optional double total_cost = 20; * @return Whether the totalCost field is set. @@ -16673,8 +16670,7 @@ public double getTotalCost() { } public static final int QUEUE_NAME_FIELD_NUMBER = 21; - @SuppressWarnings("serial") - private volatile java.lang.Object queueName_ = "-"; + private volatile java.lang.Object queueName_; /** * optional string queue_name = 21 [default = "-"]; * @return Whether the queueName field is set. @@ -16722,8 +16718,7 @@ public java.lang.String getQueueName() { } public static final int QUERYID_FIELD_NUMBER = 22; - @SuppressWarnings("serial") - private volatile java.lang.Object queryId_ = ""; + private volatile java.lang.Object queryId_; /** * optional string queryId = 22; * @return Whether the queryId field is set. @@ -16771,7 +16766,7 @@ public java.lang.String getQueryId() { } public static final int AUTOLIMIT_FIELD_NUMBER = 23; - private int autoLimit_ = 0; + private int autoLimit_; /** * optional int32 autoLimit = 23; * @return Whether the autoLimit field is set. @@ -16790,9 +16785,7 @@ public int getAutoLimit() { } public static final int SCANNED_PLUGINS_FIELD_NUMBER = 24; - @SuppressWarnings("serial") - private com.google.protobuf.LazyStringArrayList scannedPlugins_ = - com.google.protobuf.LazyStringArrayList.emptyList(); + private com.google.protobuf.LazyStringList scannedPlugins_; /** * repeated string scanned_plugins = 24; * @return A list containing the scannedPlugins. @@ -16827,7 +16820,7 @@ public java.lang.String getScannedPlugins(int index) { } public static final int PLAN_CACHE_HIT_FIELD_NUMBER = 25; - private boolean planCacheHit_ = false; + private boolean planCacheHit_; /** * optional bool plan_cache_hit = 25; * @return Whether the planCacheHit field is set. @@ -17336,13 +17329,11 @@ public static org.apache.drill.exec.proto.UserBitShared.QueryProfile parseFrom( return com.google.protobuf.GeneratedMessageV3 .parseWithIOException(PARSER, input, extensionRegistry); } - public static org.apache.drill.exec.proto.UserBitShared.QueryProfile parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException { return com.google.protobuf.GeneratedMessageV3 .parseDelimitedWithIOException(PARSER, input); } - public static org.apache.drill.exec.proto.UserBitShared.QueryProfile parseDelimitedFrom( java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) @@ -17425,25 +17416,34 @@ private void maybeForceBuilderInitialization() { @java.lang.Override public Builder clear() { super.clear(); - bitField0_ = 0; - id_ = null; - if (idBuilder_ != null) { - idBuilder_.dispose(); - idBuilder_ = null; + if (idBuilder_ == null) { + id_ = null; + } else { + idBuilder_.clear(); } + bitField0_ = (bitField0_ & ~0x00000001); type_ = 1; + bitField0_ = (bitField0_ & ~0x00000002); start_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); end_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); query_ = ""; + bitField0_ = (bitField0_ & ~0x00000010); plan_ = ""; - foreman_ = null; - if (foremanBuilder_ != null) { - foremanBuilder_.dispose(); - foremanBuilder_ = null; + bitField0_ = (bitField0_ & ~0x00000020); + if (foremanBuilder_ == null) { + foreman_ = null; + } else { + foremanBuilder_.clear(); } + bitField0_ = (bitField0_ & ~0x00000040); state_ = 0; + bitField0_ = (bitField0_ & ~0x00000080); totalFragments_ = 0; + bitField0_ = (bitField0_ & ~0x00000100); finishedFragments_ = 0; + bitField0_ = (bitField0_ & ~0x00000200); if (fragmentProfileBuilder_ == null) { fragmentProfile_ = java.util.Collections.emptyList(); } else { @@ -17452,20 +17452,33 @@ public Builder clear() { } bitField0_ = (bitField0_ & ~0x00000400); user_ = "-"; + bitField0_ = (bitField0_ & ~0x00000800); error_ = ""; + bitField0_ = (bitField0_ & ~0x00001000); verboseError_ = ""; + bitField0_ = (bitField0_ & ~0x00002000); errorId_ = ""; + bitField0_ = (bitField0_ & ~0x00004000); errorNode_ = ""; + bitField0_ = (bitField0_ & ~0x00008000); optionsJson_ = ""; + bitField0_ = (bitField0_ & ~0x00010000); planEnd_ = 0L; + bitField0_ = (bitField0_ & ~0x00020000); queueWaitEnd_ = 0L; + bitField0_ = (bitField0_ & ~0x00040000); totalCost_ = 0D; + bitField0_ = (bitField0_ & ~0x00080000); queueName_ = "-"; + bitField0_ = (bitField0_ & ~0x00100000); queryId_ = ""; + bitField0_ = (bitField0_ & ~0x00200000); autoLimit_ = 0; - scannedPlugins_ = - com.google.protobuf.LazyStringArrayList.emptyList(); + bitField0_ = (bitField0_ & ~0x00400000); + scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00800000); planCacheHit_ = false; + bitField0_ = (bitField0_ & ~0x01000000); return this; } @@ -17492,37 +17505,20 @@ public org.apache.drill.exec.proto.UserBitShared.QueryProfile build() { @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryProfile buildPartial() { org.apache.drill.exec.proto.UserBitShared.QueryProfile result = new org.apache.drill.exec.proto.UserBitShared.QueryProfile(this); - buildPartialRepeatedFields(result); - if (bitField0_ != 0) { buildPartial0(result); } - onBuilt(); - return result; - } - - private void buildPartialRepeatedFields(org.apache.drill.exec.proto.UserBitShared.QueryProfile result) { - if (fragmentProfileBuilder_ == null) { - if (((bitField0_ & 0x00000400) != 0)) { - fragmentProfile_ = java.util.Collections.unmodifiableList(fragmentProfile_); - bitField0_ = (bitField0_ & ~0x00000400); - } - result.fragmentProfile_ = fragmentProfile_; - } else { - result.fragmentProfile_ = fragmentProfileBuilder_.build(); - } - } - - private void buildPartial0(org.apache.drill.exec.proto.UserBitShared.QueryProfile result) { int from_bitField0_ = bitField0_; int to_bitField0_ = 0; if (((from_bitField0_ & 0x00000001) != 0)) { - result.id_ = idBuilder_ == null - ? id_ - : idBuilder_.build(); + if (idBuilder_ == null) { + result.id_ = id_; + } else { + result.id_ = idBuilder_.build(); + } to_bitField0_ |= 0x00000001; } if (((from_bitField0_ & 0x00000002) != 0)) { - result.type_ = type_; to_bitField0_ |= 0x00000002; } + result.type_ = type_; if (((from_bitField0_ & 0x00000004) != 0)) { result.start_ = start_; to_bitField0_ |= 0x00000004; @@ -17532,23 +17528,25 @@ private void buildPartial0(org.apache.drill.exec.proto.UserBitShared.QueryProfil to_bitField0_ |= 0x00000008; } if (((from_bitField0_ & 0x00000010) != 0)) { - result.query_ = query_; to_bitField0_ |= 0x00000010; } + result.query_ = query_; if (((from_bitField0_ & 0x00000020) != 0)) { - result.plan_ = plan_; to_bitField0_ |= 0x00000020; } + result.plan_ = plan_; if (((from_bitField0_ & 0x00000040) != 0)) { - result.foreman_ = foremanBuilder_ == null - ? foreman_ - : foremanBuilder_.build(); + if (foremanBuilder_ == null) { + result.foreman_ = foreman_; + } else { + result.foreman_ = foremanBuilder_.build(); + } to_bitField0_ |= 0x00000040; } if (((from_bitField0_ & 0x00000080) != 0)) { - result.state_ = state_; to_bitField0_ |= 0x00000080; } + result.state_ = state_; if (((from_bitField0_ & 0x00000100) != 0)) { result.totalFragments_ = totalFragments_; to_bitField0_ |= 0x00000100; @@ -17557,30 +17555,39 @@ private void buildPartial0(org.apache.drill.exec.proto.UserBitShared.QueryProfil result.finishedFragments_ = finishedFragments_; to_bitField0_ |= 0x00000200; } + if (fragmentProfileBuilder_ == null) { + if (((bitField0_ & 0x00000400) != 0)) { + fragmentProfile_ = java.util.Collections.unmodifiableList(fragmentProfile_); + bitField0_ = (bitField0_ & ~0x00000400); + } + result.fragmentProfile_ = fragmentProfile_; + } else { + result.fragmentProfile_ = fragmentProfileBuilder_.build(); + } if (((from_bitField0_ & 0x00000800) != 0)) { - result.user_ = user_; to_bitField0_ |= 0x00000400; } + result.user_ = user_; if (((from_bitField0_ & 0x00001000) != 0)) { - result.error_ = error_; to_bitField0_ |= 0x00000800; } + result.error_ = error_; if (((from_bitField0_ & 0x00002000) != 0)) { - result.verboseError_ = verboseError_; to_bitField0_ |= 0x00001000; } + result.verboseError_ = verboseError_; if (((from_bitField0_ & 0x00004000) != 0)) { - result.errorId_ = errorId_; to_bitField0_ |= 0x00002000; } + result.errorId_ = errorId_; if (((from_bitField0_ & 0x00008000) != 0)) { - result.errorNode_ = errorNode_; to_bitField0_ |= 0x00004000; } + result.errorNode_ = errorNode_; if (((from_bitField0_ & 0x00010000) != 0)) { - result.optionsJson_ = optionsJson_; to_bitField0_ |= 0x00008000; } + result.optionsJson_ = optionsJson_; if (((from_bitField0_ & 0x00020000) != 0)) { result.planEnd_ = planEnd_; to_bitField0_ |= 0x00010000; @@ -17594,26 +17601,29 @@ private void buildPartial0(org.apache.drill.exec.proto.UserBitShared.QueryProfil to_bitField0_ |= 0x00040000; } if (((from_bitField0_ & 0x00100000) != 0)) { - result.queueName_ = queueName_; to_bitField0_ |= 0x00080000; } + result.queueName_ = queueName_; if (((from_bitField0_ & 0x00200000) != 0)) { - result.queryId_ = queryId_; to_bitField0_ |= 0x00100000; } + result.queryId_ = queryId_; if (((from_bitField0_ & 0x00400000) != 0)) { result.autoLimit_ = autoLimit_; to_bitField0_ |= 0x00200000; } - if (((from_bitField0_ & 0x00800000) != 0)) { - scannedPlugins_.makeImmutable(); - result.scannedPlugins_ = scannedPlugins_; + if (((bitField0_ & 0x00800000) != 0)) { + scannedPlugins_ = scannedPlugins_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00800000); } + result.scannedPlugins_ = scannedPlugins_; if (((from_bitField0_ & 0x01000000) != 0)) { result.planCacheHit_ = planCacheHit_; to_bitField0_ |= 0x00400000; } - result.bitField0_ |= to_bitField0_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; } @java.lang.Override @@ -17673,13 +17683,13 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile setEnd(other.getEnd()); } if (other.hasQuery()) { - query_ = other.query_; bitField0_ |= 0x00000010; + query_ = other.query_; onChanged(); } if (other.hasPlan()) { - plan_ = other.plan_; bitField0_ |= 0x00000020; + plan_ = other.plan_; onChanged(); } if (other.hasForeman()) { @@ -17721,33 +17731,33 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile } } if (other.hasUser()) { - user_ = other.user_; bitField0_ |= 0x00000800; + user_ = other.user_; onChanged(); } if (other.hasError()) { - error_ = other.error_; bitField0_ |= 0x00001000; + error_ = other.error_; onChanged(); } if (other.hasVerboseError()) { - verboseError_ = other.verboseError_; bitField0_ |= 0x00002000; + verboseError_ = other.verboseError_; onChanged(); } if (other.hasErrorId()) { - errorId_ = other.errorId_; bitField0_ |= 0x00004000; + errorId_ = other.errorId_; onChanged(); } if (other.hasErrorNode()) { - errorNode_ = other.errorNode_; bitField0_ |= 0x00008000; + errorNode_ = other.errorNode_; onChanged(); } if (other.hasOptionsJson()) { - optionsJson_ = other.optionsJson_; bitField0_ |= 0x00010000; + optionsJson_ = other.optionsJson_; onChanged(); } if (other.hasPlanEnd()) { @@ -17760,13 +17770,13 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile setTotalCost(other.getTotalCost()); } if (other.hasQueueName()) { - queueName_ = other.queueName_; bitField0_ |= 0x00100000; + queueName_ = other.queueName_; onChanged(); } if (other.hasQueryId()) { - queryId_ = other.queryId_; bitField0_ |= 0x00200000; + queryId_ = other.queryId_; onChanged(); } if (other.hasAutoLimit()) { @@ -17775,7 +17785,7 @@ public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.QueryProfile if (!other.scannedPlugins_.isEmpty()) { if (scannedPlugins_.isEmpty()) { scannedPlugins_ = other.scannedPlugins_; - bitField0_ |= 0x00800000; + bitField0_ = (bitField0_ & ~0x00800000); } else { ensureScannedPluginsIsMutable(); scannedPlugins_.addAll(other.scannedPlugins_); @@ -18010,11 +18020,11 @@ public Builder setId(org.apache.drill.exec.proto.UserBitShared.QueryId value) { throw new NullPointerException(); } id_ = value; + onChanged(); } else { idBuilder_.setMessage(value); } bitField0_ |= 0x00000001; - onChanged(); return this; } /** @@ -18024,11 +18034,11 @@ public Builder setId( org.apache.drill.exec.proto.UserBitShared.QueryId.Builder builderForValue) { if (idBuilder_ == null) { id_ = builderForValue.build(); + onChanged(); } else { idBuilder_.setMessage(builderForValue.build()); } bitField0_ |= 0x00000001; - onChanged(); return this; } /** @@ -18037,32 +18047,31 @@ public Builder setId( public Builder mergeId(org.apache.drill.exec.proto.UserBitShared.QueryId value) { if (idBuilder_ == null) { if (((bitField0_ & 0x00000001) != 0) && - id_ != null && - id_ != org.apache.drill.exec.proto.UserBitShared.QueryId.getDefaultInstance()) { - getIdBuilder().mergeFrom(value); + id_ != null && + id_ != org.apache.drill.exec.proto.UserBitShared.QueryId.getDefaultInstance()) { + id_ = + org.apache.drill.exec.proto.UserBitShared.QueryId.newBuilder(id_).mergeFrom(value).buildPartial(); } else { id_ = value; } + onChanged(); } else { idBuilder_.mergeFrom(value); } - if (id_ != null) { - bitField0_ |= 0x00000001; - onChanged(); - } + bitField0_ |= 0x00000001; return this; } /** * optional .exec.shared.QueryId id = 1; */ public Builder clearId() { - bitField0_ = (bitField0_ & ~0x00000001); - id_ = null; - if (idBuilder_ != null) { - idBuilder_.dispose(); - idBuilder_ = null; + if (idBuilder_ == null) { + id_ = null; + onChanged(); + } else { + idBuilder_.clear(); } - onChanged(); + bitField0_ = (bitField0_ & ~0x00000001); return this; } /** @@ -18115,7 +18124,8 @@ public org.apache.drill.exec.proto.UserBitShared.QueryIdOrBuilder getIdOrBuilder */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryType getType() { - org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.forNumber(type_); + @SuppressWarnings("deprecation") + org.apache.drill.exec.proto.UserBitShared.QueryType result = org.apache.drill.exec.proto.UserBitShared.QueryType.valueOf(type_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryType.SQL : result; } /** @@ -18166,9 +18176,8 @@ public long getStart() { * @return This builder for chaining. */ public Builder setStart(long value) { - - start_ = value; bitField0_ |= 0x00000004; + start_ = value; onChanged(); return this; } @@ -18206,9 +18215,8 @@ public long getEnd() { * @return This builder for chaining. */ public Builder setEnd(long value) { - - end_ = value; bitField0_ |= 0x00000008; + end_ = value; onChanged(); return this; } @@ -18273,9 +18281,11 @@ public java.lang.String getQuery() { */ public Builder setQuery( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; query_ = value; - bitField0_ |= 0x00000010; onChanged(); return this; } @@ -18284,8 +18294,8 @@ public Builder setQuery( * @return This builder for chaining. */ public Builder clearQuery() { - query_ = getDefaultInstance().getQuery(); bitField0_ = (bitField0_ & ~0x00000010); + query_ = getDefaultInstance().getQuery(); onChanged(); return this; } @@ -18296,9 +18306,11 @@ public Builder clearQuery() { */ public Builder setQueryBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; query_ = value; - bitField0_ |= 0x00000010; onChanged(); return this; } @@ -18353,9 +18365,11 @@ public java.lang.String getPlan() { */ public Builder setPlan( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; plan_ = value; - bitField0_ |= 0x00000020; onChanged(); return this; } @@ -18364,8 +18378,8 @@ public Builder setPlan( * @return This builder for chaining. */ public Builder clearPlan() { - plan_ = getDefaultInstance().getPlan(); bitField0_ = (bitField0_ & ~0x00000020); + plan_ = getDefaultInstance().getPlan(); onChanged(); return this; } @@ -18376,9 +18390,11 @@ public Builder clearPlan() { */ public Builder setPlanBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; plan_ = value; - bitField0_ |= 0x00000020; onChanged(); return this; } @@ -18413,11 +18429,11 @@ public Builder setForeman(org.apache.drill.exec.proto.CoordinationProtos.Drillbi throw new NullPointerException(); } foreman_ = value; + onChanged(); } else { foremanBuilder_.setMessage(value); } bitField0_ |= 0x00000040; - onChanged(); return this; } /** @@ -18427,11 +18443,11 @@ public Builder setForeman( org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.Builder builderForValue) { if (foremanBuilder_ == null) { foreman_ = builderForValue.build(); + onChanged(); } else { foremanBuilder_.setMessage(builderForValue.build()); } bitField0_ |= 0x00000040; - onChanged(); return this; } /** @@ -18440,32 +18456,31 @@ public Builder setForeman( public Builder mergeForeman(org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint value) { if (foremanBuilder_ == null) { if (((bitField0_ & 0x00000040) != 0) && - foreman_ != null && - foreman_ != org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.getDefaultInstance()) { - getForemanBuilder().mergeFrom(value); + foreman_ != null && + foreman_ != org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.getDefaultInstance()) { + foreman_ = + org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint.newBuilder(foreman_).mergeFrom(value).buildPartial(); } else { foreman_ = value; } + onChanged(); } else { foremanBuilder_.mergeFrom(value); } - if (foreman_ != null) { - bitField0_ |= 0x00000040; - onChanged(); - } + bitField0_ |= 0x00000040; return this; } /** * optional .exec.DrillbitEndpoint foreman = 7; */ public Builder clearForeman() { - bitField0_ = (bitField0_ & ~0x00000040); - foreman_ = null; - if (foremanBuilder_ != null) { - foremanBuilder_.dispose(); - foremanBuilder_ = null; + if (foremanBuilder_ == null) { + foreman_ = null; + onChanged(); + } else { + foremanBuilder_.clear(); } - onChanged(); + bitField0_ = (bitField0_ & ~0x00000040); return this; } /** @@ -18518,7 +18533,8 @@ public org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpointOrBuilder */ @java.lang.Override public org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState getState() { - org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.forNumber(state_); + @SuppressWarnings("deprecation") + org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState result = org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.valueOf(state_); return result == null ? org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState.STARTING : result; } /** @@ -18569,9 +18585,8 @@ public int getTotalFragments() { * @return This builder for chaining. */ public Builder setTotalFragments(int value) { - - totalFragments_ = value; bitField0_ |= 0x00000100; + totalFragments_ = value; onChanged(); return this; } @@ -18609,9 +18624,8 @@ public int getFinishedFragments() { * @return This builder for chaining. */ public Builder setFinishedFragments(int value) { - - finishedFragments_ = value; bitField0_ |= 0x00000200; + finishedFragments_ = value; onChanged(); return this; } @@ -18916,9 +18930,11 @@ public java.lang.String getUser() { */ public Builder setUser( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000800; user_ = value; - bitField0_ |= 0x00000800; onChanged(); return this; } @@ -18927,8 +18943,8 @@ public Builder setUser( * @return This builder for chaining. */ public Builder clearUser() { - user_ = getDefaultInstance().getUser(); bitField0_ = (bitField0_ & ~0x00000800); + user_ = getDefaultInstance().getUser(); onChanged(); return this; } @@ -18939,9 +18955,11 @@ public Builder clearUser() { */ public Builder setUserBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000800; user_ = value; - bitField0_ |= 0x00000800; onChanged(); return this; } @@ -18996,9 +19014,11 @@ public java.lang.String getError() { */ public Builder setError( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00001000; error_ = value; - bitField0_ |= 0x00001000; onChanged(); return this; } @@ -19007,8 +19027,8 @@ public Builder setError( * @return This builder for chaining. */ public Builder clearError() { - error_ = getDefaultInstance().getError(); bitField0_ = (bitField0_ & ~0x00001000); + error_ = getDefaultInstance().getError(); onChanged(); return this; } @@ -19019,9 +19039,11 @@ public Builder clearError() { */ public Builder setErrorBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00001000; error_ = value; - bitField0_ |= 0x00001000; onChanged(); return this; } @@ -19076,9 +19098,11 @@ public java.lang.String getVerboseError() { */ public Builder setVerboseError( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00002000; verboseError_ = value; - bitField0_ |= 0x00002000; onChanged(); return this; } @@ -19087,8 +19111,8 @@ public Builder setVerboseError( * @return This builder for chaining. */ public Builder clearVerboseError() { - verboseError_ = getDefaultInstance().getVerboseError(); bitField0_ = (bitField0_ & ~0x00002000); + verboseError_ = getDefaultInstance().getVerboseError(); onChanged(); return this; } @@ -19099,9 +19123,11 @@ public Builder clearVerboseError() { */ public Builder setVerboseErrorBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00002000; verboseError_ = value; - bitField0_ |= 0x00002000; onChanged(); return this; } @@ -19156,9 +19182,11 @@ public java.lang.String getErrorId() { */ public Builder setErrorId( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00004000; errorId_ = value; - bitField0_ |= 0x00004000; onChanged(); return this; } @@ -19167,8 +19195,8 @@ public Builder setErrorId( * @return This builder for chaining. */ public Builder clearErrorId() { - errorId_ = getDefaultInstance().getErrorId(); bitField0_ = (bitField0_ & ~0x00004000); + errorId_ = getDefaultInstance().getErrorId(); onChanged(); return this; } @@ -19179,9 +19207,11 @@ public Builder clearErrorId() { */ public Builder setErrorIdBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00004000; errorId_ = value; - bitField0_ |= 0x00004000; onChanged(); return this; } @@ -19236,9 +19266,11 @@ public java.lang.String getErrorNode() { */ public Builder setErrorNode( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00008000; errorNode_ = value; - bitField0_ |= 0x00008000; onChanged(); return this; } @@ -19247,8 +19279,8 @@ public Builder setErrorNode( * @return This builder for chaining. */ public Builder clearErrorNode() { - errorNode_ = getDefaultInstance().getErrorNode(); bitField0_ = (bitField0_ & ~0x00008000); + errorNode_ = getDefaultInstance().getErrorNode(); onChanged(); return this; } @@ -19259,9 +19291,11 @@ public Builder clearErrorNode() { */ public Builder setErrorNodeBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00008000; errorNode_ = value; - bitField0_ |= 0x00008000; onChanged(); return this; } @@ -19316,9 +19350,11 @@ public java.lang.String getOptionsJson() { */ public Builder setOptionsJson( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00010000; optionsJson_ = value; - bitField0_ |= 0x00010000; onChanged(); return this; } @@ -19327,8 +19363,8 @@ public Builder setOptionsJson( * @return This builder for chaining. */ public Builder clearOptionsJson() { - optionsJson_ = getDefaultInstance().getOptionsJson(); bitField0_ = (bitField0_ & ~0x00010000); + optionsJson_ = getDefaultInstance().getOptionsJson(); onChanged(); return this; } @@ -19339,9 +19375,11 @@ public Builder clearOptionsJson() { */ public Builder setOptionsJsonBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00010000; optionsJson_ = value; - bitField0_ |= 0x00010000; onChanged(); return this; } @@ -19369,9 +19407,8 @@ public long getPlanEnd() { * @return This builder for chaining. */ public Builder setPlanEnd(long value) { - - planEnd_ = value; bitField0_ |= 0x00020000; + planEnd_ = value; onChanged(); return this; } @@ -19409,9 +19446,8 @@ public long getQueueWaitEnd() { * @return This builder for chaining. */ public Builder setQueueWaitEnd(long value) { - - queueWaitEnd_ = value; bitField0_ |= 0x00040000; + queueWaitEnd_ = value; onChanged(); return this; } @@ -19449,9 +19485,8 @@ public double getTotalCost() { * @return This builder for chaining. */ public Builder setTotalCost(double value) { - - totalCost_ = value; bitField0_ |= 0x00080000; + totalCost_ = value; onChanged(); return this; } @@ -19516,9 +19551,11 @@ public java.lang.String getQueueName() { */ public Builder setQueueName( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00100000; queueName_ = value; - bitField0_ |= 0x00100000; onChanged(); return this; } @@ -19527,8 +19564,8 @@ public Builder setQueueName( * @return This builder for chaining. */ public Builder clearQueueName() { - queueName_ = getDefaultInstance().getQueueName(); bitField0_ = (bitField0_ & ~0x00100000); + queueName_ = getDefaultInstance().getQueueName(); onChanged(); return this; } @@ -19539,9 +19576,11 @@ public Builder clearQueueName() { */ public Builder setQueueNameBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00100000; queueName_ = value; - bitField0_ |= 0x00100000; onChanged(); return this; } @@ -19596,9 +19635,11 @@ public java.lang.String getQueryId() { */ public Builder setQueryId( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00200000; queryId_ = value; - bitField0_ |= 0x00200000; onChanged(); return this; } @@ -19607,8 +19648,8 @@ public Builder setQueryId( * @return This builder for chaining. */ public Builder clearQueryId() { - queryId_ = getDefaultInstance().getQueryId(); bitField0_ = (bitField0_ & ~0x00200000); + queryId_ = getDefaultInstance().getQueryId(); onChanged(); return this; } @@ -19619,9 +19660,11 @@ public Builder clearQueryId() { */ public Builder setQueryIdBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00200000; queryId_ = value; - bitField0_ |= 0x00200000; onChanged(); return this; } @@ -19649,9 +19692,8 @@ public int getAutoLimit() { * @return This builder for chaining. */ public Builder setAutoLimit(int value) { - - autoLimit_ = value; bitField0_ |= 0x00400000; + autoLimit_ = value; onChanged(); return this; } @@ -19666,13 +19708,12 @@ public Builder clearAutoLimit() { return this; } - private com.google.protobuf.LazyStringArrayList scannedPlugins_ = - com.google.protobuf.LazyStringArrayList.emptyList(); + private com.google.protobuf.LazyStringList scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; private void ensureScannedPluginsIsMutable() { - if (!scannedPlugins_.isModifiable()) { + if (!((bitField0_ & 0x00800000) != 0)) { scannedPlugins_ = new com.google.protobuf.LazyStringArrayList(scannedPlugins_); - } - bitField0_ |= 0x00800000; + bitField0_ |= 0x00800000; + } } /** * repeated string scanned_plugins = 24; @@ -19680,8 +19721,7 @@ private void ensureScannedPluginsIsMutable() { */ public com.google.protobuf.ProtocolStringList getScannedPluginsList() { - scannedPlugins_.makeImmutable(); - return scannedPlugins_; + return scannedPlugins_.getUnmodifiableView(); } /** * repeated string scanned_plugins = 24; @@ -19715,10 +19755,11 @@ public java.lang.String getScannedPlugins(int index) { */ public Builder setScannedPlugins( int index, java.lang.String value) { - if (value == null) { throw new NullPointerException(); } - ensureScannedPluginsIsMutable(); + if (value == null) { + throw new NullPointerException(); + } + ensureScannedPluginsIsMutable(); scannedPlugins_.set(index, value); - bitField0_ |= 0x00800000; onChanged(); return this; } @@ -19729,10 +19770,11 @@ public Builder setScannedPlugins( */ public Builder addScannedPlugins( java.lang.String value) { - if (value == null) { throw new NullPointerException(); } - ensureScannedPluginsIsMutable(); + if (value == null) { + throw new NullPointerException(); + } + ensureScannedPluginsIsMutable(); scannedPlugins_.add(value); - bitField0_ |= 0x00800000; onChanged(); return this; } @@ -19746,7 +19788,6 @@ public Builder addAllScannedPlugins( ensureScannedPluginsIsMutable(); com.google.protobuf.AbstractMessageLite.Builder.addAll( values, scannedPlugins_); - bitField0_ |= 0x00800000; onChanged(); return this; } @@ -19755,9 +19796,8 @@ public Builder addAllScannedPlugins( * @return This builder for chaining. */ public Builder clearScannedPlugins() { - scannedPlugins_ = - com.google.protobuf.LazyStringArrayList.emptyList(); - bitField0_ = (bitField0_ & ~0x00800000);; + scannedPlugins_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00800000); onChanged(); return this; } @@ -19768,10 +19808,11 @@ public Builder clearScannedPlugins() { */ public Builder addScannedPluginsBytes( com.google.protobuf.ByteString value) { - if (value == null) { throw new NullPointerException(); } - ensureScannedPluginsIsMutable(); + if (value == null) { + throw new NullPointerException(); + } + ensureScannedPluginsIsMutable(); scannedPlugins_.add(value); - bitField0_ |= 0x00800000; onChanged(); return this; } @@ -19799,9 +19840,8 @@ public boolean getPlanCacheHit() { * @return This builder for chaining. */ public Builder setPlanCacheHit(boolean value) { - - planCacheHit_ = value; bitField0_ |= 0x01000000; + planCacheHit_ = value; onChanged(); return this; } From 8f1249a1c9d97f22cd753f2fa7294e9861576476 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Tue, 6 Oct 2026 14:33:49 +0800 Subject: [PATCH 05/16] Fix plan cache eligibility and BIGINT plan round trips --- PLAN_CACHE_DESIGN.md | 3 +- PLAN_CACHE_PLUGIN_GUIDE.md | 30 +-- .../exec/store/iceberg/IcebergGroupScan.java | 11 -- .../exec/store/hbase/HBaseGroupScan.java | 6 - .../drill/exec/physical/base/GroupScan.java | 9 - .../exec/planner/sql/DrillSqlWorker.java | 107 +++-------- .../drill/exec/planner/sql/PlanCache.java | 173 ++++++++++-------- .../drill/exec/store/StoragePlugin.java | 8 +- .../exec/store/dfs/FileSystemPlugin.java | 10 + .../drill/exec/store/dfs/FormatPlugin.java | 2 +- .../common/expression/parser/ExprParser.g4 | 21 ++- .../expression/ExpressionStringBuilder.java | 2 +- 12 files changed, 168 insertions(+), 214 deletions(-) diff --git a/PLAN_CACHE_DESIGN.md b/PLAN_CACHE_DESIGN.md index 06fe2446b72..77fe952e3af 100644 --- a/PLAN_CACHE_DESIGN.md +++ b/PLAN_CACHE_DESIGN.md @@ -72,8 +72,7 @@ predicate. Reusing the first execution's filters or file list would be incorrect ## What can be reused Read queries can include joins, aggregates, subqueries, CTEs, ordering and limits. -Every `GroupScan` in a plan must explicitly support caching. This change opts in -HBase and Iceberg; other plugins retain the default of no support. +Every source plugin must explicitly support caching and guarantee safe reconstruction of its scans. This change opts in HBase and Iceberg; other plugins retain the default of no support. Constants that determine SQL structure stay in the key: group/order expressions, window definitions, limit/offset, field selectors, and function configuration diff --git a/PLAN_CACHE_PLUGIN_GUIDE.md b/PLAN_CACHE_PLUGIN_GUIDE.md index 42528923066..72e718924f6 100644 --- a/PLAN_CACHE_PLUGIN_GUIDE.md +++ b/PLAN_CACHE_PLUGIN_GUIDE.md @@ -18,10 +18,7 @@ limitations under the License. # Adding plan-cache support to a plugin -Plan caching is disabled by default. A plugin is eligible only when both its -metadata interface and every participating `GroupScan` opt in. The engine binds -SQL literals in serialized Drill expressions; the plugin remains responsible for -rebuilding scan state derived from those expressions. +Plan caching is disabled by default. A plugin opts in through its metadata interface and guarantees that every scan of an eligible selection can be safely rebuilt. The engine binds SQL literals in serialized Drill expressions; the plugin remains responsible for rebuilding scan state derived from those expressions. ## Storage-plugin API @@ -30,15 +27,11 @@ Implement these methods in [StoragePlugin](exec/java-exec/src/main/java/org/apac ```java boolean supportPlanCache(); PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException; -String planCacheTableVersion(String identifier) throws IOException; ``` -`planCacheTable` is called during a miss. Return a stable, opaque physical table -identifier and a compatibility version. Return `null` for unsupported selections. -`planCacheTableVersion` is called on a hit with that same identifier; it should -read the current version without rebuilding the SQL schema. Return `null` if the -table is missing or unsupported. Throwing also causes a fallback to normal -planning. The engine separately compares the storage-plugin configuration. +`planCacheTable` is called before each lookup to capture the current query's table dependencies. Return a stable, opaque physical table identifier and a compatibility version. Return `null` for missing tables or unsupported selections. Failed metadata reads also cause a fallback to normal planning. The engine compares this complete context snapshot, including effective options and storage-plugin configurations, with the cached entry's snapshot. A miss retains the same snapshot for publication after successful execution. + +The engine first checks every participating plugin's eligibility and returns immediately if any plugin does not support caching. Override `supportPlanCache(DrillTableSelection)` when support depends on the selection; its default delegates to `supportPlanCache()`. Keep this check free of table-version reads. DFS uses it to check the selected format plugin. Only after all checks pass does the engine call `planCacheTable` and compute configuration and option fingerprints. The version must change whenever the cached plan's table definition becomes incompatible. Do not use a data snapshot number if compatible data updates should @@ -49,13 +42,11 @@ For a DFS format, implement `supportPlanCache()` and both `planCacheTableVersion(FileSelection)` and `planCacheTableVersion(Path)` in [FormatPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java). [FileSystemPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java) -provides the storage-plugin bridge. The selection overload validates the original -selection; the path overload validates a hit without rediscovering the schema. +provides the storage-plugin bridge. The selection overload validates the current selection and supplies its compatibility version; it can delegate metadata reads to the path overload. ## Scan API and JSON reconstruction -Override `GroupScan.supportPlanCache()`; its default is `false`. The engine rejects -a plan containing even one unsupported scan. +Declaring plugin support covers every `GroupScan` produced for an eligible selection. Ensure that all planning, pushdown and cloning paths preserve the inputs needed for reconstruction. Unsupported selections must be rejected during snapshot construction. Keep the bound `LogicalExpression` in the scan's JSON. Converting a predicate to a native filter or file list and dropping the source expression loses the @@ -70,7 +61,7 @@ information needed to bind the next query. On deserialization: Serialize stable inputs rather than live connections, execution resources or old endpoint assignments. Validate the round trip with Drill's `PhysicalPlanReader`. If a parameter changes projection, schema, limit or another derived field that -cannot be rebuilt, keep that argument structural or leave the scan ineligible. +cannot be rebuilt, keep that argument structural or leave the selection ineligible. For explain output containing expressions, use `ExpressionStringBuilder.toExplainString()` via `getExplainDigest()` to display @@ -96,7 +87,7 @@ pushdown and residual filtering, and an empty scan after rebinding. Verify that compatible data updates are visible on hits, while incompatible schema changes, deletions, plugin configuration changes and failed metadata reads reject the old entry. Include joins or subqueries so every table dependency is checked, and -confirm that a mixed plan containing an unsupported scan is rejected. +confirm that queries involving an unsupported plugin or selection bypass caching. Check `planCacheHit` in the query profile. In isolated tests, the Drillbit's hit counter can also establish reuse: @@ -115,7 +106,4 @@ workloads. `awaitWrites()` is a test helper, not a SQL operation. ## Scope of the contract -The cache validates saved physical identifiers. Sessions with temporary tables -or mutable user/public aliases bypass caching, because those mappings can redirect -a SQL name without changing the old physical table. Statistics changes alone do -not trigger re-optimization. +The cache compares the current query's physical identifiers and versions with those captured for the cached plan. Sessions with temporary tables or mutable user/public aliases bypass caching. Statistics changes alone do not trigger re-optimization. diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java index 3af3b9885ce..4beedd4fd0d 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java @@ -37,7 +37,6 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.drill.exec.store.dfs.DrillFileSystem; -import org.apache.drill.exec.store.iceberg.format.IcebergFormatLocationTransformer; import org.apache.drill.exec.store.iceberg.format.IcebergFormatPlugin; import org.apache.drill.exec.store.iceberg.plan.DrillExprToIcebergTranslator; import org.apache.drill.exec.store.iceberg.snapshot.Snapshot; @@ -185,16 +184,6 @@ public IcebergGroupScan clone(List columns) { } } - @Override - public boolean supportPlanCache() { - return formatPlugin.supportPlanCache() && !isMetadataTablePath(); - } - - private boolean isMetadataTablePath() { - // Metadata table paths use the form #. - return path.contains(IcebergFormatLocationTransformer.METADATA_SEPARATOR); - } - @Override public IcebergGroupScan applyLimit(int maxRecords) { IcebergGroupScan clone = new IcebergGroupScan(this); diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java index 99ac66a49dd..f9bebaae3f3 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java @@ -504,12 +504,6 @@ public boolean isPushedFilterFullyConverted() { return pushedFilterFullyConverted; } - @Override - @JsonIgnore - public boolean supportPlanCache() { - return storagePlugin.supportPlanCache() && (pushedFilter == null || baseScanSpec != null); - } - @Override @JsonIgnore public boolean canPushdownProjects(List columns) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index 48df200a8dd..cb75b2f03f4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -43,15 +43,6 @@ */ public interface GroupScan extends Scan, HasAffinity { - /** - * The scan implementation must reconstruct value-dependent work from rebound - * expressions during physical-plan deserialization, including pruning and - * assignments that depend on the original predicate. - */ - default boolean supportPlanCache() { - return false; - } - /** * columns list in GroupScan : 1) empty_column is for skipAll query. * 2) NULL is interpreted as ALL_COLUMNS. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index 40fd162662b..91e872185d1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -18,9 +18,6 @@ package org.apache.drill.exec.planner.sql; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.TreeMap; import org.apache.calcite.sql.SqlDescribeSchema; import org.apache.calcite.sql.SqlKind; @@ -38,7 +35,6 @@ import org.apache.drill.exec.exception.MetadataException; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.ops.QueryContext.SqlStatementType; -import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler; @@ -66,7 +62,6 @@ import org.apache.drill.exec.work.foreman.ForemanSetupException; import org.apache.drill.exec.work.foreman.SqlUnsupportedException; import com.google.common.base.Throwables; -import com.google.common.hash.Hashing; import org.apache.hadoop.security.AccessControlException; public class DrillSqlWorker { @@ -306,55 +301,40 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point && !hasMutableAliases(context) && PlanCacheEligibility.isSafeToCache(sqlNode, context.getFunctionRegistry())) { try { - PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(sqlNode); - PlanCache cache = context.getDrillbitContext().getPlanCache(); - String key = context.getQueryUserName() + '\n' - + context.getSession().getDefaultSchemaPath() + '\n' - + candidate.template; - PlanCache.Entry entry = cache.get(key); - String optionsFingerprint = null; - if (entry != null) { - try { - optionsFingerprint = optionFingerprint(context); - if (entry.matchesContext(optionsFingerprint, context.getStorage())) { - PhysicalPlan bound = entry.bind(candidate.literals, - context.getDrillbitContext().getPlanReader()); - if (textPlan != null) { - textPlan.value = PlanCache.textWithBindings(entry.getTextPlan(), candidate.literals); - } - cache.recordBind(); - context.setPlanCacheHit(); - return bound; - } - } catch (RuntimeException | IOException e) { - logger.debug("Cached plan could not be rebound", e); - } - cache.invalidate(key); - } - // Capture the context before planning. Changes during planning invalidate - // this entry on its first hit. + // Capture the context once, before lookup and planning. Changes during + // planning invalidate this entry on its first hit. PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve( - parser.getDefaultSchema(), sqlNode, - context.getStorage()); + parser.getDefaultSchema(), sqlNode, context); if (snapshot != null) { - if (optionsFingerprint == null) { - optionsFingerprint = optionFingerprint(context); + PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(sqlNode); + PlanCache cache = context.getDrillbitContext().getPlanCache(); + String key = context.getQueryUserName() + '\n' + + context.getSession().getDefaultSchemaPath() + '\n' + + candidate.template; + PlanCache.Entry entry = cache.get(key); + if (entry != null) { + try { + if (entry.matchesContext(snapshot)) { + PhysicalPlan bound = entry.bind(candidate.literals, + context.getDrillbitContext().getPlanReader()); + PlanCache.bindTextPlan(textPlan, entry.getTextPlan(), candidate.literals); + cache.recordBind(); + context.setPlanCacheHit(); + return bound; + } + } catch (RuntimeException | IOException e) { + logger.debug("Cached plan could not be rebound", e); + } + cache.invalidate(key); } - snapshot = snapshot.withOptionsFingerprint(optionsFingerprint); - } - PhysicalPlan planned = handler.getPlan(candidate.sql); - final String templateTextPlan = textPlan == null ? null : textPlan.value; - if (textPlan != null) { - textPlan.value = PlanCache.textWithBindings(templateTextPlan, candidate.literals); - } - if (snapshot != null && cache.canCache(planned)) { + PhysicalPlan planned = handler.getPlan(candidate.sql); + final String templateTextPlan = PlanCache.bindTextPlan(textPlan, candidate.literals); final org.apache.drill.exec.planner.PhysicalPlanReader cacheReader = context.getDrillbitContext().getPlanReader(); - final PlanCache.ContextSnapshot cacheContext = snapshot; context.setPendingPlanCacheInsert(() -> cache.writeAfterSuccess(key, planned, - templateTextPlan, cacheReader, cacheContext)); + templateTextPlan, cacheReader, snapshot)); + return planned; } - return planned; } catch (Exception e) { logger.debug("Plan cache attempt fell back to ordinary planning", e); } @@ -363,39 +343,6 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point return handler.getPlan(sqlNode); } - private static String optionFingerprint(QueryContext context) { - // OptionManager iteration walks system defaults, system overrides, session - // overrides, and query overrides in precedence order. getOptionList() only - // exposes local session/query values and misses system-level changes. - Map options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - for (OptionValue option : context.getOptions()) { - options.put(option.name, option); - } - StringBuilder fingerprint = new StringBuilder(); - for (OptionValue option : options.values()) { - fingerprint.append(option.name).append(':').append(option.kind).append('='); - switch (option.kind) { - case BOOLEAN: - fingerprint.append(option.bool_val); - break; - case LONG: - fingerprint.append(option.num_val); - break; - case DOUBLE: - fingerprint.append(option.float_val); - break; - case STRING: - fingerprint.append(option.string_val.length()).append(':').append(option.string_val); - break; - default: - throw new IllegalArgumentException("Unsupported option kind"); - } - fingerprint.append('\n'); - } - // Keep the cache key bounded even when the system has many registered options. - return Hashing.sha256().hashString(fingerprint, StandardCharsets.UTF_8).toString(); - } - private static boolean hasMutableAliases(QueryContext context) { // Table-version checks cannot detect an alias redirected to another table. AliasRegistry[] registries = { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java index 0dfda14891a..31ca90485d6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; @@ -54,16 +55,17 @@ import org.apache.drill.common.parser.LogicalExpressionParser; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.PhysicalPlan; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.planner.PhysicalPlanReader; import org.apache.drill.exec.planner.logical.DrillTable; import org.apache.drill.exec.planner.logical.DrillTableSelection; import org.apache.drill.exec.rpc.NamedThreadFactory; +import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.store.StoragePlugin; import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.StoragePluginRegistry.PluginException; import org.apache.drill.exec.store.PlanCacheTable; +import org.apache.drill.exec.util.Pointer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Objects; @@ -100,8 +102,22 @@ public void recordBind() { successfulBinds.incrementAndGet(); } + /** Updates optional explain output using the cached template. */ + static void bindTextPlan(Pointer textPlan, String template, List literals) { + if (textPlan != null) { + textPlan.value = textWithBindings(template, literals); + } + } + + /** Updates optional explain output and returns its original template for cache publication. */ + static String bindTextPlan(Pointer textPlan, List literals) { + String template = textPlan == null ? null : textPlan.value; + bindTextPlan(textPlan, template, literals); + return template; + } + /** Appends values from this query without changing the cached explain template. */ - static String textWithBindings(String template, List literals) { + private static String textWithBindings(String template, List literals) { if (template == null || literals.isEmpty()) { return template; } @@ -129,11 +145,6 @@ public void awaitWrites() throws InterruptedException, ExecutionException, Timeo writer.submit(() -> { }).get(10, TimeUnit.SECONDS); } - public boolean canCache(PhysicalPlan plan) { - return plan.getSortedOperators().stream() - .noneMatch(op -> op instanceof GroupScan && !((GroupScan) op).supportPlanCache()); - } - public void writeAfterSuccess(String key, PhysicalPlan plan, String textPlan, PhysicalPlanReader reader, ContextSnapshot context) { try { @@ -156,8 +167,8 @@ public void invalidate(String key) { public boolean put(String key, PhysicalPlan plan, String textPlan, PhysicalPlanReader reader, ContextSnapshot context) throws IOException { // The engine checks generic context and expression compatibility. Each - // opted-in scan is responsible for rebuilding its value-dependent state. - if (context == null || context.optionsFingerprint == null || !canCache(plan)) { + // opted-in plugin is responsible for rebuilding its scans' value-dependent state. + if (context == null) { return false; } String json = reader.writeJson(plan); @@ -183,8 +194,8 @@ private Entry(String json, String textPlan, ContextSnapshot context) { this.context = context; } - public boolean matchesContext(String optionsFingerprint, StoragePluginRegistry plugins) { - return context.isCurrent(optionsFingerprint, plugins); + public boolean matchesContext(ContextSnapshot current) { + return context.matches(current); } public String getTextPlan() { @@ -199,7 +210,7 @@ public PhysicalPlan bind(List literals, PhysicalPlanReader reader) } } - /** Options, plugin configurations, and table versions checked after lookup. */ + /** Options, plugin configurations, and table versions captured before lookup and planning. */ public static final class ContextSnapshot { private final String optionsFingerprint; private final Map tableVersions; @@ -207,15 +218,11 @@ public static final class ContextSnapshot { private ContextSnapshot(String optionsFingerprint, Map tableVersions, Map pluginConfigs) { - this.optionsFingerprint = optionsFingerprint; + this.optionsFingerprint = Objects.requireNonNull(optionsFingerprint, "optionsFingerprint"); this.tableVersions = java.util.Collections.unmodifiableMap(tableVersions); this.pluginConfigs = java.util.Collections.unmodifiableMap(pluginConfigs); } - public ContextSnapshot withOptionsFingerprint(String fingerprint) { - return new ContextSnapshot(Objects.requireNonNull(fingerprint), tableVersions, pluginConfigs); - } - private static final class TableIdentifier { private final String storageName; private final String tableId; @@ -246,24 +253,26 @@ public int hashCode() { private static final class ResolvedTable { private final List tableNames; - private final TableIdentifier identifier; - private final String version; - private final StoragePluginConfig pluginConfig; + private final String storageName; + private final StoragePlugin plugin; + private final DrillTableSelection selection; - private ResolvedTable(List tableNames, TableIdentifier identifier, String version, - StoragePluginConfig pluginConfig) { + private ResolvedTable(List tableNames, String storageName, + StoragePlugin plugin, DrillTableSelection selection) { this.tableNames = tableNames; - this.identifier = identifier; - this.version = version; - this.pluginConfig = pluginConfig; + this.storageName = storageName; + this.plugin = plugin; + this.selection = selection; } } /** Resolves every physical table in a query, including joins and subqueries. */ public static ContextSnapshot resolve(SchemaPlus defaultSchema, SqlNode query, - StoragePluginRegistry plugins) { + QueryContext context) { + StoragePluginRegistry plugins = context.getStorage(); List dependencies = new ArrayList<>(); try { + // Check every participating plugin before reading any table version. if (!collectQuery(defaultSchema, query, new HashSet<>(), dependencies)) { return null; } @@ -274,23 +283,28 @@ public static ContextSnapshot resolve(SchemaPlus defaultSchema, SqlNode query, Map versions = new LinkedHashMap<>(); Map configs = new LinkedHashMap<>(); for (ResolvedTable dependency : dependencies) { - String previous = versions.putIfAbsent(dependency.identifier, dependency.version); - if (previous != null && !previous.equals(dependency.version)) { - return null; - } try { - String fingerprint = configFingerprint(plugins, dependency.pluginConfig); - String previousConfig = configs.putIfAbsent(dependency.identifier.storageName, - fingerprint); + PlanCacheTable source = dependency.plugin.planCacheTable(dependency.selection); + if (source == null) { + return null; + } + TableIdentifier identifier = new TableIdentifier(dependency.storageName, + source.getIdentifier()); + String previous = versions.putIfAbsent(identifier, source.getVersion()); + if (previous != null && !previous.equals(source.getVersion())) { + return null; + } + String fingerprint = configFingerprint(plugins, dependency.plugin.getConfig()); + String previousConfig = configs.putIfAbsent(dependency.storageName, fingerprint); if (previousConfig != null && !previousConfig.equals(fingerprint)) { return null; } - } catch (RuntimeException e) { - logger.debug("Plugin configuration could not be fingerprinted for the plan cache", e); + } catch (IOException | RuntimeException e) { + logger.debug("Table context could not be captured for the plan cache", e); return null; } } - return new ContextSnapshot(null, versions, configs); + return new ContextSnapshot(optionFingerprint(context), versions, configs); } private static boolean collectQuery(SchemaPlus schema, SqlNode node, @@ -400,56 +414,53 @@ private static ResolvedTable resolveTable(SchemaPlus defaultSchema, List return null; } StoragePlugin storagePlugin = drillTable.getPlugin(); - if (!storagePlugin.supportPlanCache()) { - return null; - } - try { - PlanCacheTable source = storagePlugin.planCacheTable((DrillTableSelection) selection); - if (source == null) { - return null; - } - return new ResolvedTable( - java.util.Collections.unmodifiableList(new java.util.ArrayList<>(names)), - new TableIdentifier(drillTable.getStorageEngineName(), source.getIdentifier()), - source.getVersion(), storagePlugin.getConfig()); - } catch (IOException e) { - logger.debug("Table version could not be read for the plan cache", e); + if (!storagePlugin.supportPlanCache() + || !storagePlugin.supportPlanCache((DrillTableSelection) selection)) { return null; } + return new ResolvedTable( + java.util.Collections.unmodifiableList(new java.util.ArrayList<>(names)), + drillTable.getStorageEngineName(), storagePlugin, (DrillTableSelection) selection); } - public boolean isCurrent(String currentOptionsFingerprint, StoragePluginRegistry plugins) { - if (!Objects.equals(optionsFingerprint, currentOptionsFingerprint)) { - return false; - } - Map currentPlugins = new HashMap<>(); - for (Map.Entry entry : pluginConfigs.entrySet()) { - try { - StoragePlugin plugin = plugins.getPlugin(entry.getKey()); - if (plugin == null || !plugin.supportPlanCache() - || !entry.getValue().equals(configFingerprint(plugins, plugin.getConfig()))) { - return false; - } - currentPlugins.put(entry.getKey(), plugin); - } catch (PluginException | RuntimeException e) { - logger.debug("Plugin configuration could not be checked for the plan cache", e); - return false; - } - } - for (Map.Entry entry : tableVersions.entrySet()) { - TableIdentifier identifier = entry.getKey(); - try { - StoragePlugin storagePlugin = currentPlugins.get(identifier.storageName); - if (!Objects.equals(entry.getValue(), - storagePlugin.planCacheTableVersion(identifier.tableId))) { - return false; - } - } catch (IOException e) { - logger.debug("Table version could not be read for the plan cache", e); - return false; + private boolean matches(ContextSnapshot current) { + return current != null + && optionsFingerprint.equals(current.optionsFingerprint) + && tableVersions.equals(current.tableVersions) + && pluginConfigs.equals(current.pluginConfigs); + } + + private static String optionFingerprint(QueryContext context) { + // OptionManager iteration walks system defaults, system overrides, session + // overrides, and query overrides in precedence order. getOptionList() only + // exposes local session/query values and misses system-level changes. + Map options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (OptionValue option : context.getOptions()) { + options.put(option.name, option); + } + StringBuilder fingerprint = new StringBuilder(); + for (OptionValue option : options.values()) { + fingerprint.append(option.name).append(':').append(option.kind).append('='); + switch (option.kind) { + case BOOLEAN: + fingerprint.append(option.bool_val); + break; + case LONG: + fingerprint.append(option.num_val); + break; + case DOUBLE: + fingerprint.append(option.float_val); + break; + case STRING: + fingerprint.append(option.string_val.length()).append(':').append(option.string_val); + break; + default: + throw new IllegalArgumentException("Unsupported option kind"); } + fingerprint.append('\n'); } - return true; + // Keep the cache key bounded even when the system has many registered options. + return Hashing.sha256().hashString(fingerprint, StandardCharsets.UTF_8).toString(); } private static String configFingerprint(StoragePluginRegistry plugins, diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java index f61f4dab4ef..a481fbabd79 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -75,12 +75,18 @@ public interface StoragePlugin extends SchemaFactory, AutoCloseable { /** * Opts this plugin into plan caching. The engine validates query context and * rebinds expression parameters; the plugin must provide a plan-compatible - * table version and rebuild value-dependent scan state for each execution. + * table version and safely rebuild every scan for an eligible selection, + * including value-dependent state for each execution. */ default boolean supportPlanCache() { return false; } + /** Checks selection eligibility without reading table versions or building cache metadata. */ + default boolean supportPlanCache(DrillTableSelection selection) { + return supportPlanCache(); + } + /** * Returns a stable table identifier and a version that changes whenever the * cached operator structure is no longer compatible. Data-only changes may diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index e4f99b026b3..40a4bf8ef69 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -338,6 +338,16 @@ public boolean supportPlanCache() { return true; } + @Override + public boolean supportPlanCache(DrillTableSelection selection) { + if (!(selection instanceof FormatSelection)) { + return false; + } + FormatSelection formatSelection = (FormatSelection) selection; + FormatPlugin format = getFormatPlugin(formatSelection.getFormat()); + return format != null && format.supportPlanCache(); + } + @Override public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { if (!(selection instanceof FormatSelection)) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java index ba88f33d0dc..7ec89974e1f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java @@ -48,7 +48,7 @@ public interface FormatPlugin { boolean supportsWrite(); - /** Whether scans of this format can be rebuilt for a cached plan. */ + /** Whether every scan of an eligible selection can be safely rebuilt for a cached plan. */ default boolean supportPlanCache() { return false; } diff --git a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 index 5d2dcae9288..2abd4f1b636 100644 --- a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 +++ b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 @@ -47,6 +47,25 @@ import org.apache.drill.common.exceptions.ExpressionParsingException; public ExpressionPosition pos(Token token){ return new ExpressionPosition(fullExpression, token.getTokenIndex()); } + + private LogicalExpression createCast(MajorType type, ExpressionPosition position, + LogicalExpression input) { + // BIGINT casts emitted for integer literals preserve their width in JSON. + // Restore the literal node so scan translators see the original predicate. + // Keep parameter casts intact to retain their slot identity and conversion. + if (type.equals(Types.required(MinorType.BIGINT)) + && input instanceof LiteralExpression + && ((LiteralExpression) input).getDynamicParamIndex() < 0) { + if (input instanceof ValueExpressions.IntExpression) { + return new ValueExpressions.LongExpression( + ((ValueExpressions.IntExpression) input).getInt(), position); + } + if (input instanceof ValueExpressions.LongExpression) { + return input; + } + } + return FunctionCallFactory.createCast(type, position, input); + } } parse returns [LogicalExpression e] @@ -83,7 +102,7 @@ castCall returns [LogicalExpression e] { if ($repeat.ctx != null && $repeat.isRep.compareTo(Boolean.TRUE)==0) $e = FunctionCallFactory.createCast(TypeProtos.MajorType.newBuilder().mergeFrom($dataType.type).setMode(DataMode.REPEATED).build(), pos($Cast), $expression.e); else - $e = FunctionCallFactory.createCast($dataType.type, pos($Cast), $expression.e);} + $e = createCast($dataType.type, pos($Cast), $expression.e);} ; repeat returns [Boolean isRep] diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 682c25951bd..9bb6f787907 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -201,7 +201,7 @@ public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws Run return null; } if (lExpr.getDynamicParamIndex() < 0) { - // Small BIGINT values otherwise parse back as INT during a plan round trip. + // Preserve integer width in JSON; the parser restores this as a BIGINT literal. sb.append("cast(").append(lExpr.getLong()).append(" as BIGINT)"); } else { sb.append(lExpr.getLong()); From 3abcce86b46ee77e42932f86e8497caa5730e4cc Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Tue, 6 Oct 2026 15:03:45 +0800 Subject: [PATCH 06/16] Keep planning failures outside plan cache fallback --- .../exec/planner/sql/DrillSqlWorker.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index 91e872185d1..e4968445b69 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.planner.sql; import java.io.IOException; +import java.util.function.Consumer; import org.apache.calcite.sql.SqlDescribeSchema; import org.apache.calcite.sql.SqlKind; @@ -296,6 +297,9 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point context.getOptions().setLocalOption(ExecConstants.RETURN_RESULT_SET_FOR_DDL, true); } + // Keep the original SQL if no safe snapshot is available or cache preparation fails. + SqlNode planningSql = sqlNode; + Consumer prepareCacheInsert = null; if (context.getOptions().getOption(PlannerSettings.PLAN_CACHE) && !context.getSession().hasTemporaryTables() && !hasMutableAliases(context) @@ -327,20 +331,29 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point } cache.invalidate(key); } - PhysicalPlan planned = handler.getPlan(candidate.sql); - final String templateTextPlan = PlanCache.bindTextPlan(textPlan, candidate.literals); final org.apache.drill.exec.planner.PhysicalPlanReader cacheReader = context.getDrillbitContext().getPlanReader(); - context.setPendingPlanCacheInsert(() -> cache.writeAfterSuccess(key, planned, - templateTextPlan, cacheReader, snapshot)); - return planned; + // Prepare insertion after planning; Foreman publishes it only after the query succeeds. + prepareCacheInsert = planned -> { + String templateTextPlan = PlanCache.bindTextPlan(textPlan, candidate.literals); + context.setPendingPlanCacheInsert(() -> cache.writeAfterSuccess(key, planned, + templateTextPlan, cacheReader, snapshot)); + }; + // On a cache miss, plan the parameterized SQL to produce a reusable template. + planningSql = candidate.sql; } - } catch (Exception e) { + } catch (RuntimeException e) { logger.debug("Plan cache attempt fell back to ordinary planning", e); } } - return handler.getPlan(sqlNode); + // Plan exactly once, outside cache fallback. Validation can mutate the SQL tree, + // so retrying a failed planning attempt could reuse partially rewritten nodes. + PhysicalPlan planned = handler.getPlan(planningSql); + if (prepareCacheInsert != null) { + prepareCacheInsert.accept(planned); + } + return planned; } private static boolean hasMutableAliases(QueryContext context) { From d7c9ba62e601891023bc65477abc354ca9e4a336 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Tue, 6 Oct 2026 15:33:24 +0800 Subject: [PATCH 07/16] Simplify plan cache plugin APIs and expression serialization --- PLAN_CACHE_PLUGIN_GUIDE.md | 22 ++--- .../iceberg/format/IcebergFormatPlugin.java | 24 ++--- .../exec/store/hbase/HBaseStoragePlugin.java | 17 ++-- .../drill/exec/physical/base/GroupScan.java | 6 -- .../exec/planner/physical/DirectScanPrel.java | 2 +- .../drill/exec/planner/physical/ScanPrel.java | 2 +- .../drill/exec/planner/sql/PlanCache.java | 8 +- .../drill/exec/store/StoragePlugin.java | 15 +--- .../exec/store/dfs/FileSystemPlugin.java | 39 ++------ .../drill/exec/store/dfs/FormatPlugin.java | 20 +++-- .../drill/exec/store/plan/rel/PluginPrel.java | 2 +- .../common/expression/parser/ExprParser.g4 | 2 +- .../expression/ExpressionStringBuilder.java | 88 ++++++------------- .../common/expression/LiteralExpression.java | 6 ++ 14 files changed, 86 insertions(+), 167 deletions(-) diff --git a/PLAN_CACHE_PLUGIN_GUIDE.md b/PLAN_CACHE_PLUGIN_GUIDE.md index 72e718924f6..2cabace36a2 100644 --- a/PLAN_CACHE_PLUGIN_GUIDE.md +++ b/PLAN_CACHE_PLUGIN_GUIDE.md @@ -25,24 +25,27 @@ Plan caching is disabled by default. A plugin opts in through its metadata inter Implement these methods in [StoragePlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java): ```java -boolean supportPlanCache(); +boolean supportPlanCache(DrillTableSelection selection); PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException; ``` `planCacheTable` is called before each lookup to capture the current query's table dependencies. Return a stable, opaque physical table identifier and a compatibility version. Return `null` for missing tables or unsupported selections. Failed metadata reads also cause a fallback to normal planning. The engine compares this complete context snapshot, including effective options and storage-plugin configurations, with the cached entry's snapshot. A miss retains the same snapshot for publication after successful execution. -The engine first checks every participating plugin's eligibility and returns immediately if any plugin does not support caching. Override `supportPlanCache(DrillTableSelection)` when support depends on the selection; its default delegates to `supportPlanCache()`. Keep this check free of table-version reads. DFS uses it to check the selected format plugin. Only after all checks pass does the engine call `planCacheTable` and compute configuration and option fingerprints. +The engine first calls `supportPlanCache(selection)` for every participating plugin and returns immediately if any selection is unsupported. Its default returns `false`. A plugin can cast `DrillTableSelection` to its own selection type to decide eligibility. Keep this check free of table-version reads. Only after all checks pass does the engine call `planCacheTable` and compute configuration and option fingerprints. The version must change whenever the cached plan's table definition becomes incompatible. Do not use a data snapshot number if compatible data updates should keep hitting. A path alone is insufficient when replacement can change the schema or table identity. -For a DFS format, implement `supportPlanCache()` and both -`planCacheTableVersion(FileSelection)` and `planCacheTableVersion(Path)` in -[FormatPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java). -[FileSystemPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java) -provides the storage-plugin bridge. The selection overload validates the current selection and supplies its compatibility version; it can delegate metadata reads to the path overload. +DFS formats use the same two-method contract in [FormatPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java), with `FileSelection` as the input: + +```java +boolean supportPlanCache(FileSelection selection); +PlanCacheTable planCacheTable(FileSelection selection) throws IOException; +``` + +[FileSystemPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java) delegates eligibility and metadata reads to the selected format plugin. It prefixes the returned table identifier with the format name to distinguish formats within the storage plugin. ## Scan API and JSON reconstruction @@ -63,10 +66,7 @@ endpoint assignments. Validate the round trip with Drill's `PhysicalPlanReader`. If a parameter changes projection, schema, limit or another derived field that cannot be rebuilt, keep that argument structural or leave the selection ineligible. -For explain output containing expressions, use -`ExpressionStringBuilder.toExplainString()` via `getExplainDigest()` to display -`?index`. Continue using the regular expression serialization for JSON so slot -numbers survive the round trip. +Keep the regular expression serialization for JSON so parameter slot numbers survive the round trip. ## Reference implementations diff --git a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java index e7c872452bd..749ba813e20 100644 --- a/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java +++ b/contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java @@ -29,6 +29,7 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.metadata.schema.SchemaProvider; import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.PlanCacheTable; import org.apache.drill.exec.store.PluginRulesProviderImpl; import org.apache.drill.exec.store.StoragePluginRulesSupplier; import org.apache.drill.exec.store.dfs.FileSelection; @@ -109,31 +110,24 @@ public boolean supportsRead() { } @Override - public boolean supportPlanCache() { - return config.getSnapshot() == null; + public boolean supportPlanCache(FileSelection selection) { + return selection != null && config.getSnapshot() == null + && !(selection instanceof IcebergMetadataFileSelection) && selection.getSelectionRoot() != null; } @Override - public String planCacheTableVersion(FileSelection selection) throws IOException { - if (!supportPlanCache() || selection instanceof IcebergMetadataFileSelection - || selection.getSelectionRoot() == null) { + public PlanCacheTable planCacheTable(FileSelection selection) throws IOException { + if (!supportPlanCache(selection)) { return null; } - return planCacheTableVersion(selection.getSelectionRoot()); - } - - @Override - public String planCacheTableVersion(Path tablePath) throws IOException { - if (!supportPlanCache()) { - return null; - } - Table table = new HadoopTables(fsConf).load(tablePath.toString()); + String identifier = selection.getSelectionRoot().toString(); + Table table = new HadoopTables(fsConf).load(identifier); if (!(table instanceof HasTableOperations)) { return null; } TableMetadata metadata = ((HasTableOperations) table).operations().current(); return metadata == null || metadata.uuid() == null ? null - : metadata.uuid() + ":" + metadata.currentSchemaId(); + : new PlanCacheTable(identifier, metadata.uuid() + ":" + metadata.currentSchemaId()); } @Override diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java index d6e2eed1b0a..16dc1e7498a 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java @@ -64,28 +64,23 @@ public boolean supportsRead() { } @Override - public boolean supportPlanCache() { - return true; + public boolean supportPlanCache(DrillTableSelection selection) { + return selection instanceof HBaseScanSpec; } @Override public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOException { - if (!(selection instanceof HBaseScanSpec)) { + if (!supportPlanCache(selection)) { return null; } - String tableName = ((HBaseScanSpec) selection).getTableName(); - String version = planCacheTableVersion(tableName); - return version == null ? null : new PlanCacheTable(tableName, version); - } - - @Override - public String planCacheTableVersion(String identifier) throws IOException { + String identifier = ((HBaseScanSpec) selection).getTableName(); try (Admin admin = getConnection().getAdmin()) { TableName tableName = TableName.valueOf(identifier); if (!admin.tableExists(tableName)) { return null; } - return Hashing.sha256().hashBytes(admin.getTableDescriptor(tableName).toByteArray()).toString(); + String version = Hashing.sha256().hashBytes(admin.getTableDescriptor(tableName).toByteArray()).toString(); + return new PlanCacheTable(identifier, version); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index cb75b2f03f4..bfaa4cee19e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -87,12 +87,6 @@ public interface GroupScan extends Scan, HasAffinity { @JsonIgnore String getDigest(); - /** Text shown for this scan in an explain plan; may hide bound literal values. */ - @JsonIgnore - default String getExplainDigest() { - return getDigest(); - } - @JsonIgnore ScanStats getScanStats(PlannerSettings settings); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java index d7b90992b14..dac84082e8e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DirectScanPrel.java @@ -128,7 +128,7 @@ public RelDataType deriveRowType() { @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("groupscan", this.getGroupScan().getExplainDigest()); + return super.explainTerms(pw).item("groupscan", this.getGroupScan().getDigest()); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java index 84aeb63efa3..1e0bdf428a3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java @@ -87,7 +87,7 @@ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("groupscan", this.getGroupScan().getExplainDigest()); + return super.explainTerms(pw).item("groupscan", this.getGroupScan().getDigest()); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java index 31ca90485d6..fc714b005dd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -414,8 +414,7 @@ private static ResolvedTable resolveTable(SchemaPlus defaultSchema, List return null; } StoragePlugin storagePlugin = drillTable.getPlugin(); - if (!storagePlugin.supportPlanCache() - || !storagePlugin.supportPlanCache((DrillTableSelection) selection)) { + if (!storagePlugin.supportPlanCache((DrillTableSelection) selection)) { return null; } return new ResolvedTable( @@ -597,8 +596,9 @@ private static int findMarker(String input, int offset) { private static void collectTypes(LogicalExpression expression, Map types) { if (expression instanceof LiteralExpression) { - int index = ((LiteralExpression) expression).getDynamicParamIndex(); - if (index >= 0) { + LiteralExpression literal = (LiteralExpression) expression; + if (literal.isDynamicParam()) { + int index = literal.getDynamicParamIndex(); MajorType type = expression.getMajorType(); MajorType previous = types.putIfAbsent(index, type); if (previous != null && !previous.equals(type)) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java index a481fbabd79..b810510c264 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -76,15 +76,11 @@ public interface StoragePlugin extends SchemaFactory, AutoCloseable { * Opts this plugin into plan caching. The engine validates query context and * rebinds expression parameters; the plugin must provide a plan-compatible * table version and safely rebuild every scan for an eligible selection, - * including value-dependent state for each execution. + * including value-dependent state for each execution. Check selection + * eligibility without reading table versions or building cache metadata. */ - default boolean supportPlanCache() { - return false; - } - - /** Checks selection eligibility without reading table versions or building cache metadata. */ default boolean supportPlanCache(DrillTableSelection selection) { - return supportPlanCache(); + return false; } /** @@ -97,11 +93,6 @@ default PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOEx return null; } - /** Reads the current version by a previously resolved identifier, without schema traversal. */ - default String planCacheTableVersion(String identifier) throws IOException { - return null; - } - /** * Indicates if Drill can insert to a table to this plugin. */ diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 40a4bf8ef69..1791c27d554 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -64,7 +64,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.sftp.SFTPFileSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -333,11 +332,6 @@ public FormatPlugin getFormatPlugin(String name) { return formatCreator.getFormatPluginByName(name); } - @Override - public boolean supportPlanCache() { - return true; - } - @Override public boolean supportPlanCache(DrillTableSelection selection) { if (!(selection instanceof FormatSelection)) { @@ -345,7 +339,7 @@ public boolean supportPlanCache(DrillTableSelection selection) { } FormatSelection formatSelection = (FormatSelection) selection; FormatPlugin format = getFormatPlugin(formatSelection.getFormat()); - return format != null && format.supportPlanCache(); + return format != null && format.supportPlanCache(formatSelection.getSelection()); } @Override @@ -356,40 +350,19 @@ public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOExc FormatSelection formatSelection = (FormatSelection) selection; FileSelection files = formatSelection.getSelection(); FormatPlugin format = getFormatPlugin(formatSelection.getFormat()); - if (files == null || files.getSelectionRoot() == null - || format == null || !format.supportPlanCache()) { + if (format == null || !format.supportPlanCache(files)) { return null; } - String version = format.planCacheTableVersion(files); - if (version == null) { + PlanCacheTable table = format.planCacheTable(files); + if (table == null) { return null; } String formatName = format.getName(); if (formatName == null || formatName.isEmpty()) { return null; } - String identifier = formatName.length() + ":" + formatName + files.getSelectionRoot(); - return new PlanCacheTable(identifier, version); - } - - @Override - public String planCacheTableVersion(String identifier) throws IOException { - int separator = identifier.indexOf(':'); - if (separator < 1) { - return null; - } - try { - int nameLength = Integer.parseInt(identifier.substring(0, separator)); - if (nameLength < 1 || nameLength >= identifier.length() - separator - 1) { - return null; - } - int nameEnd = separator + 1 + nameLength; - FormatPlugin format = getFormatPlugin(identifier.substring(separator + 1, nameEnd)); - return format == null || !format.supportPlanCache() ? null - : format.planCacheTableVersion(new Path(identifier.substring(nameEnd))); - } catch (NumberFormatException e) { - return null; - } + String identifier = formatName.length() + ":" + formatName + table.getIdentifier(); + return new PlanCacheTable(identifier, table.getVersion()); } /** diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java index 7ec89974e1f..d72bbd5d0a0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java @@ -34,6 +34,7 @@ import org.apache.drill.exec.planner.common.DrillStatsTable.TableStatistics; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.OptionManager; +import org.apache.drill.exec.store.PlanCacheTable; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -48,18 +49,19 @@ public interface FormatPlugin { boolean supportsWrite(); - /** Whether every scan of an eligible selection can be safely rebuilt for a cached plan. */ - default boolean supportPlanCache() { + /** + * Whether every scan of this selection can be safely rebuilt for a cached plan. + * Check eligibility without reading table versions or building cache metadata. + */ + default boolean supportPlanCache(FileSelection selection) { return false; } - /** Version of the table definition that determines physical-plan compatibility. */ - default String planCacheTableVersion(FileSelection selection) throws IOException { - return null; - } - - /** Reads the current version of a previously resolved table without rebuilding its schema. */ - default String planCacheTableVersion(Path tablePath) throws IOException { + /** + * Returns a stable table identifier and a physical-plan compatibility version, + * or null when the selection cannot be cached. + */ + default PlanCacheTable planCacheTable(FileSelection selection) throws IOException { return null; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java index 70743476e0d..7334ad6cd0d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/plan/rel/PluginPrel.java @@ -75,7 +75,7 @@ public boolean needsFinalColumnReordering() { @Override public RelWriter explainTerms(RelWriter pw) { - return super.explainTerms(pw).item("groupScan", groupScan.getExplainDigest()); + return super.explainTerms(pw).item("groupScan", groupScan.getDigest()); } @Override diff --git a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 index 2abd4f1b636..a9a5c760dd7 100644 --- a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 +++ b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 @@ -55,7 +55,7 @@ import org.apache.drill.common.exceptions.ExpressionParsingException; // Keep parameter casts intact to retain their slot identity and conversion. if (type.equals(Types.required(MinorType.BIGINT)) && input instanceof LiteralExpression - && ((LiteralExpression) input).getDynamicParamIndex() < 0) { + && !((LiteralExpression) input).isDynamicParam()) { if (input instanceof ValueExpressions.IntExpression) { return new ValueExpressions.LongExpression( ((ValueExpressions.IntExpression) input).getInt(), position); diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 9bb6f787907..ca9d57900bf 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -45,29 +45,25 @@ public class ExpressionStringBuilder extends AbstractExprVisitor{ + /** Parser syntax for a literal carrying a parameter slot; not an executable function. */ public static final String BOUND_DYNAMIC_PARAM = "bound_dynamic_param"; static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder(); - private final boolean explainParameters; - public ExpressionStringBuilder() { - this(false); - } - - private ExpressionStringBuilder(boolean explainParameters) { - this.explainParameters = explainParameters; - } - - private boolean startBoundLiteral(LiteralExpression expression, StringBuilder sb) { - int index = expression.getDynamicParamIndex(); - if (index >= 0) { - if (explainParameters) { - sb.append('?').append(index); - return true; - } + /** + * Starts bound_dynamic_param(index, type, value) for a parameterized literal. + * The visitor writes the value and endBoundLiteral closes the wrapper. + * This format preserves the slot, type and current value for plan rebinding + * and deserialization, and is also used when displaying the expression. + * Ordinary literals have no slot and retain their usual representation. + */ + private void startBoundLiteral(LiteralExpression expression, StringBuilder sb) { + if (expression.isDynamicParam()) { + int index = expression.getDynamicParamIndex(); MajorType type = expression.getMajorType(); sb.append(BOUND_DYNAMIC_PARAM).append('(').append(index).append(", ") .append(type.getMinorType().name()); + // Preserve length, precision and scale so rebinding restores the same type. switch (type.getMinorType()) { case VARCHAR: sb.append('(').append(type.getPrecision()).append(')'); @@ -81,11 +77,10 @@ private boolean startBoundLiteral(LiteralExpression expression, StringBuilder sb } sb.append(", "); } - return false; } private static void endBoundLiteral(LiteralExpression expression, StringBuilder sb) { - if (expression.getDynamicParamIndex() >= 0) { + if (expression.isDynamicParam()) { sb.append(')'); } } @@ -96,13 +91,6 @@ public static String toString(LogicalExpression expr) { return sb.toString(); } - /** Renders parameter slots without exposing the literal stored in a cached plan. */ - public static String toExplainString(LogicalExpression expr) { - StringBuilder sb = new StringBuilder(); - expr.accept(new ExpressionStringBuilder(true), sb); - return sb.toString(); - } - public static void toString(LogicalExpression expr, StringBuilder sb) { expr.accept(INSTANCE, sb); } @@ -197,10 +185,8 @@ public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeExc @Override public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(lExpr, sb)) { - return null; - } - if (lExpr.getDynamicParamIndex() < 0) { + startBoundLiteral(lExpr, sb); + if (!lExpr.isDynamicParam()) { // Preserve integer width in JSON; the parser restores this as a BIGINT literal. sb.append("cast(").append(lExpr.getLong()).append(" as BIGINT)"); } else { @@ -252,9 +238,7 @@ public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder @Override public Void visitDecimal9Constant(Decimal9Expression decExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(decExpr, sb)) { - return null; - } + startBoundLiteral(decExpr, sb); BigDecimal value = new BigDecimal(decExpr.getIntFromDecimal()); sb.append((value.setScale(decExpr.getScale())).toString()); endBoundLiteral(decExpr, sb); @@ -263,9 +247,7 @@ public Void visitDecimal9Constant(Decimal9Expression decExpr, StringBuilder sb) @Override public Void visitDecimal18Constant(Decimal18Expression decExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(decExpr, sb)) { - return null; - } + startBoundLiteral(decExpr, sb); BigDecimal value = new BigDecimal(decExpr.getLongFromDecimal()); sb.append((value.setScale(decExpr.getScale())).toString()); endBoundLiteral(decExpr, sb); @@ -274,9 +256,7 @@ public Void visitDecimal18Constant(Decimal18Expression decExpr, StringBuilder sb @Override public Void visitDecimal28Constant(Decimal28Expression decExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(decExpr, sb)) { - return null; - } + startBoundLiteral(decExpr, sb); sb.append(decExpr.toString()); endBoundLiteral(decExpr, sb); return null; @@ -284,9 +264,7 @@ public Void visitDecimal28Constant(Decimal28Expression decExpr, StringBuilder sb @Override public Void visitDecimal38Constant(Decimal38Expression decExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(decExpr, sb)) { - return null; - } + startBoundLiteral(decExpr, sb); sb.append(decExpr.getBigDecimal().toString()); endBoundLiteral(decExpr, sb); return null; @@ -294,9 +272,7 @@ public Void visitDecimal38Constant(Decimal38Expression decExpr, StringBuilder sb @Override public Void visitVarDecimalConstant(VarDecimalExpression decExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(decExpr, sb)) { - return null; - } + startBoundLiteral(decExpr, sb); sb.append(decExpr.getBigDecimal().toString()); endBoundLiteral(decExpr, sb); return null; @@ -304,9 +280,7 @@ public Void visitVarDecimalConstant(VarDecimalExpression decExpr, StringBuilder @Override public Void visitDoubleConstant(DoubleExpression dExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(dExpr, sb)) { - return null; - } + startBoundLiteral(dExpr, sb); sb.append(dExpr.getDouble()); endBoundLiteral(dExpr, sb); return null; @@ -314,9 +288,7 @@ public Void visitDoubleConstant(DoubleExpression dExpr, StringBuilder sb) throws @Override public Void visitBooleanConstant(BooleanExpression e, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(e, sb)) { - return null; - } + startBoundLiteral(e, sb); sb.append(e.getBoolean()); endBoundLiteral(e, sb); return null; @@ -324,9 +296,7 @@ public Void visitBooleanConstant(BooleanExpression e, StringBuilder sb) throws R @Override public Void visitQuotedStringConstant(QuotedString e, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(e, sb)) { - return null; - } + startBoundLiteral(e, sb); sb.append("'"); sb.append(escapeSingleQuote(e.value)); sb.append("'"); @@ -415,9 +385,7 @@ public Void visitCastExpression(CastExpression e, StringBuilder sb) throws Runti @Override public Void visitFloatConstant(FloatExpression fExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(fExpr, sb)) { - return null; - } + startBoundLiteral(fExpr, sb); sb.append(fExpr.getFloat()); endBoundLiteral(fExpr, sb); return null; @@ -425,9 +393,7 @@ public Void visitFloatConstant(FloatExpression fExpr, StringBuilder sb) throws R @Override public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(intExpr, sb)) { - return null; - } + startBoundLiteral(intExpr, sb); sb.append(intExpr.getInt()); endBoundLiteral(intExpr, sb); return null; @@ -435,9 +401,7 @@ public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws Run @Override public Void visitNullConstant(TypedNullConstant e, StringBuilder sb) throws RuntimeException { - if (startBoundLiteral(e, sb)) { - return null; - } + startBoundLiteral(e, sb); sb.append("NULL"); endBoundLiteral(e, sb); return null; diff --git a/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java b/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java index 73258f32588..e591f2867cf 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java +++ b/logical/src/main/java/org/apache/drill/common/expression/LiteralExpression.java @@ -32,6 +32,12 @@ public int getDynamicParamIndex() { return dynamicParamIndex; } + /** Whether this literal is bound to a zero-based SQL parameter slot. */ + @JsonIgnore + public boolean isDynamicParam() { + return dynamicParamIndex >= 0; + } + public void setDynamicParamIndex(int dynamicParamIndex) { if (dynamicParamIndex < 0) { throw new IllegalArgumentException("Dynamic parameter index must be non-negative"); From 8ff6bb64c388cab8b7632842c0e771b49f925ca8 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Tue, 6 Oct 2026 15:38:45 +0800 Subject: [PATCH 08/16] Use imports for plan cache type references --- .../exec/planner/logical/DrillOptiq.java | 12 +++++----- .../exec/planner/sql/DrillSqlWorker.java | 3 ++- .../drill/exec/planner/sql/PlanCache.java | 23 +++++++++++-------- .../planner/sql/PlanCacheParameterizer.java | 7 +++--- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java index 128e08f113a..5e0eea1f21f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java @@ -40,6 +40,7 @@ import org.apache.drill.common.expression.FunctionCallFactory; import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.IfExpression.IfCondition; +import org.apache.drill.common.expression.LiteralExpression; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.NullExpression; import org.apache.drill.common.expression.SchemaPath; @@ -76,6 +77,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam; import org.apache.drill.exec.work.ExecErrorConstants; import static org.apache.drill.exec.planner.physical.PlannerSettings.ENABLE_DECIMAL_DATA_TYPE; @@ -469,15 +471,13 @@ public LogicalExpression visitCorrelVariable(RexCorrelVariable correlVariable) { @Override public LogicalExpression visitDynamicParam(RexDynamicParam dynamicParam) { - if (dynamicParam instanceof org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam) { - org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam bound = - (org.apache.drill.exec.planner.sql.conversion.RexBoundDynamicParam) dynamicParam; + if (dynamicParam instanceof RexBoundDynamicParam) { + RexBoundDynamicParam bound = (RexBoundDynamicParam) dynamicParam; LogicalExpression expression = bound.getLiteral().accept(this); - if (!(expression instanceof org.apache.drill.common.expression.LiteralExpression)) { + if (!(expression instanceof LiteralExpression)) { return doUnknown(dynamicParam); } - ((org.apache.drill.common.expression.LiteralExpression) expression) - .setDynamicParamIndex(bound.getIndex()); + ((LiteralExpression) expression).setDynamicParamIndex(bound.getIndex()); return expression; } return doUnknown(dynamicParam); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index e4968445b69..62ebbc3e4ef 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -37,6 +37,7 @@ import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.ops.QueryContext.SqlStatementType; import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.PhysicalPlanReader; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler; import org.apache.drill.exec.planner.sql.handlers.AnalyzeTableHandler; @@ -331,7 +332,7 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point } cache.invalidate(key); } - final org.apache.drill.exec.planner.PhysicalPlanReader cacheReader = + final PhysicalPlanReader cacheReader = context.getDrillbitContext().getPlanReader(); // Prepare insertion after planning; Foreman publishes it only after the query succeeds. prepareCacheInsert = planned -> { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java index fc714b005dd..772090af3df 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -18,14 +18,17 @@ package org.apache.drill.exec.planner.sql; import java.io.IOException; +import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashSet; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ArrayBlockingQueue; @@ -47,6 +50,7 @@ import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; +import org.apache.calcite.util.NlsString; import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.LiteralExpression; @@ -68,7 +72,6 @@ import org.apache.drill.exec.util.Pointer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Objects; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -219,8 +222,8 @@ public static final class ContextSnapshot { private ContextSnapshot(String optionsFingerprint, Map tableVersions, Map pluginConfigs) { this.optionsFingerprint = Objects.requireNonNull(optionsFingerprint, "optionsFingerprint"); - this.tableVersions = java.util.Collections.unmodifiableMap(tableVersions); - this.pluginConfigs = java.util.Collections.unmodifiableMap(pluginConfigs); + this.tableVersions = Collections.unmodifiableMap(tableVersions); + this.pluginConfigs = Collections.unmodifiableMap(pluginConfigs); } private static final class TableIdentifier { @@ -418,7 +421,7 @@ private static ResolvedTable resolveTable(SchemaPlus defaultSchema, List return null; } return new ResolvedTable( - java.util.Collections.unmodifiableList(new java.util.ArrayList<>(names)), + Collections.unmodifiableList(new ArrayList<>(names)), drillTable.getStorageEngineName(), storagePlugin, (DrillTableSelection) selection); } @@ -478,10 +481,10 @@ private static String configFingerprint(StoragePluginRegistry plugins, private static String serialize(SqlLiteral literal, MajorType type, int index) { Object value = literal.getValue(); LogicalExpression expression; - if (value instanceof java.math.BigDecimal) { - java.math.BigDecimal number = (java.math.BigDecimal) value; + if (value instanceof BigDecimal) { + BigDecimal number = (BigDecimal) value; if (type.getMinorType() == MinorType.VARDECIMAL) { - java.math.BigDecimal decimal = number.setScale(type.getScale(), RoundingMode.UNNECESSARY); + BigDecimal decimal = number.setScale(type.getScale(), RoundingMode.UNNECESSARY); if (decimal.precision() > type.getPrecision()) { throw new IllegalArgumentException("Decimal parameter exceeds cached precision"); } @@ -500,8 +503,8 @@ private static String serialize(SqlLiteral literal, MajorType type, int index) { } } else if (value instanceof Boolean) { expression = ValueExpressions.getBit((Boolean) value); - } else if (value instanceof org.apache.calcite.util.NlsString) { - String string = ((org.apache.calcite.util.NlsString) value).getValue(); + } else if (value instanceof NlsString) { + String string = ((NlsString) value).getValue(); expression = ValueExpressions.getChar(string, type.getPrecision()); } else { throw new IllegalArgumentException("Unsupported parameter type"); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java index 34bd405f4ba..a21aa67fac5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java @@ -34,6 +34,7 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.calcite.util.NlsString; import org.apache.drill.exec.planner.sql.conversion.SqlBoundDynamicParam; /** Extracts SQL literals into typed slots independently of cache eligibility. */ @@ -162,7 +163,7 @@ public SqlNode visit(SqlLiteral literal) { } Object value = literal.getValue(); if (!(value instanceof BigDecimal) && !(value instanceof Boolean) - && !(value instanceof org.apache.calcite.util.NlsString)) { + && !(value instanceof NlsString)) { // Temporal literals, including unresolved DATE literals, and structural // literals such as EXTRACT's time unit stay in the template key. return literal; @@ -174,8 +175,8 @@ public SqlNode visit(SqlLiteral literal) { if (value instanceof BigDecimal) { BigDecimal number = (BigDecimal) value; shape += ":" + number.precision() + ":" + number.scale(); - } else if (value instanceof org.apache.calcite.util.NlsString) { - org.apache.calcite.util.NlsString string = (org.apache.calcite.util.NlsString) value; + } else if (value instanceof NlsString) { + NlsString string = (NlsString) value; shape += ":" + string.getValue().length() + ":" + string.getCharsetName() + ":" + string.getCollation(); } From 53306c16b4912ce9284d5aec4ed05b5dab5b439e Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Tue, 6 Oct 2026 16:08:36 +0800 Subject: [PATCH 09/16] Preserve literal types and bind cached expression nodes safely --- .../exec/planner/sql/DrillSqlWorker.java | 3 +- .../drill/exec/planner/sql/PlanCache.java | 135 +++++++++--------- .../planner/sql/PlanCacheParameterizer.java | 15 +- .../expression/ExpressionStringBuilder.java | 10 -- .../common/expression/TypedNullConstant.java | 2 +- 5 files changed, 83 insertions(+), 82 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index 62ebbc3e4ef..a496080c5b4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -311,7 +311,8 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve( parser.getDefaultSchema(), sqlNode, context); if (snapshot != null) { - PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(sqlNode); + PlanCacheParameterizer.Candidate candidate = + PlanCacheParameterizer.parameterize(sqlNode, parser.getTypeFactory()); PlanCache cache = context.getDrillbitContext().getPlanCache(); String key = context.getQueryUserName() + '\n' + context.getSession().getDefaultSchemaPath() + '\n' diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java index 772090af3df..73ac000eea6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -51,9 +51,18 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.util.NlsString; +import org.apache.drill.common.expression.AnyValueExpression; +import org.apache.drill.common.expression.BooleanOperator; +import org.apache.drill.common.expression.CastExpression; +import org.apache.drill.common.expression.ConvertExpression; import org.apache.drill.common.expression.ExpressionStringBuilder; -import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.LiteralExpression; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.NullExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.common.parser.LogicalExpressionParser; @@ -90,7 +99,6 @@ public final class PlanCache implements AutoCloseable { .maximumWeight(32 * 1024 * 1024) .weigher((String key, Entry value) -> value.json.length()) .expireAfterWrite(10, TimeUnit.MINUTES) - .recordStats() .build(); private final AtomicLong successfulBinds = new AtomicLong(); private final ThreadPoolExecutor writer = new ThreadPoolExecutor(1, 1, 0L, @@ -478,7 +486,7 @@ private static String configFingerprint(StoragePluginRegistry plugins, } } - private static String serialize(SqlLiteral literal, MajorType type, int index) { + private static LiteralExpression bindLiteral(SqlLiteral literal, MajorType type, int index) { Object value = literal.getValue(); LogicalExpression expression; if (value instanceof BigDecimal) { @@ -509,8 +517,9 @@ private static String serialize(SqlLiteral literal, MajorType type, int index) { } else { throw new IllegalArgumentException("Unsupported parameter type"); } - ((LiteralExpression) expression).setDynamicParamIndex(index); - return ExpressionStringBuilder.toString(expression); + LiteralExpression bound = (LiteralExpression) expression; + bound.setDynamicParamIndex(index); + return bound; } /** Visits every serialized field, including nested plugin-specific fields. */ @@ -539,36 +548,15 @@ private static void walk(JsonNode node, List replacements) { } private static String rewrite(String input, List replacements) { - // Parsing first ensures that a marker belongs to a Drill expression field. + // Bind expression nodes rather than text: a quoted field name or string + // containing the marker must never be interpreted as a parameter slot. LogicalExpression original = LogicalExpressionParser.parse(input); Map originalTypes = new HashMap<>(); collectTypes(original, originalTypes); - StringBuilder output = new StringBuilder(); - int offset = 0; - while (offset < input.length()) { - int start = findMarker(input, offset); - if (start < 0) { - output.append(input, offset, input.length()); - break; - } - output.append(input, offset, start); - int indexEnd = input.indexOf(',', start + MARKER.length()); - if (indexEnd < 0) { - throw new IllegalArgumentException("Malformed cached parameter"); - } - int index = Integer.parseInt(input.substring(start + MARKER.length(), indexEnd).trim()); - if (!originalTypes.containsKey(index)) { - throw new IllegalArgumentException("Cached marker is not an expression parameter"); - } - int end = matchingClose(input, indexEnd + 1); - if (index < 0 || index >= replacements.size()) { - throw new IllegalArgumentException("Unexpected cached parameter index"); - } - // Keep the slot marker in the execution plan for round-trip checks. - output.append(serialize(replacements.get(index), originalTypes.get(index), index)); - offset = end + 1; + if (originalTypes.isEmpty()) { + return input; } - String result = output.toString(); + String result = ExpressionStringBuilder.toString(bindExpression(original, replacements)); LogicalExpression parsed = LogicalExpressionParser.parse(result); Map newTypes = new HashMap<>(); collectTypes(parsed, newTypes); @@ -578,22 +566,58 @@ private static String rewrite(String input, List replacements) { return result; } - private static int findMarker(String input, int offset) { - boolean quoted = false; - boolean escaped = false; - for (int i = 0; i < input.length(); i++) { - char c = input.charAt(i); - if (escaped) { - escaped = false; - } else if (c == '\\' && quoted) { - escaped = true; - } else if (c == '\'') { - quoted = !quoted; - } else if (!quoted && i >= offset && input.startsWith(MARKER, i)) { - return i; + private static LogicalExpression bindExpression(LogicalExpression expression, + List replacements) { + if (expression instanceof LiteralExpression) { + LiteralExpression literal = (LiteralExpression) expression; + if (!literal.isDynamicParam()) { + return literal; + } + int index = literal.getDynamicParamIndex(); + if (index >= replacements.size()) { + throw new IllegalArgumentException("Unexpected cached parameter index"); } + return bindLiteral(replacements.get(index), literal.getMajorType(), index); } - return -1; + if (expression instanceof FunctionCall) { + FunctionCall call = (FunctionCall) expression; + List args = new ArrayList<>(call.args().size()); + for (LogicalExpression arg : call.args()) { + args.add(bindExpression(arg, replacements)); + } + return call instanceof BooleanOperator + ? new BooleanOperator(call.getName(), args, call.getPosition()) + : new FunctionCall(call.getName(), args, call.getPosition()); + } + if (expression instanceof CastExpression) { + CastExpression cast = (CastExpression) expression; + return new CastExpression(bindExpression(cast.getInput(), replacements), + cast.getMajorType(), cast.getPosition()); + } + if (expression instanceof ConvertExpression) { + ConvertExpression convert = (ConvertExpression) expression; + return new ConvertExpression(convert.getConvertFunction(), convert.getEncodingType(), + bindExpression(convert.getInput(), replacements), convert.getPosition()); + } + if (expression instanceof IfExpression) { + IfExpression conditional = (IfExpression) expression; + return IfExpression.newBuilder() + .setPosition(conditional.getPosition()) + .setIfCondition(new IfExpression.IfCondition( + bindExpression(conditional.ifCondition.condition, replacements), + bindExpression(conditional.ifCondition.expression, replacements))) + .setElse(bindExpression(conditional.elseExpression, replacements)) + .build(); + } + if (expression instanceof AnyValueExpression) { + AnyValueExpression any = (AnyValueExpression) expression; + return new AnyValueExpression(bindExpression(any.getInput(), replacements), any.getPosition()); + } + if (expression instanceof SchemaPath || expression instanceof NullExpression + || expression instanceof TypedNullConstant) { + return expression; + } + throw new IllegalArgumentException("Unsupported cached expression: " + expression.getClass().getName()); } private static void collectTypes(LogicalExpression expression, @@ -613,25 +637,4 @@ private static void collectTypes(LogicalExpression expression, collectTypes(child, types); } } - - private static int matchingClose(String input, int start) { - boolean quoted = false; - boolean escaped = false; - int depth = 1; - for (int i = start; i < input.length(); i++) { - char c = input.charAt(i); - if (escaped) { - escaped = false; - } else if (c == '\\' && quoted) { - escaped = true; - } else if (c == '\'') { - quoted = !quoted; - } else if (!quoted && c == '(') { - depth++; - } else if (!quoted && c == ')' && --depth == 0) { - return i; - } - } - throw new IllegalArgumentException("Unclosed cached parameter"); - } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java index a21aa67fac5..fb28d258f27 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java @@ -24,6 +24,7 @@ import java.util.Locale; import java.util.Objects; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlJoin; import org.apache.calcite.sql.SqlKind; @@ -41,11 +42,16 @@ final class PlanCacheParameterizer extends SqlShuttle { private final List literals = new ArrayList<>(); private final List types = new ArrayList<>(); + private final RelDataTypeFactory typeFactory; private boolean preserveProjection; - static Candidate parameterize(SqlNode parsed) { + private PlanCacheParameterizer(RelDataTypeFactory typeFactory) { + this.typeFactory = Objects.requireNonNull(typeFactory, "typeFactory"); + } + + static Candidate parameterize(SqlNode parsed, RelDataTypeFactory typeFactory) { Objects.requireNonNull(parsed, "parsed"); - PlanCacheParameterizer visitor = new PlanCacheParameterizer(); + PlanCacheParameterizer visitor = new PlanCacheParameterizer(typeFactory); SqlNode parameterized = parsed.accept(visitor); if (visitor.literals.isEmpty()) { parameterized = parsed; @@ -170,8 +176,9 @@ public SqlNode visit(SqlLiteral literal) { } int index = literals.size(); literals.add(literal); - // Precision and scale affect inferred types and must be part of the key. - String shape = literal.getTypeName().name(); + // SqlNumericLiteral reports DECIMAL even for INT and BIGINT values. Resolve + // its actual type so equal precision and scale cannot hide a width change. + String shape = literal.createSqlType(typeFactory).getSqlTypeName().name(); if (value instanceof BigDecimal) { BigDecimal number = (BigDecimal) value; shape += ":" + number.precision() + ":" + number.scale(); diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index ca9d57900bf..94596cfde6d 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -238,35 +238,27 @@ public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder @Override public Void visitDecimal9Constant(Decimal9Expression decExpr, StringBuilder sb) throws RuntimeException { - startBoundLiteral(decExpr, sb); BigDecimal value = new BigDecimal(decExpr.getIntFromDecimal()); sb.append((value.setScale(decExpr.getScale())).toString()); - endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDecimal18Constant(Decimal18Expression decExpr, StringBuilder sb) throws RuntimeException { - startBoundLiteral(decExpr, sb); BigDecimal value = new BigDecimal(decExpr.getLongFromDecimal()); sb.append((value.setScale(decExpr.getScale())).toString()); - endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDecimal28Constant(Decimal28Expression decExpr, StringBuilder sb) throws RuntimeException { - startBoundLiteral(decExpr, sb); sb.append(decExpr.toString()); - endBoundLiteral(decExpr, sb); return null; } @Override public Void visitDecimal38Constant(Decimal38Expression decExpr, StringBuilder sb) throws RuntimeException { - startBoundLiteral(decExpr, sb); sb.append(decExpr.getBigDecimal().toString()); - endBoundLiteral(decExpr, sb); return null; } @@ -401,9 +393,7 @@ public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws Run @Override public Void visitNullConstant(TypedNullConstant e, StringBuilder sb) throws RuntimeException { - startBoundLiteral(e, sb); sb.append("NULL"); - endBoundLiteral(e, sb); return null; } diff --git a/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java b/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java index 415a6ca2331..3e773ffbe65 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java +++ b/logical/src/main/java/org/apache/drill/common/expression/TypedNullConstant.java @@ -25,7 +25,7 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; -public class TypedNullConstant extends LiteralExpression { +public class TypedNullConstant extends LogicalExpressionBase { private final MajorType type; From 39d86b1b127b562bcf6ef3e93bee2dc9d9a5b4e8 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Thu, 8 Oct 2026 22:38:17 +0800 Subject: [PATCH 10/16] Make physical plan cache size and expiration configurable --- PLAN_CACHE_DESIGN.md | 39 +++- .../org/apache/drill/exec/ExecConstants.java | 4 + .../drill/exec/planner/sql/PlanCache.java | 42 +++- .../drill/exec/server/DrillbitContext.java | 4 +- .../src/main/resources/drill-module.conf | 10 + .../exec/planner/sql/TestPlanCacheConfig.java | 196 ++++++++++++++++++ 6 files changed, 286 insertions(+), 9 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java diff --git a/PLAN_CACHE_DESIGN.md b/PLAN_CACHE_DESIGN.md index 77fe952e3af..007a52a7829 100644 --- a/PLAN_CACHE_DESIGN.md +++ b/PLAN_CACHE_DESIGN.md @@ -90,9 +90,42 @@ snapshot selections use normal planning. An entry is published only after the first query succeeds. A bounded background queue performs serialization and read-back validation; a full queue simply skips -publication. The cache is bounded by serialized-plan weight and entries expire -after ten minutes. Cached row-count estimates come from the first optimization; -new values do not trigger a new cost-based plan choice. +publication. Cached row-count estimates come from the first optimization; +new values and statistics changes alone do not trigger a new cost-based plan choice. + +The following boot options can be overridden in `drill-override.conf`. They are +read once per Drillbit and changing them requires a restart. The effective settings +are logged at startup. `planner.enable_plan_cache` remains the runtime on/off switch. + +```hocon +drill.exec.plan_cache: { + max_size_bytes: 33554432, + expire_after_write: 0, + expire_after_access: 10m +} +``` + +- `max_size_bytes` bounds the total UTF-8 byte size of cached physical-plan JSON + per Drillbit. The default is 32 MiB; keys, entry metadata and Java object overhead + are not included in this limit. A value of `0` disables cache storage. +- `expire_after_write` sets a fixed lifetime since creation or replacement. + The default of `0` disables this policy; reads do not extend a configured lifetime. +- `expire_after_access` sets an idle lifetime since the last read or write. + The default of `10m` removes unused entries while allowing active plans to remain + cached, subject to capacity eviction and compatibility checks. + +Durations accept HOCON units such as `30s`, `10m`, or `1h`. A duration of `0` +disables only that expiration policy. When both policies are enabled, an entry +expires as soon as either deadline is reached. When both are disabled, capacity +eviction and explicit invalidation still apply. All three settings must be +non-negative; invalid settings fail Drillbit startup. + +Capacity eviction favors recently accessed entries using Guava's segmented LRU +policy, so entries can be evicted before their expiration deadlines or before the +global size limit is reached. Expired entries are no longer returned by lookups; +physical removal occurs during cache maintenance rather than on a background timer. +Without write-based expiration, hot plans do not periodically re-optimize. Operators +who need that behavior can configure a nonzero `expire_after_write` lifetime. The query profile's **Plan Cache Hit** field, or JSON `planCacheHit`, identifies a successful hit. Explain shows slot names and this execution's parameter values; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index b511daa9c2f..535981b5df0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -48,6 +48,10 @@ private ExecConstants() { // Don't allow instantiation } + public static final String PLAN_CACHE_MAX_SIZE_BYTES = "drill.exec.plan_cache.max_size_bytes"; + public static final String PLAN_CACHE_EXPIRE_AFTER_WRITE = "drill.exec.plan_cache.expire_after_write"; + public static final String PLAN_CACHE_EXPIRE_AFTER_ACCESS = "drill.exec.plan_cache.expire_after_access"; + public static final String ZK_RETRY_TIMES = "drill.exec.zk.retry.count"; public static final String ZK_RETRY_DELAY = "drill.exec.zk.retry.delay"; public static final String ZK_CONNECTION = "drill.exec.zk.connect"; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java index 73ac000eea6..7af2ba81d8f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -17,10 +17,13 @@ */ package org.apache.drill.exec.planner.sql; +import static com.google.common.base.Preconditions.checkArgument; + import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -68,6 +71,7 @@ import org.apache.drill.common.parser.LogicalExpressionParser; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.planner.PhysicalPlanReader; @@ -86,6 +90,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ticker; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.hash.Hashing; @@ -95,16 +101,40 @@ public final class PlanCache implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(PlanCache.class); private static final ObjectMapper JSON = new ObjectMapper(); private static final String MARKER = ExpressionStringBuilder.BOUND_DYNAMIC_PARAM + "("; - private final Cache entries = CacheBuilder.newBuilder() - .maximumWeight(32 * 1024 * 1024) - .weigher((String key, Entry value) -> value.json.length()) - .expireAfterWrite(10, TimeUnit.MINUTES) - .build(); + private final Cache entries; private final AtomicLong successfulBinds = new AtomicLong(); private final ThreadPoolExecutor writer = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(64), new NamedThreadFactory("plan-cache-writer-")); + /** A zero expiration duration disables that policy; a zero capacity disables storage. */ + public PlanCache(long maxSizeBytes, Duration expireAfterWrite, Duration expireAfterAccess) { + this(maxSizeBytes, expireAfterWrite, expireAfterAccess, Ticker.systemTicker()); + } + + @VisibleForTesting + PlanCache(long maxSizeBytes, Duration expireAfterWrite, Duration expireAfterAccess, Ticker ticker) { + checkArgument(maxSizeBytes >= 0, "%s must be non-negative", ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES); + checkArgument(!expireAfterWrite.isNegative(), "%s must be non-negative", + ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE); + checkArgument(!expireAfterAccess.isNegative(), "%s must be non-negative", + ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS); + CacheBuilder builder = CacheBuilder.newBuilder() + .maximumWeight(maxSizeBytes) + .weigher((String key, Entry value) -> value.jsonSizeBytes) + .ticker(ticker); + // Passing zero to Guava would expire entries immediately, rather than disable the policy. + if (!expireAfterWrite.isZero()) { + builder.expireAfterWrite(expireAfterWrite); + } + if (!expireAfterAccess.isZero()) { + builder.expireAfterAccess(expireAfterAccess); + } + entries = builder.build(); + logger.info("Plan cache settings: max_size_bytes={}, expire_after_write={}, expire_after_access={}" + + " (zero expiration disables that policy)", maxSizeBytes, expireAfterWrite, expireAfterAccess); + } + public long getHitCount() { return successfulBinds.get(); } @@ -196,11 +226,13 @@ public void close() { public static final class Entry { private final String json; + private final int jsonSizeBytes; private final String textPlan; private final ContextSnapshot context; private Entry(String json, String textPlan, ContextSnapshot context) { this.json = json; + this.jsonSizeBytes = json.getBytes(StandardCharsets.UTF_8).length; this.textPlan = textPlan; this.context = context; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java index 1dee205ec34..44c98c30a83 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java @@ -124,7 +124,9 @@ public DrillbitContext( ExecConstants.STORAGE_PLUGIN_REGISTRY_IMPL, StoragePluginRegistry.class, this); reader = new PhysicalPlanReader(config, classpathScan, lpPersistence, endpoint, storagePlugins); - planCache = new PlanCache(); + planCache = new PlanCache(config.getLong(ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES), + config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE), + config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS)); operatorCreatorRegistry = new OperatorCreatorRegistry(classpathScan); systemOptions = new SystemOptionManager(lpPersistence, provider, config, context.getDefinitions()); functionRegistry = new FunctionImplementationRegistry(config, classpathScan, systemOptions); diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 6176a873053..0a8c05afe96 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -281,6 +281,16 @@ drill.exec: { max_size: 500000000 } }, + plan_cache: { + # Maximum total UTF-8 byte size of cached plan JSON per Drillbit; 0 disables storage. + # This does not include keys, entry metadata, or Java object overhead. + max_size_bytes: 33554432, + # Fixed lifetime since creation or replacement; 0 disables this expiration policy. + expire_after_write: 0, + # Idle lifetime since the last read or write; 0 disables this expiration policy. + # When both policies are enabled, either deadline expires the entry. + expire_after_access: 10m + }, compile: { codegen.debug.topn: false, compiler: "DEFAULT", diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java new file mode 100644 index 00000000000..d04b8588d5e --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.time.Duration; + +import com.google.common.base.Ticker; +import org.apache.drill.categories.PlannerTest; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.test.BaseTest; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Category(PlannerTest.class) +public class TestPlanCacheConfig extends BaseTest { + private final ManualTicker ticker = new ManualTicker(); + private PhysicalPlan plan; + private PhysicalPlanReader reader; + private PlanCache.ContextSnapshot context; + + @Before + public void setup() throws Exception { + plan = mock(PhysicalPlan.class); + reader = mock(PhysicalPlanReader.class); + context = mock(PlanCache.ContextSnapshot.class); + when(reader.writeJson(plan)).thenReturn("{}"); + } + + @Test + public void testDefaultSettingsKeepHotPlansAndExpireIdlePlans() throws Exception { + DrillConfig config = DrillConfig.create(); + long maxSizeBytes = config.getLong(ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES); + Duration write = config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE); + Duration access = config.getDuration(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS); + assertEquals(32L * 1024 * 1024, maxSizeBytes); + assertEquals(Duration.ZERO, write); + assertEquals(Duration.ofMinutes(10), access); + try (PlanCache cache = new PlanCache(maxSizeBytes, write, access, ticker)) { + put(cache, "hot"); + put(cache, "idle"); + for (int i = 0; i < 3; i++) { + ticker.advance(Duration.ofMinutes(5)); + assertNotNull(cache.get("hot")); + } + assertNull(cache.get("idle")); + ticker.advance(Duration.ofMinutes(10)); + assertNull(cache.get("hot")); + } + } + + @Test + public void testWriteExpirationDoesNotExtendOnAccess() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ofMinutes(10), Duration.ZERO, ticker)) { + put(cache, "plan"); + ticker.advance(Duration.ofMinutes(9)); + assertNotNull(cache.get("plan")); + ticker.advance(Duration.ofMinutes(1)); + assertNull(cache.get("plan")); + } + } + + @Test + public void testReplacementRestartsWriteExpiration() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ofMinutes(10), Duration.ZERO, ticker)) { + put(cache, "plan"); + ticker.advance(Duration.ofMinutes(9)); + put(cache, "plan"); + ticker.advance(Duration.ofMinutes(9)); + assertNotNull(cache.get("plan")); + ticker.advance(Duration.ofMinutes(1)); + assertNull(cache.get("plan")); + } + } + + @Test + public void testCombinedPoliciesExpireAtEitherDeadline() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ofMinutes(10), Duration.ofMinutes(3), ticker)) { + put(cache, "idle"); + ticker.advance(Duration.ofMinutes(3)); + assertNull(cache.get("idle")); + + put(cache, "hot"); + for (int i = 0; i < 4; i++) { + ticker.advance(Duration.ofMinutes(2)); + assertNotNull(cache.get("hot")); + } + ticker.advance(Duration.ofMinutes(2)); + assertNull(cache.get("hot")); + } + } + + @Test + public void testBothExpirationPoliciesCanBeDisabled() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + ticker.advance(Duration.ofDays(365)); + assertNotNull(cache.get("plan")); + } + } + + @Test + public void testCapacityCountsUtf8Bytes() throws Exception { + // Five Java characters, but eight bytes when encoded as UTF-8. + when(reader.writeJson(plan)).thenReturn("\"ééé\""); + try (PlanCache cache = new PlanCache(7, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + assertNull(cache.get("plan")); + } + try (PlanCache cache = new PlanCache(8, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + assertNotNull(cache.get("plan")); + } + } + + @Test + public void testCapacityEvictsBeforeIdleExpiration() throws Exception { + // Small limits use a single Guava segment, making eviction order deterministic. + try (PlanCache cache = new PlanCache(10, Duration.ZERO, Duration.ofMinutes(10), ticker)) { + when(reader.writeJson(plan)).thenReturn("\"aa\""); + put(cache, "first"); + when(reader.writeJson(plan)).thenReturn("\"bbb\""); + put(cache, "second"); + assertNotNull(cache.get("first")); + when(reader.writeJson(plan)).thenReturn("\"cc\""); + put(cache, "third"); + assertNull(cache.get("second")); + assertNotNull(cache.get("first")); + assertNotNull(cache.get("third")); + } + } + + @Test + public void testZeroCapacityDisablesStorage() throws Exception { + try (PlanCache cache = new PlanCache(0, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "plan"); + assertNull(cache.get("plan")); + } + } + + @Test + public void testNegativeSettingsAreRejected() { + IllegalArgumentException size = assertThrows(IllegalArgumentException.class, + () -> new PlanCache(-1, Duration.ZERO, Duration.ZERO, ticker)); + assertTrue(size.getMessage().contains(ExecConstants.PLAN_CACHE_MAX_SIZE_BYTES)); + IllegalArgumentException write = assertThrows(IllegalArgumentException.class, + () -> new PlanCache(1024, Duration.ofSeconds(-1), Duration.ZERO, ticker)); + assertTrue(write.getMessage().contains(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_WRITE)); + IllegalArgumentException access = assertThrows(IllegalArgumentException.class, + () -> new PlanCache(1024, Duration.ZERO, Duration.ofSeconds(-1), ticker)); + assertTrue(access.getMessage().contains(ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS)); + } + + private void put(PlanCache cache, String key) throws Exception { + assertTrue(cache.put(key, plan, null, reader, context)); + } + + private static final class ManualTicker extends Ticker { + private long nanos; + + @Override + public long read() { + return nanos; + } + + void advance(Duration duration) { + nanos += duration.toNanos(); + } + } +} From e16de1eefa042f19837954d9130fc0fdf5290e64 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Thu, 8 Oct 2026 22:44:00 +0800 Subject: [PATCH 11/16] Include cache keys and text plans in plan cache weight --- PLAN_CACHE_DESIGN.md | 6 +- .../drill/exec/planner/sql/PlanCache.java | 9 ++- .../src/main/resources/drill-module.conf | 4 +- .../exec/planner/sql/TestPlanCacheConfig.java | 61 +++++++++++++++---- 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/PLAN_CACHE_DESIGN.md b/PLAN_CACHE_DESIGN.md index 007a52a7829..028c3c627b4 100644 --- a/PLAN_CACHE_DESIGN.md +++ b/PLAN_CACHE_DESIGN.md @@ -105,9 +105,9 @@ drill.exec.plan_cache: { } ``` -- `max_size_bytes` bounds the total UTF-8 byte size of cached physical-plan JSON - per Drillbit. The default is 32 MiB; keys, entry metadata and Java object overhead - are not included in this limit. A value of `0` disables cache storage. +- `max_size_bytes` bounds the total UTF-8 byte size of cache keys, physical-plan JSON, + and optional explain text per Drillbit. The default is 32 MiB; entry metadata and + Java object overhead are not included in this limit. A value of `0` disables cache storage. - `expire_after_write` sets a fixed lifetime since creation or replacement. The default of `0` disables this policy; reads do not extend a configured lifetime. - `expire_after_access` sets an idle lifetime since the last read or write. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java index 7af2ba81d8f..f98ecb57e0d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -95,6 +95,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.hash.Hashing; +import com.google.common.primitives.Ints; /** Drillbit-scoped, immutable JSON snapshots of query plans. */ public final class PlanCache implements AutoCloseable { @@ -121,7 +122,8 @@ public PlanCache(long maxSizeBytes, Duration expireAfterWrite, Duration expireAf ExecConstants.PLAN_CACHE_EXPIRE_AFTER_ACCESS); CacheBuilder builder = CacheBuilder.newBuilder() .maximumWeight(maxSizeBytes) - .weigher((String key, Entry value) -> value.jsonSizeBytes) + .weigher((String key, Entry value) -> Ints.saturatedCast( + key.getBytes(StandardCharsets.UTF_8).length + value.contentSizeBytes)) .ticker(ticker); // Passing zero to Guava would expire entries immediately, rather than disable the policy. if (!expireAfterWrite.isZero()) { @@ -226,13 +228,14 @@ public void close() { public static final class Entry { private final String json; - private final int jsonSizeBytes; + private final long contentSizeBytes; private final String textPlan; private final ContextSnapshot context; private Entry(String json, String textPlan, ContextSnapshot context) { this.json = json; - this.jsonSizeBytes = json.getBytes(StandardCharsets.UTF_8).length; + this.contentSizeBytes = (long) json.getBytes(StandardCharsets.UTF_8).length + + (textPlan == null ? 0 : textPlan.getBytes(StandardCharsets.UTF_8).length); this.textPlan = textPlan; this.context = context; } diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 0a8c05afe96..7781aa486e1 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -282,8 +282,8 @@ drill.exec: { } }, plan_cache: { - # Maximum total UTF-8 byte size of cached plan JSON per Drillbit; 0 disables storage. - # This does not include keys, entry metadata, or Java object overhead. + # Maximum total UTF-8 byte size of cache keys, plan JSON, and explain text per Drillbit. + # 0 disables storage. Entry metadata and Java object overhead are not included. max_size_bytes: 33554432, # Fixed lifetime since creation or replacement; 0 disables this expiration policy. expire_after_write: 0, diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java index d04b8588d5e..c7029bcdbb9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java @@ -127,32 +127,71 @@ public void testBothExpirationPoliciesCanBeDisabled() throws Exception { @Test public void testCapacityCountsUtf8Bytes() throws Exception { - // Five Java characters, but eight bytes when encoded as UTF-8. + // Eight JSON bytes plus four key bytes when encoded as UTF-8. when(reader.writeJson(plan)).thenReturn("\"ééé\""); - try (PlanCache cache = new PlanCache(7, Duration.ZERO, Duration.ZERO, ticker)) { + try (PlanCache cache = new PlanCache(11, Duration.ZERO, Duration.ZERO, ticker)) { put(cache, "plan"); assertNull(cache.get("plan")); } - try (PlanCache cache = new PlanCache(8, Duration.ZERO, Duration.ZERO, ticker)) { + try (PlanCache cache = new PlanCache(12, Duration.ZERO, Duration.ZERO, ticker)) { put(cache, "plan"); assertNotNull(cache.get("plan")); } } + @Test + public void testCapacityCountsUtf8KeyBytes() throws Exception { + // Six key bytes plus two JSON bytes. + try (PlanCache cache = new PlanCache(7, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "ééé"); + assertNull(cache.get("ééé")); + } + try (PlanCache cache = new PlanCache(8, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "ééé"); + assertNotNull(cache.get("ééé")); + } + } + + @Test + public void testCapacityCountsUtf8ExplainBytes() throws Exception { + // Two key bytes, two JSON bytes, and three explain text bytes. + try (PlanCache cache = new PlanCache(6, Duration.ZERO, Duration.ZERO, ticker)) { + assertTrue(cache.put("é", plan, "计", reader, context)); + assertNull(cache.get("é")); + } + try (PlanCache cache = new PlanCache(7, Duration.ZERO, Duration.ZERO, ticker)) { + assertTrue(cache.put("é", plan, "计", reader, context)); + assertNotNull(cache.get("é")); + assertEquals("计", cache.get("é").getTextPlan()); + } + } + + @Test + public void testNullAndEmptyExplainTextHaveZeroWeight() throws Exception { + try (PlanCache cache = new PlanCache(4, Duration.ZERO, Duration.ZERO, ticker)) { + put(cache, "é"); + assertNotNull(cache.get("é")); + assertNull(cache.get("é").getTextPlan()); + assertTrue(cache.put("é", plan, "", reader, context)); + assertNotNull(cache.get("é")); + assertEquals("", cache.get("é").getTextPlan()); + } + } + @Test public void testCapacityEvictsBeforeIdleExpiration() throws Exception { // Small limits use a single Guava segment, making eviction order deterministic. - try (PlanCache cache = new PlanCache(10, Duration.ZERO, Duration.ofMinutes(10), ticker)) { + try (PlanCache cache = new PlanCache(11, Duration.ZERO, Duration.ofMinutes(10), ticker)) { when(reader.writeJson(plan)).thenReturn("\"aa\""); - put(cache, "first"); + put(cache, "a"); when(reader.writeJson(plan)).thenReturn("\"bbb\""); - put(cache, "second"); - assertNotNull(cache.get("first")); + put(cache, "b"); + assertNotNull(cache.get("a")); when(reader.writeJson(plan)).thenReturn("\"cc\""); - put(cache, "third"); - assertNull(cache.get("second")); - assertNotNull(cache.get("first")); - assertNotNull(cache.get("third")); + put(cache, "c"); + assertNull(cache.get("b")); + assertNotNull(cache.get("a")); + assertNotNull(cache.get("c")); } } From 9ce73619e41b57d2c5135f18e81788a0e77dca17 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Thu, 8 Oct 2026 22:55:42 +0800 Subject: [PATCH 12/16] Document plan cache storage plugin roadmap --- PLAN_CACHE_DESIGN.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/PLAN_CACHE_DESIGN.md b/PLAN_CACHE_DESIGN.md index 028c3c627b4..2e21b77e396 100644 --- a/PLAN_CACHE_DESIGN.md +++ b/PLAN_CACHE_DESIGN.md @@ -86,6 +86,37 @@ sessions containing temporary tables or mutable aliases bypass the cache. Unsupported scans, views without a compatible table identity, Iceberg metadata tables and explicit snapshot selections use normal planning. +## Storage plugin roadmap + +HBase was chosen first because short queries under high concurrency are expected +to benefit most from plan caching: validation and optimization can account for a +substantial share of their latency. Since Drill is also an analytical engine, +Iceberg was included to evaluate the benefit for analytical workloads over a +data lake format. These initial integrations cover both point/range reads and +analytical scans. + +Support for other commonly used plugins is planned in follow-up changes, in the +following order: + +1. **DFS formats without filter pushdown**, such as CSV/TSV, JSON and Avro. + Filtering remains in Drill's Filter operator. On bind, reconstruct the current + file selection and redo directory partition pruning for the new values, with + a table compatibility version that detects schema changes. +2. **Parquet.** Rebuild partition, file and row-group pruning for the current + values and data files. Reuse the existing Parquet metadata cache where possible + to reduce the cost of reconstruction. +3. **JDBC.** Rebuild pushed-down SQL for the current parameter values rather than + reuse SQL containing the first execution's literals. Evaluate shared plugin + hooks for rebuilding pushdown on bind, so JDBC and other plugins that construct + native queries can avoid duplicating the integration logic. + +Each integration must retain the inputs needed to reconstruct value-dependent +scan state and provide reliable compatibility checks, following the +[plugin developer guide](PLAN_CACHE_PLUGIN_GUIDE.md). Benchmark cache-hit planning +and end-to-end latency against ordinary planning, including metadata reads and +bind-time reconstruction. For DFS and Parquet in particular, measure file listing +and pruning costs to establish how much planning work a hit actually saves. + ## Lifetime and observation An entry is published only after the first query succeeds. A bounded background From 419403d89eb090252a66c43aca4b7cafe554012e Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Thu, 8 Oct 2026 23:26:29 +0800 Subject: [PATCH 13/16] Preserve matching HAVING and QUALIFY expressions in plan cache templates --- .../planner/sql/PlanCacheParameterizer.java | 9 +- .../planner/sql/TestPlanCachePlanning.java | 111 ++++++++++++++++++ 2 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java index fb28d258f27..815b50a572d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCacheParameterizer.java @@ -117,7 +117,8 @@ public SqlNode visit(SqlCall call) { } if (call instanceof SqlSelect) { SqlSelect select = (SqlSelect) call; - // Keep structural SQL operands verbatim while visiting expressions. + // Preserve matching expressions across SELECT, GROUP BY, HAVING and + // QUALIFY. Parameterizing only some occurrences breaks validation or rewrites. boolean structuralProjection = preserveProjection || (select.getGroup() != null && !select.getGroup().isEmpty()) || (select.getOrderList() != null && !select.getOrderList().isEmpty()) @@ -127,8 +128,10 @@ public SqlNode visit(SqlCall call) { structuralProjection ? select.getSelectList() : (SqlNodeList) visitNullable(select.getSelectList()), visitNullable(select.getFrom()), visitNullable(select.getWhere()), - select.getGroup(), visitNullable(select.getHaving()), - select.getWindowList(), visitNullable(select.getQualify()), + select.getGroup(), + structuralProjection ? select.getHaving() : visitNullable(select.getHaving()), + select.getWindowList(), + structuralProjection ? select.getQualify() : visitNullable(select.getQualify()), select.getOrderList(), select.getOffset(), select.getFetch(), select.getHints()); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java new file mode 100644 index 00000000000..99393e2e90c --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import org.apache.drill.categories.SqlTest; +import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category(SqlTest.class) +public class TestPlanCachePlanning extends ClusterTest { + @BeforeClass + public static void setupCluster() throws Exception { + startCluster(ClusterFixture.builder(dirTestWatcher)); + } + + @Before + public void disableCacheForBaseline() { + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + } + + @Test + public void testHavingMatchesGroupedConcatenationOnMissAndHit() throws Exception { + String sql = "SELECT x || 'a' AS k, COUNT(*) AS n " + + "FROM (VALUES ('b', 1), ('b', 2), ('c', 3)) AS t(x, v) " + + "WHERE v > %d GROUP BY x || 'a' HAVING x || 'a' = 'ba'"; + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "n").baselineValues("ba", 2L).go(); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "n").baselineValues("ba", 2L).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + + // WHERE values remain bindable even though the matching HAVING literals are preserved. + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("k", "n").baselineValues("ba", 1L).go(); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testHavingMatchesGroupedArithmeticAndKeepsLiteralsInKey() throws Exception { + String sql = "SELECT x + 1 AS k, COUNT(*) AS n " + + "FROM (VALUES (1), (1), (2)) AS t(x) GROUP BY x + 1 HAVING x + 1 = %d"; + client.testBuilder().sqlQuery(sql, 2).unOrdered() + .baselineColumns("k", "n").baselineValues(2, 2L).go(); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + client.testBuilder().sqlQuery(sql, 2).unOrdered() + .baselineColumns("k", "n").baselineValues(2, 2L).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 2).unOrdered() + .baselineColumns("k", "n").baselineValues(2, 2L).go(); + assertEquals(hits + 1, cache().getHitCount()); + + // A different preserved HAVING literal must not reuse the first template. + client.testBuilder().sqlQuery(sql, 3).unOrdered() + .baselineColumns("k", "n").baselineValues(3, 1L).go(); + cache().awaitWrites(); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testQualifyMatchesOrderedProjectionOnMissAndHit() throws Exception { + String sql = "SELECT x + 1 AS k, ROW_NUMBER() OVER (ORDER BY x + 1) AS rn " + + "FROM (VALUES (1, 1), (1, 2), (2, 3)) AS t(x, v) " + + "WHERE v > %d QUALIFY x + 1 = 2 " + + "AND ROW_NUMBER() OVER (ORDER BY x + 1) = 1 ORDER BY x + 1"; + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "rn").baselineValues(2, 1L).go(); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("k", "rn").baselineValues(2, 1L).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("k", "rn").baselineValues(2, 1L).go(); + assertEquals(hits + 1, cache().getHitCount()); + } + + private PlanCache cache() { + return cluster.drillbit().getContext().getPlanCache(); + } +} From 6aa9618cbec0f01594adc915f14b03298cce7af3 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Fri, 9 Oct 2026 00:12:46 +0800 Subject: [PATCH 14/16] Address plan cache review feedback and add SQL cache management --- docs/dev/DevDocs.md | 5 + .../dev/PLAN_CACHE_DESIGN.md | 64 +++++++- .../dev/PLAN_CACHE_PLUGIN_GUIDE.md | 10 +- .../src/main/codegen/data/Parser.tdd | 5 + .../src/main/codegen/includes/parserImpls.ftl | 18 ++- .../exec/planner/sql/DrillSqlWorker.java | 31 +++- .../drill/exec/planner/sql/PlanCache.java | 85 +++++++++- .../sql/handlers/AbstractSqlHandler.java | 16 ++ .../sql/handlers/AbstractSqlSetHandler.java | 13 +- .../sql/handlers/ClearPlanCacheHandler.java | 45 ++++++ .../planner/sql/parser/SqlClearPlanCache.java | 61 ++++++++ .../exec/planner/sql/TestPlanCacheConfig.java | 75 ++++++++- .../planner/sql/TestPlanCacheContext.java | 120 ++++++++++++++ .../planner/sql/TestPlanCachePlanning.java | 147 ++++++++++++++++++ .../parser/TestClearPlanCacheSqlParser.java | 66 ++++++++ .../exec/server/TestOptionsAuthEnabled.java | 18 +++ .../common/expression/parser/ExprParser.g4 | 20 +-- .../expression/ExpressionStringBuilder.java | 9 +- .../common/expression/parser/TreeTest.java | 27 ++++ 19 files changed, 772 insertions(+), 63 deletions(-) rename PLAN_CACHE_DESIGN.md => docs/dev/PLAN_CACHE_DESIGN.md (69%) rename PLAN_CACHE_PLUGIN_GUIDE.md => docs/dev/PLAN_CACHE_PLUGIN_GUIDE.md (79%) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ClearPlanCacheHandler.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlClearPlanCache.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheContext.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/parser/TestClearPlanCacheSqlParser.java diff --git a/docs/dev/DevDocs.md b/docs/dev/DevDocs.md index eef8105b4d6..03951d2a505 100644 --- a/docs/dev/DevDocs.md +++ b/docs/dev/DevDocs.md @@ -27,3 +27,8 @@ For information about the Jetty 12 upgrade, known limitations, and developer gui ## Materialized Views For information about materialized view support, including SQL syntax, query rewriting, and metastore integration, see [MaterializedViews.md](MaterializedViews.md) + +## Physical Plan Cache + +For cache design, configuration and trade-offs, see [PLAN_CACHE_DESIGN.md](PLAN_CACHE_DESIGN.md). +For storage-plugin integration and scan reconstruction requirements, see [PLAN_CACHE_PLUGIN_GUIDE.md](PLAN_CACHE_PLUGIN_GUIDE.md). diff --git a/PLAN_CACHE_DESIGN.md b/docs/dev/PLAN_CACHE_DESIGN.md similarity index 69% rename from PLAN_CACHE_DESIGN.md rename to docs/dev/PLAN_CACHE_DESIGN.md index 2e21b77e396..14927f345e6 100644 --- a/PLAN_CACHE_DESIGN.md +++ b/docs/dev/PLAN_CACHE_DESIGN.md @@ -60,6 +60,9 @@ The cache belongs to one Drillbit and can be shared across connections. Its key includes the query user, default schema and SQL template. Each entry also records effective planner options, plugin configurations and the compatibility version of every referenced table, including tables inside joins, CTEs and subqueries. +Effective-option and plugin-configuration fingerprints are also part of the key, +so sessions with different settings retain separate entries. Table versions stay +in the entry's context snapshot and trigger invalidation when compatibility changes. A mismatch or failure falls back to ordinary planning. The cached representation is immutable physical-plan JSON. A hit creates a new @@ -81,6 +84,11 @@ TIMESTAMP and INTERVAL literals also stay in the key because their parameter binding is not supported. Other eligible literals in the same query can still be rebound. Queries with no slots can reuse their complete template. +When SELECT expressions are preserved for grouping, ordering or window definitions, +HAVING and QUALIFY expressions are also preserved to keep matching expressions +consistent during validation and rewriting. Their literals remain part of the key; +eligible WHERE literals can still be rebound. + Writes, DDL, existing dynamic parameters, volatile/query-context functions and sessions containing temporary tables or mutable aliases bypass the cache. Unsupported scans, views without a compatible table identity, Iceberg metadata tables and explicit @@ -119,9 +127,18 @@ and pruning costs to establish how much planning work a hit actually saves. ## Lifetime and observation -An entry is published only after the first query succeeds. A bounded background -queue performs serialization and read-back validation; a full queue simply skips -publication. Cached row-count estimates come from the first optimization; +An immutable physical-plan JSON snapshot is serialized immediately after planning, +before parallelization or execution can mutate the operators. Publication happens +only after the first query succeeds. A bounded background queue holds JSON strings +and performs read-back validation; a full queue simply skips publication. + +This is intentionally a general plan for the parameterized template. Constant +folding that depends on bindable parameter values cannot be applied without +specializing the plan to one execution and losing the slots needed by later queries. +Structural constants can still be optimized. Value-dependent selectivity estimates, +join ordering and distribution choices may therefore be less effective than when +planning literal SQL, including on the first cache miss. This is an expected trade-off +for avoiding repeated optimization. Cached row-count estimates come from the first optimization; new values and statistics changes alone do not trigger a new cost-based plan choice. The following boot options can be overridden in `drill-override.conf`. They are @@ -162,6 +179,47 @@ The query profile's **Plan Cache Hit** field, or JSON `planCacheHit`, identifies successful hit. Explain shows slot names and this execution's parameter values; those alone do not prove a hit. +DEBUG logging from `DrillSqlWorker` reports cache hits, misses and invalidation due +to context changes or binding failures. `PlanCache` logs publication and skipped +publication caused by serialization errors or a busy writer. Foreman thread names +include the query ID, allowing these messages to be correlated with query profiles. + +The following per-cache gauges are registered in `DrillMetrics` under +`drill.plan_cache.`: + +| Gauge | Meaning | +| --- | --- | +| `hits` | Plans successfully rebound and returned by eligible lookups. | +| `misses` | Eligible lookups without a usable entry, including stale entries and binding failures. Queries that bypass caching are excluded. | +| `invalidations` | Explicit invalidation requests, including whole-cache clear commands. | +| `evictions` | Guava automatic removals due to capacity or expiration. | +| `entries` | Current entry count reported by Guava; expired entries may remain counted until maintenance. | + +JMX reporting is enabled by default and can be controlled with the JVM property +`drill.metrics.jmx.enabled`. Periodic metric logging can be enabled with +`-Ddrill.metrics.log.enabled=true`; `drill.metrics.log.interval` defaults to 60 seconds. +Closing the cache unregisters its gauges. + +Administrators can clear the receiving Drillbit's cache with: + +```sql +ALTER SYSTEM CLEAR PLAN CACHE; +``` + +The command follows the SYSTEM option administrator policy: when user +authentication is enabled, only configured administrator users or members of +administrator groups can run it. With authentication disabled, the command is +available to all users. It returns `ok` and a `summary` identifying the Drillbit +whose cache was cleared. It does not broadcast to other Drillbits; to clear a +cluster, connect to each Drillbit and issue the command there. + +Clearing also advances a cache generation. Plans prepared before the clear, +including queued writes and publications pending query completion, cannot +repopulate the cache. Already-bound queries continue normally. Queries planned +after the clear can populate it again. Repeated clears, including while plan +caching is disabled for the session, succeed. Hit/miss counters remain cumulative; +each clear counts as one invalidation request. + See the [plugin developer guide](PLAN_CACHE_PLUGIN_GUIDE.md) for the integration contract. The accompanying pull request description contains validation results and benchmark methodology. diff --git a/PLAN_CACHE_PLUGIN_GUIDE.md b/docs/dev/PLAN_CACHE_PLUGIN_GUIDE.md similarity index 79% rename from PLAN_CACHE_PLUGIN_GUIDE.md rename to docs/dev/PLAN_CACHE_PLUGIN_GUIDE.md index 2cabace36a2..61d67ad603f 100644 --- a/PLAN_CACHE_PLUGIN_GUIDE.md +++ b/docs/dev/PLAN_CACHE_PLUGIN_GUIDE.md @@ -22,7 +22,7 @@ Plan caching is disabled by default. A plugin opts in through its metadata inter ## Storage-plugin API -Implement these methods in [StoragePlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java): +Implement these methods in [StoragePlugin](../../exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java): ```java boolean supportPlanCache(DrillTableSelection selection); @@ -38,14 +38,14 @@ incompatible. Do not use a data snapshot number if compatible data updates shoul keep hitting. A path alone is insufficient when replacement can change the schema or table identity. -DFS formats use the same two-method contract in [FormatPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java), with `FileSelection` as the input: +DFS formats use the same two-method contract in [FormatPlugin](../../exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java), with `FileSelection` as the input: ```java boolean supportPlanCache(FileSelection selection); PlanCacheTable planCacheTable(FileSelection selection) throws IOException; ``` -[FileSystemPlugin](exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java) delegates eligibility and metadata reads to the selected format plugin. It prefixes the returned table identifier with the format name to distinguish formats within the storage plugin. +[FileSystemPlugin](../../exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java) delegates eligibility and metadata reads to the selected format plugin. It prefixes the returned table identifier with the format name to distinguish formats within the storage plugin. ## Scan API and JSON reconstruction @@ -72,8 +72,8 @@ Keep the regular expression serialization for JSON so parameter slot numbers sur | Plugin | Compatibility version | Reconstruction | | --- | --- | --- | -| [HBaseStoragePlugin](contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java) / [HBaseGroupScan](contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java) | Table descriptor fingerprint | Retains the selection before pushdown and the original predicate; rebuilds native row-key bounds/filters and discovers current regions. | -| [IcebergFormatPlugin](contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java) / [IcebergGroupScan](contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java) | Table UUID and schema ID | Reloads the table and runs `planTasks()` for the current predicate and snapshot. Metadata tables and explicit snapshot selection return no support. | +| [HBaseStoragePlugin](../../contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java) / [HBaseGroupScan](../../contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java) | Table descriptor fingerprint | Retains the selection before pushdown and the original predicate; rebuilds native row-key bounds/filters and discovers current regions. | +| [IcebergFormatPlugin](../../contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/format/IcebergFormatPlugin.java) / [IcebergGroupScan](../../contrib/format-iceberg/src/main/java/org/apache/drill/exec/store/iceberg/IcebergGroupScan.java) | Table UUID and schema ID | Reloads the table and runs `planTasks()` for the current predicate and snapshot. Metadata tables and explicit snapshot selection return no support. | HBase descriptor equality means definition compatibility; a replacement with an identical descriptor can still be read safely because regions and filters are diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd b/exec/java-exec/src/main/codegen/data/Parser.tdd index ede92c22aca..b0eecadea78 100644 --- a/exec/java-exec/src/main/codegen/data/Parser.tdd +++ b/exec/java-exec/src/main/codegen/data/Parser.tdd @@ -30,6 +30,8 @@ # List of keywords. keywords: [ + "CLEAR", + "CACHE", "DAFFODIL", "DATABASES", "SCHEMAS", @@ -111,6 +113,7 @@ # List of methods for parsing extensions to "ALTER " calls. # Each must accept arguments "(SqlParserPos pos, String scope)". alterStatementParserMethods: [ + "SqlClearPlanCache", "DrillSqlSetOption", "DrillSqlResetOption" ] @@ -124,6 +127,8 @@ # Copied from calcite-core config.fmpp # For details please see comment under CALCITE-2405. nonReservedKeywords: [ + "CLEAR" + "CACHE" "A" "ABSENT" "ABSOLUTE" diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl index fe1d3159c31..0ad4bbcad27 100644 --- a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl +++ b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl @@ -976,12 +976,26 @@ DrillSqlSetOption DrillSqlSetOption(Span s, String scope) : } } +/** + * Parses CLEAR PLAN CACHE after ALTER SYSTEM. + */ +SqlClearPlanCache SqlClearPlanCache(Span s, String scope) : +{ +} +{ + + { + if (!"SYSTEM".equalsIgnoreCase(scope)) { + throw new ParseException("CLEAR PLAN CACHE requires ALTER SYSTEM."); + } + return new SqlClearPlanCache(s.end(this)); + } +} + /** * Parses a RESET statement without a leading "ALTER ": * * RESET { | ALL } - *

- * Statement handles in: {@link SetAndResetOptionHandler} */ DrillSqlResetOption DrillSqlResetOption(Span s, String scope) : { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index a496080c5b4..d0d3e238721 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -40,6 +40,7 @@ import org.apache.drill.exec.planner.PhysicalPlanReader; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.handlers.AbstractSqlHandler; +import org.apache.drill.exec.planner.sql.handlers.ClearPlanCacheHandler; import org.apache.drill.exec.planner.sql.handlers.AnalyzeTableHandler; import org.apache.drill.exec.planner.sql.handlers.DefaultSqlHandler; import org.apache.drill.exec.planner.sql.handlers.DescribeSchemaHandler; @@ -55,6 +56,7 @@ import org.apache.drill.exec.planner.sql.parser.DrillSqlCall; import org.apache.drill.exec.planner.sql.parser.DrillSqlDescribeTable; import org.apache.drill.exec.planner.sql.parser.DrillSqlResetOption; +import org.apache.drill.exec.planner.sql.parser.SqlClearPlanCache; import org.apache.drill.exec.planner.sql.parser.SqlSchema; import org.apache.drill.exec.planner.sql.conversion.SqlConverter; import org.apache.drill.exec.proto.UserBitShared.DrillPBError; @@ -273,6 +275,11 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point case DROP_VIEW: case OTHER_DDL: case OTHER: + if (sqlNode instanceof SqlClearPlanCache) { + handler = new ClearPlanCacheHandler(context); + context.setSQLStatementType(SqlStatementType.OTHER); + break; + } if (sqlNode instanceof DrillSqlCall) { handler = ((DrillSqlCall) sqlNode).getSqlHandler(config); if (handler instanceof AnalyzeTableHandler || handler instanceof MetastoreAnalyzeTableHandler) { @@ -306,6 +313,8 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point && !hasMutableAliases(context) && PlanCacheEligibility.isSafeToCache(sqlNode, context.getFunctionRegistry())) { try { + PlanCache cache = context.getDrillbitContext().getPlanCache(); + long cacheGeneration = cache.getGeneration(); // Capture the context once, before lookup and planning. Changes during // planning invalidate this entry on its first hit. PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve( @@ -313,9 +322,9 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point if (snapshot != null) { PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(sqlNode, parser.getTypeFactory()); - PlanCache cache = context.getDrillbitContext().getPlanCache(); String key = context.getQueryUserName() + '\n' + context.getSession().getDefaultSchemaPath() + '\n' + + snapshot.keyFingerprint() + '\n' + candidate.template; PlanCache.Entry entry = cache.get(key); if (entry != null) { @@ -326,20 +335,34 @@ private static PhysicalPlan getQueryPlan(QueryContext context, String sql, Point PlanCache.bindTextPlan(textPlan, entry.getTextPlan(), candidate.literals); cache.recordBind(); context.setPlanCacheHit(); + logger.debug("Plan cache hit"); return bound; } + logger.debug("Plan cache entry invalidated because its context changed; replanning"); } catch (RuntimeException | IOException e) { - logger.debug("Cached plan could not be rebound", e); + logger.debug("Plan cache entry invalidated because rebinding failed; replanning", e); } cache.invalidate(key); + } else { + logger.debug("Plan cache miss"); } + cache.recordMiss(); final PhysicalPlanReader cacheReader = context.getDrillbitContext().getPlanReader(); // Prepare insertion after planning; Foreman publishes it only after the query succeeds. prepareCacheInsert = planned -> { String templateTextPlan = PlanCache.bindTextPlan(textPlan, candidate.literals); - context.setPendingPlanCacheInsert(() -> cache.writeAfterSuccess(key, planned, - templateTextPlan, cacheReader, snapshot)); + final String json; + try { + // Capture before parallelization/execution mutates the operators. + // Pending publication and the writer queue retain only immutable JSON. + json = cacheReader.writeJson(planned); + } catch (IOException | RuntimeException e) { + logger.debug("Plan could not be serialized for the plan cache; skipping publication", e); + return; + } + context.setPendingPlanCacheInsert(() -> cache.writeAfterSuccess(key, json, + templateTextPlan, cacheReader, snapshot, cacheGeneration)); }; // On a cache miss, plan the parameterized SQL to produce a reusable template. planningSql = candidate.sql; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java index f98ecb57e0d..e45ad0d82d0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/PlanCache.java @@ -72,6 +72,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.metrics.DrillMetrics; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.planner.PhysicalPlanReader; @@ -96,6 +97,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.hash.Hashing; import com.google.common.primitives.Ints; +import com.codahale.metrics.Gauge; /** Drillbit-scoped, immutable JSON snapshots of query plans. */ public final class PlanCache implements AutoCloseable { @@ -104,6 +106,10 @@ public final class PlanCache implements AutoCloseable { private static final String MARKER = ExpressionStringBuilder.BOUND_DYNAMIC_PARAM + "("; private final Cache entries; private final AtomicLong successfulBinds = new AtomicLong(); + private final AtomicLong misses = new AtomicLong(); + private final AtomicLong invalidations = new AtomicLong(); + private final Map> metrics = new LinkedHashMap<>(); + private volatile long generation; private final ThreadPoolExecutor writer = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(64), new NamedThreadFactory("plan-cache-writer-")); @@ -124,7 +130,8 @@ public PlanCache(long maxSizeBytes, Duration expireAfterWrite, Duration expireAf .maximumWeight(maxSizeBytes) .weigher((String key, Entry value) -> Ints.saturatedCast( key.getBytes(StandardCharsets.UTF_8).length + value.contentSizeBytes)) - .ticker(ticker); + .ticker(ticker) + .recordStats(); // Passing zero to Guava would expire entries immediately, rather than disable the policy. if (!expireAfterWrite.isZero()) { builder.expireAfterWrite(expireAfterWrite); @@ -135,6 +142,17 @@ public PlanCache(long maxSizeBytes, Duration expireAfterWrite, Duration expireAf entries = builder.build(); logger.info("Plan cache settings: max_size_bytes={}, expire_after_write={}, expire_after_access={}" + " (zero expiration disables that policy)", maxSizeBytes, expireAfterWrite, expireAfterAccess); + registerMetric("hits", successfulBinds::get); + registerMetric("misses", misses::get); + registerMetric("invalidations", invalidations::get); + registerMetric("evictions", () -> entries.stats().evictionCount()); + registerMetric("entries", entries::size); + } + + private void registerMetric(String suffix, Gauge gauge) { + String name = "drill.plan_cache." + suffix; + metrics.put(name, gauge); + DrillMetrics.register(name, gauge); } public long getHitCount() { @@ -145,6 +163,11 @@ public void recordBind() { successfulBinds.incrementAndGet(); } + /** Counts an eligible lookup that could not produce a compatible, bound plan. */ + public void recordMiss() { + misses.incrementAndGet(); + } + /** Updates optional explain output using the cached template. */ static void bindTextPlan(Pointer textPlan, String template, List literals) { if (textPlan != null) { @@ -188,12 +211,12 @@ public void awaitWrites() throws InterruptedException, ExecutionException, Timeo writer.submit(() -> { }).get(10, TimeUnit.SECONDS); } - public void writeAfterSuccess(String key, PhysicalPlan plan, String textPlan, - PhysicalPlanReader reader, ContextSnapshot context) { + public void writeAfterSuccess(String key, String json, String textPlan, + PhysicalPlanReader reader, ContextSnapshot context, long expectedGeneration) { try { writer.execute(() -> { try { - put(key, plan, textPlan, reader, context); + put(key, json, textPlan, reader, context, expectedGeneration); } catch (RuntimeException | IOException e) { logger.debug("Plan cache entry could not be published", e); } @@ -204,26 +227,62 @@ public void writeAfterSuccess(String key, PhysicalPlan plan, String textPlan, } public void invalidate(String key) { + invalidations.incrementAndGet(); entries.invalidate(key); } - public boolean put(String key, PhysicalPlan plan, String textPlan, + long getGeneration() { + return generation; + } + + /** Clears local entries and prevents pre-clear plans from being published later. */ + public synchronized void clear() { + generation++; + invalidations.incrementAndGet(); + entries.invalidateAll(); + entries.cleanUp(); + logger.info("Local plan cache cleared; generation={}", generation); + } + + public boolean put(String key, String json, String textPlan, PhysicalPlanReader reader, ContextSnapshot context) throws IOException { + return put(key, json, textPlan, reader, context, generation); + } + + private boolean put(String key, String json, String textPlan, + PhysicalPlanReader reader, ContextSnapshot context, long expectedGeneration) throws IOException { // The engine checks generic context and expression compatibility. Each // opted-in plugin is responsible for rebuilding its scans' value-dependent state. - if (context == null) { + if (context == null || expectedGeneration != generation) { return false; } - String json = reader.writeJson(plan); // Confirm that standard PhysicalPlan JSON can be read back before publishing. reader.readPhysicalPlan(json); - entries.put(key, new Entry(json, textPlan, context)); + // Readback may perform plugin I/O. Keep it outside the clear/publication lock, + // then check again so a concurrent clear cannot be undone by that readback. + synchronized (this) { + if (expectedGeneration != generation) { + logger.debug("Skipping plan cache publication after a cache clear"); + return false; + } + entries.put(key, new Entry(json, textPlan, context)); + } + logger.debug("Plan cache entry published; entries={}", entries.size()); return true; } @Override public void close() { writer.shutdownNow(); + // Test JVMs may contain several Drillbits. A closing cache must not remove + // gauges registered by a newer cache under the same names. + synchronized (DrillMetrics.class) { + metrics.forEach((name, gauge) -> { + if (DrillMetrics.getRegistry().getMetrics().get(name) == gauge) { + DrillMetrics.getRegistry().remove(name); + } + }); + } } public static final class Entry { @@ -475,6 +534,16 @@ private boolean matches(ContextSnapshot current) { && pluginConfigs.equals(current.pluginConfigs); } + /** Options and plugin configurations identify separate reusable templates. */ + String keyFingerprint() { + StringBuilder configs = new StringBuilder(); + new TreeMap<>(pluginConfigs).forEach((name, fingerprint) -> configs + .append(name.length()).append(':').append(name).append('=').append(fingerprint).append('\n')); + // Keep table versions outside the key so incompatible entries can be invalidated. + return optionsFingerprint + '\n' + + Hashing.sha256().hashString(configs, StandardCharsets.UTF_8).toString(); + } + private static String optionFingerprint(QueryContext context) { // OptionManager iteration walks system defaults, system overrides, session // overrides, and query overrides in precedence order. getOptionList() only diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java index 4169a58f561..fd049442cd0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlHandler.java @@ -27,12 +27,28 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.calcite.sql.SqlNode; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.server.options.QueryOptionManager; +import org.apache.drill.exec.util.ImpersonationUtil; public abstract class AbstractSqlHandler { static final Logger logger = LoggerFactory.getLogger(AbstractSqlHandler.class); public abstract PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException; + /** Applies the same administrator policy to SYSTEM options and cache management. */ + protected static void checkAdminPrivileges(QueryContext context, QueryOptionManager options, String message) { + if (context.isUserAuthenticationEnabled() + && !ImpersonationUtil.hasAdminPrivileges( + context.getQueryUserName(), + ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options), + ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) { + throw UserException.permissionError().message(message).build(logger); + } + } + @SuppressWarnings("unchecked") public static T unwrap(Object o, Class clazz) throws ForemanSetupException { if (clazz.isAssignableFrom(o.getClass())) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java index e6bf3700e97..315e60bab93 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/AbstractSqlSetHandler.java @@ -23,7 +23,6 @@ import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.server.options.QueryOptionManager; -import org.apache.drill.exec.util.ImpersonationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,16 +73,6 @@ OptionValue.OptionScope getScope(SqlSetOption statement, QueryOptionManager opti * @param options Options object */ void checkAdminPrivileges(QueryOptionManager options) { - if (context.isUserAuthenticationEnabled() - && !ImpersonationUtil.hasAdminPrivileges( - context.getQueryUserName(), - ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options), - ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) { - - throw UserException - .permissionError() - .message("Not authorized to change SYSTEM options.") - .build(logger); - } + checkAdminPrivileges(context, options, "Not authorized to change SYSTEM options."); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ClearPlanCacheHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ClearPlanCacheHandler.java new file mode 100644 index 00000000000..aba3535651f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ClearPlanCacheHandler.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.handlers; + +import org.apache.calcite.sql.SqlNode; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.sql.DirectPlan; +import org.apache.drill.exec.planner.sql.parser.SqlClearPlanCache; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.work.foreman.ForemanSetupException; + +/** Clears the cache on the Drillbit that plans this command. */ +public final class ClearPlanCacheHandler extends AbstractSqlHandler { + private final QueryContext context; + + public ClearPlanCacheHandler(QueryContext context) { + this.context = context; + } + + @Override + public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException { + unwrap(sqlNode, SqlClearPlanCache.class); + checkAdminPrivileges(context, context.getOptions(), "Not authorized to clear the SYSTEM plan cache."); + context.getDrillbitContext().getPlanCache().clear(); + DrillbitEndpoint endpoint = context.getCurrentEndpoint(); + return DirectPlan.createDirectPlan(context, true, String.format( + "Plan cache cleared on Drillbit %s:%d.", endpoint.getAddress(), endpoint.getUserPort())); + } +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlClearPlanCache.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlClearPlanCache.java new file mode 100644 index 00000000000..e2a7ce20df5 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/parser/SqlClearPlanCache.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.parser; + +import java.util.Collections; +import java.util.List; + +import org.apache.calcite.sql.SqlAlter; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSpecialOperator; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; + +/** Parse tree for clearing the receiving Drillbit's physical plan cache. */ +public final class SqlClearPlanCache extends SqlAlter { + public static final SqlSpecialOperator OPERATOR = + new SqlSpecialOperator("CLEAR_PLAN_CACHE", SqlKind.OTHER_DDL) { + @Override + public SqlCall createCall(SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... operands) { + return new SqlClearPlanCache(pos); + } + }; + + public SqlClearPlanCache(SqlParserPos pos) { + super(pos, "SYSTEM"); + } + + @Override + public SqlOperator getOperator() { + return OPERATOR; + } + + @Override + public List getOperandList() { + return Collections.emptyList(); + } + + @Override + protected void unparseAlterOperation(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword("CLEAR PLAN CACHE"); + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java index c7029bcdbb9..70ca96f6848 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheConfig.java @@ -18,11 +18,14 @@ package org.apache.drill.exec.planner.sql; import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import com.google.common.base.Ticker; import org.apache.drill.categories.PlannerTest; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.metrics.DrillMetrics; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.planner.PhysicalPlanReader; import org.apache.drill.test.BaseTest; @@ -156,11 +159,11 @@ public void testCapacityCountsUtf8KeyBytes() throws Exception { public void testCapacityCountsUtf8ExplainBytes() throws Exception { // Two key bytes, two JSON bytes, and three explain text bytes. try (PlanCache cache = new PlanCache(6, Duration.ZERO, Duration.ZERO, ticker)) { - assertTrue(cache.put("é", plan, "计", reader, context)); + assertTrue(cache.put("é", reader.writeJson(plan), "计", reader, context)); assertNull(cache.get("é")); } try (PlanCache cache = new PlanCache(7, Duration.ZERO, Duration.ZERO, ticker)) { - assertTrue(cache.put("é", plan, "计", reader, context)); + assertTrue(cache.put("é", reader.writeJson(plan), "计", reader, context)); assertNotNull(cache.get("é")); assertEquals("计", cache.get("é").getTextPlan()); } @@ -172,7 +175,7 @@ public void testNullAndEmptyExplainTextHaveZeroWeight() throws Exception { put(cache, "é"); assertNotNull(cache.get("é")); assertNull(cache.get("é").getTextPlan()); - assertTrue(cache.put("é", plan, "", reader, context)); + assertTrue(cache.put("é", reader.writeJson(plan), "", reader, context)); assertNotNull(cache.get("é")); assertEquals("", cache.get("é").getTextPlan()); } @@ -192,9 +195,36 @@ public void testCapacityEvictsBeforeIdleExpiration() throws Exception { assertNull(cache.get("b")); assertNotNull(cache.get("a")); assertNotNull(cache.get("c")); + assertEquals(1L, metric("evictions")); + assertEquals(2L, metric("entries")); } } + @Test + public void testRegisteredInvalidationAndEntryMetrics() throws Exception { + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + assertEquals(0L, metric("invalidations")); + assertEquals(0L, metric("entries")); + put(cache, "key"); + assertEquals(1L, metric("entries")); + cache.invalidate("key"); + assertEquals(1L, metric("invalidations")); + assertEquals(0L, metric("entries")); + } + assertNull(DrillMetrics.getRegistry().getGauges().get("drill.plan_cache.entries")); + } + + @Test + public void testClosingOlderCacheKeepsNewerCacheMetrics() { + try (PlanCache older = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker); + PlanCache newer = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + older.close(); + newer.recordBind(); + assertEquals(1L, metric("hits")); + } + assertNull(DrillMetrics.getRegistry().getGauges().get("drill.plan_cache.hits")); + } + @Test public void testZeroCapacityDisablesStorage() throws Exception { try (PlanCache cache = new PlanCache(0, Duration.ZERO, Duration.ZERO, ticker)) { @@ -217,7 +247,44 @@ public void testNegativeSettingsAreRejected() { } private void put(PlanCache cache, String key) throws Exception { - assertTrue(cache.put(key, plan, null, reader, context)); + assertTrue(cache.put(key, reader.writeJson(plan), null, reader, context)); + } + + @Test + public void testClearDiscardsActiveAndQueuedPublications() throws Exception { + CountDownLatch reading = new CountDownLatch(1); + CountDownLatch resume = new CountDownLatch(1); + when(reader.readPhysicalPlan("{}")).thenAnswer(invocation -> { + reading.countDown(); + assertTrue(resume.await(10, TimeUnit.SECONDS)); + return plan; + }); + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO, ticker)) { + long generation = cache.getGeneration(); + try { + cache.writeAfterSuccess("active", "{}", null, reader, context, generation); + assertTrue(reading.await(10, TimeUnit.SECONDS)); + cache.writeAfterSuccess("queued", "{}", null, reader, context, generation); + cache.clear(); + } finally { + resume.countDown(); + } + cache.awaitWrites(); + assertNull(cache.get("active")); + assertNull(cache.get("queued")); + assertEquals(0L, metric("entries")); + assertEquals(1L, metric("invalidations")); + assertEquals(0L, metric("evictions")); + + // A newly planned query can publish with the new generation. + cache.writeAfterSuccess("new", "{}", null, reader, context, cache.getGeneration()); + cache.awaitWrites(); + assertNotNull(cache.get("new")); + } + } + + private long metric(String name) { + return (Long) DrillMetrics.getRegistry().getGauges().get("drill.plan_cache." + name).getValue(); } private static final class ManualTicker extends Ticker { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheContext.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheContext.java new file mode 100644 index 00000000000..43e63087c7d --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCacheContext.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import java.time.Duration; +import java.util.Collections; + +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.drill.categories.PlannerTest; +import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.drill.exec.planner.logical.DrillTableSelection; +import org.apache.drill.exec.server.options.QueryOptionManager; +import org.apache.drill.exec.store.PlanCacheTable; +import org.apache.drill.exec.store.StoragePlugin; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.test.BaseTest; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Category(PlannerTest.class) +public class TestPlanCacheContext extends BaseTest { + private QueryContext context; + private SchemaPlus schema; + private StoragePluginRegistry registry; + private StoragePlugin pluginA; + private StoragePluginConfig configA; + private DrillTableSelection selectionA; + + @Before + public void setup() throws Exception { + context = mock(QueryContext.class); + schema = mock(SchemaPlus.class); + registry = mock(StoragePluginRegistry.class); + QueryOptionManager options = mock(QueryOptionManager.class); + when(options.iterator()).thenAnswer(call -> Collections.emptyIterator()); + when(context.getOptions()).thenReturn(options); + when(context.getStorage()).thenReturn(registry); + configA = mock(StoragePluginConfig.class); + pluginA = mock(StoragePlugin.class); + selectionA = mock(DrillTableSelection.class); + addTable("a", pluginA, configA, selectionA); + addTable("b", mock(StoragePlugin.class), mock(StoragePluginConfig.class), mock(DrillTableSelection.class)); + } + + @Test + public void testPluginConfigChangesPartitionCacheKey() throws Exception { + PlanCache.ContextSnapshot original = snapshot("SELECT * FROM \"a\""); + when(registry.encode(configA)).thenReturn("changed plugin configuration"); + PlanCache.ContextSnapshot changed = snapshot("SELECT * FROM \"a\""); + assertNotEquals(original.keyFingerprint(), changed.keyFingerprint()); + } + + @Test + public void testPluginTraversalOrderDoesNotChangeFingerprint() throws Exception { + PlanCache.ContextSnapshot first = snapshot("SELECT * FROM \"a\" JOIN \"b\" ON TRUE"); + PlanCache.ContextSnapshot reversed = snapshot("SELECT * FROM \"b\" JOIN \"a\" ON TRUE"); + assertEquals(first.keyFingerprint(), reversed.keyFingerprint()); + } + + @Test + public void testTableVersionChangeInvalidatesEntryWithoutChangingKey() throws Exception { + PlanCache.ContextSnapshot original = snapshot("SELECT * FROM \"a\""); + when(pluginA.planCacheTable(selectionA)).thenReturn(new PlanCacheTable("a", "version-2")); + PlanCache.ContextSnapshot changed = snapshot("SELECT * FROM \"a\""); + assertEquals(original.keyFingerprint(), changed.keyFingerprint()); + try (PlanCache cache = new PlanCache(1024, Duration.ZERO, Duration.ZERO)) { + assertTrue(cache.put("key", "{}", null, mock(PhysicalPlanReader.class), original)); + assertTrue(cache.get("key").matchesContext(original)); + assertFalse(cache.get("key").matchesContext(changed)); + } + } + + private void addTable(String name, StoragePlugin plugin, StoragePluginConfig config, + DrillTableSelection selection) throws Exception { + DrillTable table = mock(DrillTable.class); + when(table.getSelection()).thenReturn(selection); + when(table.getPlugin()).thenReturn(plugin); + when(table.getStorageEngineName()).thenReturn("storage-" + name); + when(schema.getTable(name)).thenReturn(table); + when(plugin.supportPlanCache(selection)).thenReturn(true); + when(plugin.planCacheTable(selection)).thenReturn(new PlanCacheTable(name, "version-1")); + when(plugin.getConfig()).thenReturn(config); + when(registry.encode(config)).thenReturn("configuration-" + name); + } + + private PlanCache.ContextSnapshot snapshot(String sql) throws Exception { + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve( + schema, SqlParser.create(sql).parseQuery(), context); + assertNotNull(snapshot); + return snapshot; + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java index 99393e2e90c..7003eb82270 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java @@ -17,8 +17,24 @@ */ package org.apache.drill.exec.planner.sql; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.calcite.sql.SqlNode; import org.apache.drill.categories.SqlTest; +import org.apache.drill.common.config.DrillProperties; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.metrics.DrillMetrics; +import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.sql.conversion.SqlConverter; +import org.apache.drill.exec.proto.UserBitShared.QueryId; +import org.apache.drill.exec.proto.UserBitShared.UserCredentials; +import org.apache.drill.exec.rpc.user.UserSession; +import org.apache.drill.exec.util.Pointer; +import org.apache.drill.test.ClientFixture; import org.apache.drill.test.ClusterFixture; import org.apache.drill.test.ClusterTest; import org.junit.Before; @@ -27,6 +43,8 @@ import org.junit.experimental.categories.Category; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotEquals; @Category(SqlTest.class) public class TestPlanCachePlanning extends ClusterTest { @@ -50,15 +68,19 @@ public void testHavingMatchesGroupedConcatenationOnMissAndHit() throws Exception client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); long hits = cache().getHitCount(); + long misses = metric("misses"); client.testBuilder().sqlQuery(sql, 0).unOrdered() .baselineColumns("k", "n").baselineValues("ba", 2L).go(); cache().awaitWrites(); assertEquals(hits, cache().getHitCount()); + assertEquals(misses + 1, metric("misses")); // WHERE values remain bindable even though the matching HAVING literals are preserved. client.testBuilder().sqlQuery(sql, 1).unOrdered() .baselineColumns("k", "n").baselineValues("ba", 1L).go(); assertEquals(hits + 1, cache().getHitCount()); + assertEquals(hits + 1, metric("hits")); + assertEquals(misses + 1, metric("misses")); } @Test @@ -108,4 +130,129 @@ public void testQualifyMatchesOrderedProjectionOnMissAndHit() throws Exception { private PlanCache cache() { return cluster.drillbit().getContext().getPlanCache(); } + + @Test + public void testClearPlanCacheForcesMissThenAllowsNewHits() throws Exception { + String sql = "SELECT x AS clear_value FROM (VALUES (1), (2)) AS t(x) WHERE x > %d"; + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("clear_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + long hits = cache().getHitCount(); + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("clear_value").baselineValues(2).go(); + assertEquals(hits + 1, cache().getHitCount()); + + long invalidations = metric("invalidations"); + clearCacheWithSql(); + assertEquals(0L, metric("entries")); + assertEquals(invalidations + 1, metric("invalidations")); + assertEquals(hits + 1, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("clear_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + assertEquals(hits + 1, cache().getHitCount()); + client.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("clear_value").baselineValues(2).go(); + assertEquals(hits + 2, cache().getHitCount()); + + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + clearCacheWithSql(); + clearCacheWithSql(); + assertEquals(0L, metric("entries")); + } + + private void clearCacheWithSql() throws Exception { + client.testBuilder().sqlQuery("ALTER SYSTEM CLEAR PLAN CACHE").unOrdered() + .baselineColumns("ok", "summary") + .baselineValues(true, String.format("Plan cache cleared on Drillbit %s:%d.", + cluster.drillbit().getContext().getEndpoint().getAddress(), + cluster.drillbit().getContext().getEndpoint().getUserPort())).go(); + } + + private long metric(String name) { + return (Long) DrillMetrics.getRegistry().getGauges().get("drill.plan_cache." + name).getValue(); + } + + @Test + public void testDifferentSessionOptionsKeepSeparateCachedPlans() throws Exception { + String sql = "SELECT x AS option_value FROM (VALUES (1), (2)) AS t(x) WHERE x > %d"; + try (ClientFixture first = cluster.clientBuilder().property(DrillProperties.USER, "options-test").build(); + ClientFixture second = cluster.clientBuilder().property(DrillProperties.USER, "options-test").build()) { + first.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + second.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + first.alterSession(ExecConstants.SLICE_TARGET, 10000); + second.alterSession(ExecConstants.SLICE_TARGET, 20000); + long hits = cache().getHitCount(); + first.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("option_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + second.testBuilder().sqlQuery(sql, 0).unOrdered() + .baselineColumns("option_value").baselineValues(1).baselineValues(2).go(); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + + // Alternating clients must keep hitting their own entry rather than evicting each other. + first.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("option_value").baselineValues(2).go(); + second.testBuilder().sqlQuery(sql, 1).unOrdered() + .baselineColumns("option_value").baselineValues(2).go(); + assertEquals(hits + 2, cache().getHitCount()); + } + } + + @Test + public void testPublicationUsesSnapshotBeforeExecutionMutatesOperators() throws Exception { + UserSession session = UserSession.Builder.newBuilder() + .withCredentials(UserCredentials.newBuilder().setUserName("snapshot-test").build()) + .withOptionManager(cluster.drillbit().getContext().getOptionManager()) + .setSupportComplexTypes(true).build(); + session.getOptions().setLocalOption(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + String sql = "SELECT x AS snapshot_value FROM (VALUES (1), (2)) AS t(x) WHERE x > 0"; + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan planned = DrillSqlWorker.getPlan(context, sql, new Pointer<>()); + Runnable publish = context.takePendingPlanCacheInsert(); + assertNotNull(publish); + List originalIds = planned.getSortedOperators().stream() + .map(PhysicalOperator::getOperatorId).collect(Collectors.toList()); + planned.getSortedOperators().forEach(operator -> operator.setOperatorId(operator.getOperatorId() + 1000)); + + publish.run(); + cache().awaitWrites(); + SqlConverter converter = new SqlConverter(context); + SqlNode parsed = converter.parse(sql); + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve(converter.getDefaultSchema(), parsed, context); + assertNotNull(snapshot); + PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(parsed, converter.getTypeFactory()); + String key = context.getQueryUserName() + '\n' + session.getDefaultSchemaPath() + '\n' + + snapshot.keyFingerprint() + '\n' + candidate.template; + PlanCache.Entry entry = cache().get(key); + assertNotNull(entry); + PhysicalPlan cached = entry.bind(candidate.literals, cluster.drillbit().getContext().getPlanReader()); + List cachedIds = cached.getSortedOperators().stream() + .map(PhysicalOperator::getOperatorId).collect(Collectors.toList()); + assertEquals(originalIds, cachedIds); + assertNotEquals(planned.getSortedOperators().stream().map(PhysicalOperator::getOperatorId) + .collect(Collectors.toList()), cachedIds); + } + } + + @Test + public void testClearPreventsPublicationPendingQueryCompletion() throws Exception { + UserSession session = UserSession.Builder.newBuilder() + .withCredentials(UserCredentials.newBuilder().setUserName("clear-pending-test").build()) + .withOptionManager(cluster.drillbit().getContext().getOptionManager()) + .setSupportComplexTypes(true).build(); + session.getOptions().setLocalOption(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + DrillSqlWorker.getPlan(context, + "SELECT x AS pending_value FROM (VALUES (1), (2)) AS t(x) WHERE x > 0", new Pointer<>()); + Runnable publish = context.takePendingPlanCacheInsert(); + assertNotNull(publish); + clearCacheWithSql(); + publish.run(); + cache().awaitWrites(); + assertEquals(0L, metric("entries")); + } + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/parser/TestClearPlanCacheSqlParser.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/parser/TestClearPlanCacheSqlParser.java new file mode 100644 index 00000000000..9ea918a54ac --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/parser/TestClearPlanCacheSqlParser.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql.parser; + +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.drill.categories.SqlTest; +import org.apache.drill.exec.planner.sql.parser.impl.DrillParserImpl; +import org.apache.drill.test.BaseTest; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +@Category(SqlTest.class) +public class TestClearPlanCacheSqlParser extends BaseTest { + private SqlNode parse(String sql) throws SqlParseException { + return SqlParser.create(sql, SqlParser.config().withParserFactory(DrillParserImpl.FACTORY)).parseStmt(); + } + + @Test + public void testClearPlanCacheRoundTrip() throws Exception { + SqlNode node = parse("alter system clear plan cache"); + assertTrue(node instanceof SqlClearPlanCache); + assertEquals(SqlKind.OTHER_DDL, node.getKind()); + assertEquals("SYSTEM", ((SqlClearPlanCache) node).getScope()); + assertTrue(parse(node.toString()) instanceof SqlClearPlanCache); + assertTrue(SqlNode.clone(node) instanceof SqlClearPlanCache); + } + + @Test + public void testOnlySystemScopeIsAccepted() { + assertThrows(SqlParseException.class, () -> parse("ALTER SESSION CLEAR PLAN CACHE")); + assertThrows(SqlParseException.class, () -> parse("CLEAR PLAN CACHE")); + assertThrows(SqlParseException.class, () -> parse("ALTER SYSTEM CLEAR PLAN")); + assertThrows(SqlParseException.class, () -> parse("ALTER SYSTEM CLEAR PLAN CACHE extra")); + } + + @Test + public void testExistingOptionSyntaxAndIdentifiersStillParse() throws Exception { + assertEquals(SqlKind.SET_OPTION, parse("ALTER SYSTEM SET foo = 1").getKind()); + assertTrue(parse("ALTER SYSTEM RESET ALL") instanceof DrillSqlResetOption); + assertEquals(SqlKind.SET_OPTION, parse("ALTER SESSION SET foo = 1").getKind()); + assertTrue(parse("ALTER SESSION RESET foo") instanceof DrillSqlResetOption); + assertEquals(SqlKind.SELECT, parse("SELECT clear, cache FROM t").getKind()); + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java index 3f7f97c2b50..acaafcd6a4a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java @@ -93,6 +93,24 @@ public void updateSysOptAsUserInAdminGroup() throws Exception { setOptHelper(); } + @Test + public void clearPlanCacheAsAdminUser() throws Exception { + updateClient(ADMIN_USER, ADMIN_USER_PASSWORD); + test("ALTER SYSTEM CLEAR PLAN CACHE"); + } + + @Test + public void clearPlanCacheAsUserInAdminGroup() throws Exception { + updateClient(TEST_USER_1, TEST_USER_1_PASSWORD); + test("ALTER SYSTEM CLEAR PLAN CACHE"); + } + + @Test + public void cannotClearPlanCacheAsNonAdminUser() throws Exception { + updateClient(TEST_USER_2, TEST_USER_2_PASSWORD); + errorMsgTestHelper("ALTER SYSTEM CLEAR PLAN CACHE", "Not authorized to clear the SYSTEM plan cache."); + } + @Test public void trySettingAdminOptsAtSessionScopeAsAdmin() throws Exception { updateClient(ADMIN_USER, ADMIN_USER_PASSWORD); diff --git a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 index a9a5c760dd7..49f41525bd8 100644 --- a/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 +++ b/logical/src/main/antlr4/org/apache/drill/common/expression/parser/ExprParser.g4 @@ -48,24 +48,6 @@ import org.apache.drill.common.exceptions.ExpressionParsingException; return new ExpressionPosition(fullExpression, token.getTokenIndex()); } - private LogicalExpression createCast(MajorType type, ExpressionPosition position, - LogicalExpression input) { - // BIGINT casts emitted for integer literals preserve their width in JSON. - // Restore the literal node so scan translators see the original predicate. - // Keep parameter casts intact to retain their slot identity and conversion. - if (type.equals(Types.required(MinorType.BIGINT)) - && input instanceof LiteralExpression - && !((LiteralExpression) input).isDynamicParam()) { - if (input instanceof ValueExpressions.IntExpression) { - return new ValueExpressions.LongExpression( - ((ValueExpressions.IntExpression) input).getInt(), position); - } - if (input instanceof ValueExpressions.LongExpression) { - return input; - } - } - return FunctionCallFactory.createCast(type, position, input); - } } parse returns [LogicalExpression e] @@ -102,7 +84,7 @@ castCall returns [LogicalExpression e] { if ($repeat.ctx != null && $repeat.isRep.compareTo(Boolean.TRUE)==0) $e = FunctionCallFactory.createCast(TypeProtos.MajorType.newBuilder().mergeFrom($dataType.type).setMode(DataMode.REPEATED).build(), pos($Cast), $expression.e); else - $e = createCast($dataType.type, pos($Cast), $expression.e);} + $e = FunctionCallFactory.createCast($dataType.type, pos($Cast), $expression.e);} ; repeat returns [Boolean isRep] diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java index 94596cfde6d..4682dbf63b8 100644 --- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java +++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java @@ -186,12 +186,9 @@ public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeExc @Override public Void visitLongConstant(LongExpression lExpr, StringBuilder sb) throws RuntimeException { startBoundLiteral(lExpr, sb); - if (!lExpr.isDynamicParam()) { - // Preserve integer width in JSON; the parser restores this as a BIGINT literal. - sb.append("cast(").append(lExpr.getLong()).append(" as BIGINT)"); - } else { - sb.append(lExpr.getLong()); - } + // Bound parameters carry BIGINT in their wrapper; ordinary literals keep + // the existing representation used by explain output and scan digests. + sb.append(lExpr.getLong()); endBoundLiteral(lExpr, sb); return null; } diff --git a/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java index 5f5ee0f2f23..83c85b2e774 100644 --- a/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java +++ b/logical/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java @@ -18,8 +18,10 @@ package org.apache.drill.common.expression.parser; import org.apache.drill.common.exceptions.ExpressionParsingException; +import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.parser.LogicalExpressionParser; import org.apache.drill.test.DrillTest; import org.junit.jupiter.api.Test; @@ -28,6 +30,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class TreeTest extends DrillTest { @@ -108,6 +111,30 @@ void testFunctionCallWithoutParams() { testExpressionParsing(expr, expr); } + @Test + void testOrdinaryBigintSerializationKeepsExistingRepresentation() { + for (long value : new long[] {0, -1, Integer.MAX_VALUE, (long) Integer.MAX_VALUE + 1, Long.MIN_VALUE}) { + assertEquals(Long.toString(value), serializeExpression(new LongExpression(value))); + } + assertTrue(LogicalExpressionParser.parse("cast(1 as BIGINT)") instanceof CastExpression); + } + + @Test + void testBoundBigintPreservesWidthAndSlotOnRoundTrip() { + for (long value : new long[] {0, -1, Integer.MIN_VALUE, Integer.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE}) { + LongExpression original = new LongExpression(value); + original.setDynamicParamIndex(3); + String serialized = serializeExpression(original); + assertEquals("bound_dynamic_param(3, BIGINT, " + value + ")", serialized); + LogicalExpression parsed = LogicalExpressionParser.parse(serialized); + assertTrue(parsed instanceof LongExpression); + LongExpression restored = (LongExpression) parsed; + assertEquals(value, restored.getLong()); + assertEquals(original.getMajorType(), restored.getMajorType()); + assertEquals(3, restored.getDynamicParamIndex()); + } + } + /** * Attempt to parse an expression. Once parsed, convert it to a string and then parse it again to make sure serialization works. */ From 6ee1c592b66c7e5ff16abcf7e6b0a16bf5b9dc41 Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Fri, 9 Oct 2026 00:24:02 +0800 Subject: [PATCH 15/16] Reduce HBase plan cache metadata lookup overhead --- .../exec/store/hbase/HBaseStoragePlugin.java | 15 ++- .../apache/drill/hbase/HBaseTestsSuite.java | 1 + .../hbase/TestHBasePlanCacheMetadata.java | 120 ++++++++++++++++++ 3 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCacheMetadata.java diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java index 16dc1e7498a..53280a545ed 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java @@ -34,9 +34,11 @@ import com.google.common.collect.ImmutableSet; import com.google.common.hash.Hashing; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import com.fasterxml.jackson.core.type.TypeReference; @@ -74,13 +76,12 @@ public PlanCacheTable planCacheTable(DrillTableSelection selection) throws IOExc return null; } String identifier = ((HBaseScanSpec) selection).getTableName(); - try (Admin admin = getConnection().getAdmin()) { - TableName tableName = TableName.valueOf(identifier); - if (!admin.tableExists(tableName)) { - return null; - } - String version = Hashing.sha256().hashBytes(admin.getTableDescriptor(tableName).toByteArray()).toString(); + try (Table table = getConnection().getTable(TableName.valueOf(identifier))) { + String version = Hashing.sha256().hashBytes( + TableDescriptorBuilder.toByteArray(table.getDescriptor())).toString(); return new PlanCacheTable(identifier, version); + } catch (TableNotFoundException e) { + return null; } } diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java index 39c1e3b1f94..cd10e9d6497 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java @@ -45,6 +45,7 @@ TestHBaseCFAsJSONString.class, TestHBaseConnectionManager.class, TestHBaseFilterPushDown.class, + TestHBasePlanCacheMetadata.class, TestHBaseProjectPushDown.class, TestHBaseQueries.class, TestHBaseRegexParser.class, diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCacheMetadata.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCacheMetadata.java new file mode 100644 index 00000000000..97f645dcffe --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCacheMetadata.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import java.io.IOException; + +import com.google.common.hash.Hashing; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.PlanCacheTable; +import org.apache.drill.exec.store.hbase.HBaseScanSpec; +import org.apache.drill.exec.store.hbase.HBaseStoragePlugin; +import org.apache.drill.exec.store.hbase.HBaseStoragePluginConfig; +import org.apache.drill.test.BaseTest; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class TestHBasePlanCacheMetadata extends BaseTest { + private static final TableName NAME = TableName.valueOf("plan_cache_metadata"); + private final HBaseScanSpec selection = new HBaseScanSpec(NAME.getNameAsString()); + private Connection connection; + private Table table; + private HBaseStoragePlugin plugin; + + @Before + public void setup() throws Exception { + connection = mock(Connection.class); + table = mock(Table.class); + when(connection.getTable(NAME)).thenReturn(table); + plugin = spy(new HBaseStoragePlugin(new HBaseStoragePluginConfig(null, false), + mock(DrillbitContext.class), "hbase")); + doReturn(connection).when(plugin).getConnection(); + } + + @Test + public void testDescriptorFingerprintPreservesExistingCompatibilityVersion() throws Exception { + TableDescriptor descriptor = descriptor("cf1"); + when(table.getDescriptor()).thenReturn(descriptor); + PlanCacheTable metadata = plugin.planCacheTable(selection); + assertEquals(NAME.getNameAsString(), metadata.getIdentifier()); + assertEquals(Hashing.sha256().hashBytes(new HTableDescriptor(descriptor).toByteArray()).toString(), + metadata.getVersion()); + verify(table).getDescriptor(); + verify(table).close(); + verifyNoMoreInteractions(table); + verify(connection).getTable(NAME); + // No Admin handle, separate existence check or connection close is needed. + verifyNoMoreInteractions(connection); + } + + @Test + public void testSchemaChangeChangesCompatibilityVersion() throws Exception { + when(table.getDescriptor()).thenReturn(descriptor("cf1"), descriptor("cf2")); + assertNotEquals(plugin.planCacheTable(selection).getVersion(), + plugin.planCacheTable(selection).getVersion()); + } + + @Test + public void testMissingDescriptorReturnsNoMetadataAndClosesTable() throws Exception { + when(table.getDescriptor()).thenThrow(new TableNotFoundException(NAME)); + assertNull(plugin.planCacheTable(selection)); + verify(table).close(); + } + + @Test + public void testMissingTableHandleReturnsNoMetadata() throws Exception { + when(connection.getTable(NAME)).thenThrow(new TableNotFoundException(NAME)); + assertNull(plugin.planCacheTable(selection)); + verifyNoInteractions(table); + } + + @Test + public void testOtherIoFailuresPropagateAndCloseTable() throws Exception { + IOException failure = new IOException("descriptor read failed"); + when(table.getDescriptor()).thenThrow(failure); + assertSame(failure, assertThrows(IOException.class, () -> plugin.planCacheTable(selection))); + verify(table).close(); + } + + private TableDescriptor descriptor(String family) { + return TableDescriptorBuilder.newBuilder(NAME) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes(family))).build(); + } +} From b40a70d2c1085e836dbc430387376375ff4f4a3a Mon Sep 17 00:00:00 2001 From: Letian Jiang Date: Fri, 9 Oct 2026 01:00:24 +0800 Subject: [PATCH 16/16] Add plan cache integration coverage for HBase and Iceberg --- .../store/iceberg/IcebergPlanCacheTest.java | 249 ++++++++++++++++++ .../apache/drill/hbase/HBaseTestsSuite.java | 1 + .../drill/hbase/TestHBasePlanCache.java | 206 +++++++++++++++ .../planner/sql/TestPlanCachePlanning.java | 68 ++++- 4 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 contrib/format-iceberg/src/test/java/org/apache/drill/exec/store/iceberg/IcebergPlanCacheTest.java create mode 100644 contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCache.java diff --git a/contrib/format-iceberg/src/test/java/org/apache/drill/exec/store/iceberg/IcebergPlanCacheTest.java b/contrib/format-iceberg/src/test/java/org/apache/drill/exec/store/iceberg/IcebergPlanCacheTest.java new file mode 100644 index 00000000000..eb658650dc4 --- /dev/null +++ b/contrib/format-iceberg/src/test/java/org/apache/drill/exec/store/iceberg/IcebergPlanCacheTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.iceberg; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.apache.drill.common.config.DrillProperties; +import org.apache.drill.common.logical.FormatPluginConfig; +import org.apache.drill.exec.metrics.DrillMetrics; +import org.apache.drill.exec.planner.physical.PlannerSettings; +import org.apache.drill.exec.planner.sql.PlanCache; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.store.iceberg.format.IcebergFormatPluginConfig; +import org.apache.drill.test.ClientFixture; +import org.apache.drill.test.ClusterFixture; +import org.apache.drill.test.ClusterTest; +import org.apache.drill.test.TestBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.GenericAppenderFactory; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class IcebergPlanCacheTest extends ClusterTest { + private static FileSystemConfig config; + private final HadoopTables tables = new HadoopTables(new Configuration()); + private Table table; + private String name; + + @BeforeClass + public static void setupCluster() throws Exception { + startCluster(ClusterFixture.builder(dirTestWatcher)); + FileSystemConfig original = (FileSystemConfig) cluster.drillbit().getContext() + .getStorage().getPlugin("dfs").getConfig(); + Map formats = new HashMap<>(original.getFormats()); + formats.put("iceberg", IcebergFormatPluginConfig.builder().build()); + config = original.copyWithFormats(formats); + } + + @Before + public void setupTable() throws Exception { + cluster.drillbit().getContext().getStorage().put("dfs", config); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + cache().awaitWrites(); + cache().clear(); + name = "cache_" + UUID.randomUUID().toString().replace("-", ""); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "label", Types.StringType.get())); + table = tables.create(schema, dirTestWatcher.getDfsTestTmpDir().toPath().resolve(name).toString()); + append(1, "one", 2, "two", 3, "three"); + } + + @Test + public void testDifferentValuesAcrossConnectionsAndEmptyHit() throws Exception { + try (ClientFixture first = cluster.clientBuilder().property(DrillProperties.USER, "iceberg-cache").build(); + ClientFixture second = cluster.clientBuilder().property(DrillProperties.USER, "iceberg-cache").build()) { + String sql = pointSql(1); + first.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, false); + rows(first, sql, 1L, "one"); + first.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + second.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + rows(first, sql, 1L, "one"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(second, pointSql(2), 2L, "two"); + assertEquals(hits + 1, cache().getHitCount()); + // Keep the literal precision unchanged so this tests an empty cache hit. + rows(second, pointSql(9)); + assertEquals(hits + 2, cache().getHitCount()); + } + } + + @Test + public void testRangePruningRebuiltForNewValues() throws Exception { + rows(client, rangeSql(1, 2), 1L, "one", 2L, "two"); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + rows(client, rangeSql(1, 2), 1L, "one", 2L, "two"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(client, rangeSql(2, 3), 2L, "two", 3L, "three"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testCompatibleAppendVisibleOnHit() throws Exception { + warmPoint(); + append(4, "four"); + long hits = cache().getHitCount(); + rows(client, pointSql(4), 4L, "four"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testSchemaChangeInvalidatesAndReplans() throws Exception { + warmPoint(); + table.updateSchema().addColumn("extra", Types.StringType.get()).commit(); + append(4, "four"); + long hits = cache().getHitCount(); + long invalidations = invalidations(); + rows(client, pointSql(4), 4L, "four"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + assertEquals(invalidations + 1, invalidations()); + rows(client, pointSql(2), 2L, "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testReplacementAtSamePathInvalidatesAndReadsNewTable() throws Exception { + warmPoint(); + Schema schema = table.schema(); + String location = table.location(); + tables.dropTable(location, true); + table = tables.create(schema, location); + append(1, "replacement"); + long hits = cache().getHitCount(); + long invalidations = invalidations(); + rows(client, pointSql(1), 1L, "replacement"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + assertEquals(invalidations + 1, invalidations()); + rows(client, pointSql(1), 1L, "replacement"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testPluginConfigurationsKeepSeparateEntries() throws Exception { + warmPoint(); + Map formats = new HashMap<>(config.getFormats()); + formats.put("iceberg", IcebergFormatPluginConfig.builder().includeColumnStats(true).build()); + cluster.drillbit().getContext().getStorage().put("dfs", config.copyWithFormats(formats)); + long hits = cache().getHitCount(); + rows(client, pointSql(2), 2L, "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(client, pointSql(3), 3L, "three"); + assertEquals(hits + 1, cache().getHitCount()); + cluster.drillbit().getContext().getStorage().put("dfs", config); + rows(client, pointSql(1), 1L, "one"); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testJoinWithUnsupportedJsonBypassesCache() throws Exception { + String json = name + ".json"; + Files.write(dirTestWatcher.getDfsTestTmpDir().toPath().resolve(json), + Arrays.asList("{\"id\":1}", "{\"id\":2}"), StandardCharsets.UTF_8); + String sql = "SELECT t.id, t.label FROM dfs.tmp.`" + name + "` t JOIN dfs.tmp.`" + json + + "` j ON t.id = j.id WHERE t.id = %d"; + rows(client, String.format(sql, 1), 1L, "one"); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + long hits = cache().getHitCount(); + rows(client, String.format(sql, 1), 1L, "one"); + rows(client, String.format(sql, 2), 2L, "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + assertEquals(0L, (long) DrillMetrics.getRegistry().getGauges() + .get("drill.plan_cache.entries").getValue()); + } + + private void warmPoint() throws Exception { + rows(client, pointSql(1), 1L, "one"); + client.alterSession(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + rows(client, pointSql(1), 1L, "one"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(client, pointSql(2), 2L, "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + private String pointSql(long id) { + return "SELECT id, label FROM dfs.tmp.`" + name + "` WHERE id = " + id; + } + + private String rangeSql(long start, long end) { + return "SELECT id, label FROM dfs.tmp.`" + name + "` WHERE id >= " + start + " AND id <= " + end; + } + + private void rows(ClientFixture target, String sql, Object... values) throws Exception { + TestBuilder builder = target.testBuilder().sqlQuery(sql).unOrdered(); + if (values.length == 0) { + builder.expectsEmptyResultSet(); + } else { + builder.baselineColumns("id", "label"); + for (int i = 0; i < values.length; i += 2) { + builder.baselineValues(values[i], values[i + 1]); + } + } + builder.go(); + } + + private void append(Object... values) throws IOException { + OutputFile output = table.io().newOutputFile(table.location() + "/" + + FileFormat.PARQUET.addExtension(UUID.randomUUID().toString())); + FileAppender appender = new GenericAppenderFactory(table.schema()).newAppender(output, FileFormat.PARQUET); + try (FileAppender closeable = appender) { + for (int i = 0; i < values.length; i += 2) { + Record record = GenericRecord.create(table.schema()); + record.setField("id", ((Number) values[i]).longValue()); + record.setField("label", values[i + 1]); + closeable.add(record); + } + } + table.newAppend().appendFile(DataFiles.builder(table.spec()).withInputFile(output.toInputFile()) + .withMetrics(appender.metrics()).build()).commit(); + } + + private PlanCache cache() { + return cluster.drillbit().getContext().getPlanCache(); + } + + private long invalidations() { + return (Long) DrillMetrics.getRegistry().getGauges().get("drill.plan_cache.invalidations").getValue(); + } +} diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java index cd10e9d6497..60657781887 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java @@ -45,6 +45,7 @@ TestHBaseCFAsJSONString.class, TestHBaseConnectionManager.class, TestHBaseFilterPushDown.class, + TestHBasePlanCache.class, TestHBasePlanCacheMetadata.class, TestHBaseProjectPushDown.class, TestHBaseQueries.class, diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCache.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCache.java new file mode 100644 index 00000000000..ed2aef3c7d4 --- /dev/null +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBasePlanCache.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.hbase; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.apache.drill.categories.HbaseStorageTest; +import org.apache.drill.categories.SlowTest; +import org.apache.drill.exec.planner.sql.PlanCache; +import org.apache.drill.exec.store.hbase.HBaseStoragePluginConfig; +import org.apache.drill.test.TestBuilder; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category({SlowTest.class, HbaseStorageTest.class}) +public class TestHBasePlanCache extends BaseHBaseTest { + private TableName tableName; + + @Before + public void createTable() throws Exception { + test("ALTER SESSION SET `planner.enable_plan_cache` = false"); + cache().awaitWrites(); + cache().clear(); + tableName = TableName.valueOf("plan_cache_" + UUID.randomUUID().toString().replace("-", "")); + HBaseTestsSuite.getAdmin().createTable(TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build()); + put("a", "one"); + put("b", "two"); + put("c", "three"); + } + + @After + public void dropTable() throws Exception { + cache().awaitWrites(); + getDrillbitContext().getStorage().put(HBASE_STORAGE_PLUGIN_NAME, storagePluginConfig); + test("ALTER SESSION SET `planner.enable_plan_cache` = false"); + if (HBaseTestsSuite.getAdmin().tableExists(tableName)) { + HBaseTestsSuite.getAdmin().disableTable(tableName); + HBaseTestsSuite.getAdmin().deleteTable(tableName); + } + } + + @Test + public void testPointBoundsReboundAndEmptyHit() throws Exception { + rows(pointSql("a"), "a", "one"); + enableCache(); + long hits = cache().getHitCount(); + rows(pointSql("a"), "a", "one"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(pointSql("b"), "b", "two"); + rows(pointSql("z")); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testRangeBoundsRebuiltForNewValues() throws Exception { + rows(rangeSql("a", "b"), "a", "one", "b", "two"); + enableCache(); + rows(rangeSql("a", "b"), "a", "one", "b", "two"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(rangeSql("b", "c"), "b", "two", "c", "three"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testDataUpdatesVisibleOnHit() throws Exception { + warmPoint(); + put("a", "updated"); + put("d", "four"); + long hits = cache().getHitCount(); + rows(pointSql("a"), "a", "updated"); + rows(pointSql("d"), "d", "four"); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testSchemaChangeInvalidatesAndReplans() throws Exception { + warmPoint(); + HBaseTestsSuite.getAdmin().addColumnFamily(tableName, ColumnFamilyDescriptorBuilder.of("extra")); + long hits = cache().getHitCount(); + rows(pointSql("a"), "a", "one"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(pointSql("b"), "b", "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + @Test + public void testPluginConfigurationsKeepSeparateEntries() throws Exception { + warmPoint(); + Map properties = new HashMap<>(storagePluginConfig.getConfig()); + properties.put("hbase.client.operation.timeout", "90000"); + HBaseStoragePluginConfig changed = new HBaseStoragePluginConfig(properties, false); + changed.setEnabled(true); + getDrillbitContext().getStorage().put(HBASE_STORAGE_PLUGIN_NAME, changed); + long hits = cache().getHitCount(); + rows(pointSql("b"), "b", "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + rows(pointSql("c"), "c", "three"); + assertEquals(hits + 1, cache().getHitCount()); + getDrillbitContext().getStorage().put(HBASE_STORAGE_PLUGIN_NAME, storagePluginConfig); + rows(pointSql("a"), "a", "one"); + assertEquals(hits + 2, cache().getHitCount()); + } + + @Test + public void testJoinWithUnsupportedJsonBypassesCache() throws Exception { + String json = tableName.getNameAsString() + ".json"; + Files.write(dirTestWatcher.getDfsTestTmpDir().toPath().resolve(json), + Arrays.asList("{\"k\":\"a\"}", "{\"k\":\"b\"}"), StandardCharsets.UTF_8); + String sql = projection() + " JOIN dfs.tmp.`" + json + + "` j ON CONVERT_FROM(t.row_key, 'UTF8') = j.k WHERE t.row_key = '%s'"; + rows(String.format(sql, "a"), "a", "one"); + enableCache(); + long hits = cache().getHitCount(); + rows(String.format(sql, "a"), "a", "one"); + cache().awaitWrites(); + rows(String.format(sql, "a"), "a", "one"); + rows(String.format(sql, "b"), "b", "two"); + cache().awaitWrites(); + assertEquals(hits, cache().getHitCount()); + } + + private void warmPoint() throws Exception { + rows(pointSql("a"), "a", "one"); + enableCache(); + rows(pointSql("a"), "a", "one"); + cache().awaitWrites(); + long hits = cache().getHitCount(); + rows(pointSql("b"), "b", "two"); + assertEquals(hits + 1, cache().getHitCount()); + } + + private void enableCache() throws Exception { + test("ALTER SESSION SET `planner.enable_plan_cache` = true"); + } + + private String projection() { + return "SELECT CONVERT_FROM(t.row_key, 'UTF8') AS k, CONVERT_FROM(t.f.v, 'UTF8') AS val FROM hbase.`" + + tableName.getNameAsString() + "` t"; + } + + private String pointSql(String key) { + return projection() + " WHERE t.row_key = '" + key + "'"; + } + + private String rangeSql(String start, String end) { + return projection() + " WHERE t.row_key >= '" + start + "' AND t.row_key <= '" + end + "'"; + } + + private void rows(String sql, Object... values) throws Exception { + TestBuilder builder = testBuilder().sqlQuery(sql).unOrdered(); + if (values.length == 0) { + builder.expectsEmptyResultSet(); + } else { + builder.baselineColumns("k", "val"); + for (int i = 0; i < values.length; i += 2) { + builder.baselineValues(values[i], values[i + 1]); + } + } + builder.go(); + } + + private void put(String key, String value) throws Exception { + try (Table table = HBaseTestsSuite.getConnection().getTable(tableName)) { + table.put(new Put(Bytes.toBytes(key)).addColumn(Bytes.toBytes("f"), Bytes.toBytes("v"), Bytes.toBytes(value))); + } + } + + private PlanCache cache() { + return getDrillbitContext().getPlanCache(); + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java index 7003eb82270..85261a4c68f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/sql/TestPlanCachePlanning.java @@ -17,6 +17,8 @@ */ package org.apache.drill.exec.planner.sql; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -28,6 +30,9 @@ import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.apache.drill.exec.physical.rowSet.DirectRowSet; +import org.apache.drill.exec.physical.rowSet.RowSetReader; +import org.apache.drill.exec.planner.PhysicalPlanReader; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.conversion.SqlConverter; import org.apache.drill.exec.proto.UserBitShared.QueryId; @@ -43,8 +48,10 @@ import org.junit.experimental.categories.Category; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; @Category(SqlTest.class) public class TestPlanCachePlanning extends ClusterTest { @@ -255,4 +262,63 @@ public void testClearPreventsPublicationPendingQueryCompletion() throws Exceptio assertEquals(0L, metric("entries")); } } + + @Test + public void testBindingFailureInvalidatesAndReplansBeforeExecution() throws Exception { + UserSession session = UserSession.Builder.newBuilder() + .withCredentials(UserCredentials.newBuilder().setUserName("binding-failure-test").build()) + .withOptionManager(cluster.drillbit().getContext().getOptionManager()) + .setSupportComplexTypes(true).build(); + session.getOptions().setLocalOption(PlannerSettings.ENABLE_PLAN_CACHE_OPTION, true); + String sql = "SELECT x AS fallback_value FROM (VALUES (1), (2)) AS t(x) WHERE x > %d"; + PhysicalPlanReader reader = cluster.drillbit().getContext().getPlanReader(); + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan planned = DrillSqlWorker.getPlan(context, String.format(sql, 0), new Pointer<>()); + context.takePendingPlanCacheInsert(); + SqlConverter converter = new SqlConverter(context); + SqlNode parsed = converter.parse(String.format(sql, 0)); + PlanCache.ContextSnapshot snapshot = PlanCache.ContextSnapshot.resolve(converter.getDefaultSchema(), parsed, context); + PlanCacheParameterizer.Candidate candidate = PlanCacheParameterizer.parameterize(parsed, converter.getTypeFactory()); + String key = context.getQueryUserName() + '\n' + session.getDefaultSchemaPath() + '\n' + + snapshot.keyFingerprint() + '\n' + candidate.template; + String json = reader.writeJson(planned); + assertTrue(json.contains("bound_dynamic_param(0,")); + // Readable plan JSON with a nonexistent slot must fail binding before execution. + assertTrue(cache().put(key, json.replace("bound_dynamic_param(0,", "bound_dynamic_param(999,"), + null, reader, snapshot)); + } + + long hits = cache().getHitCount(); + long invalidations = metric("invalidations"); + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan replanned = DrillSqlWorker.getPlan(context, String.format(sql, 0), new Pointer<>()); + assertFalse(context.isPlanCacheHit()); + assertEquals(invalidations + 1, metric("invalidations")); + assertEquals(Arrays.asList(1, 2), executeValues(replanned)); + Runnable publish = context.takePendingPlanCacheInsert(); + assertNotNull(publish); + publish.run(); + cache().awaitWrites(); + } + try (QueryContext context = new QueryContext(session, cluster.drillbit().getContext(), QueryId.getDefaultInstance())) { + PhysicalPlan rebound = DrillSqlWorker.getPlan(context, String.format(sql, 1), new Pointer<>()); + assertTrue(context.isPlanCacheHit()); + assertEquals(Arrays.asList(2), executeValues(rebound)); + assertEquals(hits + 1, cache().getHitCount()); + } + } + + private List executeValues(PhysicalPlan plan) throws Exception { + DirectRowSet rows = client.queryBuilder().physical(cluster.drillbit().getContext().getPlanReader().writeJson(plan)).rowSet(); + try { + List values = new ArrayList<>(); + RowSetReader reader = rows.reader(); + while (reader.next()) { + values.add(reader.scalar(0).getInt()); + } + return values; + } finally { + rows.clear(); + } + } }