diff --git a/fe/fe-catalog/pom.xml b/fe/fe-catalog/pom.xml index 5f31e253c85657..73635affe41230 100644 --- a/fe/fe-catalog/pom.xml +++ b/fe/fe-catalog/pom.xml @@ -38,6 +38,11 @@ under the License. fe-common ${project.version} + + ${project.groupId} + fe-type + ${project.version} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateFunction.java similarity index 88% rename from fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java rename to fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateFunction.java index 05b3f60408c9dd..7b9b6e5332cba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateFunction.java @@ -17,8 +17,7 @@ package org.apache.doris.catalog; -import org.apache.doris.common.util.URI; -import org.apache.doris.thrift.TFunctionBinaryType; +import org.apache.doris.common.URI; import com.google.gson.annotations.SerializedName; @@ -77,7 +76,7 @@ public AggregateFunction(FunctionName fnName, List argTypes, URI location, String updateFnSymbol, String initFnSymbol, String serializeFnSymbol, String mergeFnSymbol, String getValueFnSymbol, String removeFnSymbol, String finalizeFnSymbol, boolean ignoresDistinct, - boolean isAnalyticFn, boolean returnsNonNullOnEmpty, TFunctionBinaryType binaryType, + boolean isAnalyticFn, Function.BinaryType binaryType, boolean userVisible, boolean vectorized, NullableMode nullableMode) { // only `count` is always not nullable, other aggregate function is always nullable super(0, fnName, argTypes, retType, hasVarArgs, binaryType, userVisible, vectorized, nullableMode); @@ -99,8 +98,17 @@ public AggregateFunction(AggregateFunction other) { if (other == null) { return; } - isAnalyticFn = other.isAnalyticFn; - isAggregateFn = other.isAggregateFn; + this.isAnalyticFn = other.isAnalyticFn; + this.isAggregateFn = other.isAggregateFn; + this.updateFnSymbol = other.updateFnSymbol; + this.initFnSymbol = other.initFnSymbol; + this.serializeFnSymbol = other.serializeFnSymbol; + this.mergeFnSymbol = other.mergeFnSymbol; + this.getValueFnSymbol = other.getValueFnSymbol; + this.removeFnSymbol = other.removeFnSymbol; + this.finalizeFnSymbol = other.finalizeFnSymbol; + this.intermediateType = other.intermediateType; + this.symbolName = other.symbolName; } @Override @@ -129,7 +137,7 @@ public AggregateFunction(FunctionName fnName, Type[] argTypes, } public static class AggregateFunctionBuilder { - TFunctionBinaryType binaryType; + Function.BinaryType binaryType; FunctionName name; Type[] argTypes; Type retType; @@ -145,12 +153,12 @@ public static class AggregateFunctionBuilder { String getValueFnSymbol; String symbolName; - private AggregateFunctionBuilder(TFunctionBinaryType binaryType) { + private AggregateFunctionBuilder(Function.BinaryType binaryType) { this.binaryType = binaryType; } public static AggregateFunctionBuilder createUdfBuilder() { - return new AggregateFunctionBuilder(TFunctionBinaryType.JAVA_UDF); + return new AggregateFunctionBuilder(Function.BinaryType.JAVA_UDF); } public AggregateFunctionBuilder name(FunctionName name) { @@ -218,11 +226,6 @@ public AggregateFunctionBuilder removeFnSymbol(String symbol) { return this; } - public AggregateFunctionBuilder binaryType(TFunctionBinaryType binaryType) { - this.binaryType = binaryType; - return this; - } - public AggregateFunctionBuilder symbolName(String symbol) { this.symbolName = symbol; return this; @@ -291,34 +294,6 @@ public Type getIntermediateType() { return intermediateType; } - public void setUpdateFnSymbol(String fn) { - updateFnSymbol = fn; - } - - public void setInitFnSymbol(String fn) { - initFnSymbol = fn; - } - - public void setSerializeFnSymbol(String fn) { - serializeFnSymbol = fn; - } - - public void setMergeFnSymbol(String fn) { - mergeFnSymbol = fn; - } - - public void setGetValueFnSymbol(String fn) { - getValueFnSymbol = fn; - } - - public void setRemoveFnSymbol(String fn) { - removeFnSymbol = fn; - } - - public void setFinalizeFnSymbol(String fn) { - finalizeFnSymbol = fn; - } - public void setSymbolName(String fn) { symbolName = fn; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java similarity index 91% rename from fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java rename to fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java index 4865db349b899d..9981d63222588a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java @@ -17,15 +17,13 @@ package org.apache.doris.catalog; +import org.apache.doris.common.URI; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.common.util.URI; -import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.thrift.TFunctionBinaryType; +import org.apache.doris.persist.gson.GsonUtilsBase; import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; import com.google.gson.annotations.SerializedName; import org.apache.commons.io.output.NullOutputStream; @@ -52,6 +50,17 @@ public enum NullableMode { ALWAYS_NOT_NULLABLE } + public enum BinaryType { + BUILTIN, + HIVE, + NATIVE, + IR, + RPC, + JAVA_UDF, + AGG_STATE, + PYTHON_UDF + } + // Function id, every function has a unique id. Now all built-in functions' id is 0 @SerializedName("id") private long id = 0; @@ -79,7 +88,7 @@ public enum NullableMode { @SerializedName("l") private URI location; @SerializedName("bt") - private TFunctionBinaryType binaryType; + private BinaryType binaryType; @SerializedName("nm") protected NullableMode nullableMode = NullableMode.DEPEND_ON_ARGUMENT; @@ -119,12 +128,12 @@ public Function(FunctionName name, List args, Type retType, } public Function(long id, FunctionName name, List argTypes, Type retType, boolean hasVarArgs, - TFunctionBinaryType binaryType, boolean userVisible, boolean vectorized, NullableMode mode) { + Function.BinaryType binaryType, boolean userVisible, boolean vectorized, NullableMode mode) { this.id = id; this.name = name; this.hasVarArgs = hasVarArgs; - if (argTypes.size() > 0) { - this.argTypes = argTypes.toArray(new Type[argTypes.size()]); + if (!argTypes.isEmpty()) { + this.argTypes = argTypes.toArray(new Type[0]); } else { this.argTypes = new Type[0]; } @@ -137,7 +146,7 @@ public Function(long id, FunctionName name, List argTypes, Type retType, b public Function(long id, FunctionName name, List argTypes, Type retType, boolean hasVarArgs, boolean vectorized, NullableMode mode) { - this(id, name, argTypes, retType, hasVarArgs, TFunctionBinaryType.BUILTIN, true, vectorized, mode); + this(id, name, argTypes, retType, hasVarArgs, BinaryType.BUILTIN, true, vectorized, mode); } public Function(Function other) { @@ -195,7 +204,7 @@ public Type[] getArgs() { } public void setArgs(List argTypes) { - this.argTypes = argTypes.toArray(new Type[argTypes.size()]); + this.argTypes = argTypes.toArray(new Type[0]); } // Returns the number of arguments to this function. @@ -215,11 +224,11 @@ public void setName(FunctionName name) { this.name = name; } - public TFunctionBinaryType getBinaryType() { + public Function.BinaryType getBinaryType() { return binaryType; } - public void setBinaryType(TFunctionBinaryType type) { + public void setBinaryType(Function.BinaryType type) { binaryType = type; } @@ -235,14 +244,6 @@ public void setUserVisible(boolean userVisible) { this.userVisible = userVisible; } - public Type getVarArgsType() { - if (!hasVarArgs) { - return Type.INVALID; - } - Preconditions.checkState(argTypes.length > 0); - return argTypes[argTypes.length - 1]; - } - public void setHasVarArgs(boolean v) { hasVarArgs = v; } @@ -330,11 +331,11 @@ public boolean isIdentical(Function o) { @Override public void write(DataOutput output) throws IOException { - Text.writeString(output, GsonUtils.GSON.toJson(this)); + Text.writeString(output, GsonUtilsBase.GSON.toJson(this)); } public static Function read(DataInput input) throws IOException { - return GsonUtils.GSON.fromJson(Text.readString(input), Function.class); + return GsonUtilsBase.GSON.fromJson(Text.readString(input), Function.class); } public void setNullableMode(NullableMode nullableMode) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionName.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/FunctionName.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionName.java rename to fe/fe-catalog/src/main/java/org/apache/doris/catalog/FunctionName.java diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ScalarFunction.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ScalarFunction.java new file mode 100644 index 00000000000000..83c48ea3fc5725 --- /dev/null +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ScalarFunction.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.URI; + +import com.google.gson.annotations.SerializedName; + +import java.util.Arrays; +import java.util.List; + +// import org.apache.doris.thrift.TSymbolType; + +/** + * Internal representation of a scalar function. + */ +public class ScalarFunction extends Function { + // The name inside the binary at location_ that contains this particular + // function. e.g. org.example.MyUdf.class. + @SerializedName("sn") + private String symbolName; + @SerializedName("pfs") + private String prepareFnSymbol; + @SerializedName("cfs") + private String closeFnSymbol; + + DictFunction dictFunction = null; + + public static class DictFunction { + private final long id; + private final long version; + + public DictFunction(long id, long version) { + this.id = id; + this.version = version; + } + + public long getId() { + return id; + } + + public long getVersion() { + return version; + } + } + + // Only used for serialization + protected ScalarFunction() { + } + + public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, + boolean userVisible) { + this(fnName, argTypes, retType, hasVarArgs, Function.BinaryType.BUILTIN, userVisible, true); + } + + private ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, + Function.BinaryType binaryType, boolean userVisible, boolean isVec) { + super(0, fnName, argTypes, retType, hasVarArgs, binaryType, userVisible, isVec, + NullableMode.DEPEND_ON_ARGUMENT); + } + + /** + * nerieds custom scalar function + */ + public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, String symbolName, + BinaryType binaryType, boolean userVisible, boolean isVec, NullableMode nullableMode) { + super(0, fnName, argTypes, retType, hasVarArgs, binaryType, userVisible, isVec, nullableMode); + this.symbolName = symbolName; + } + + public static ScalarFunction createUdf( + Function.BinaryType binaryType, + FunctionName name, Type[] args, + Type returnType, boolean isVariadic, + URI location, String symbol, String prepareFnSymbol, String closeFnSymbol) { + ScalarFunction fn = new ScalarFunction(name, Arrays.asList(args), returnType, isVariadic, binaryType, + true, false); + fn.symbolName = symbol; + fn.prepareFnSymbol = prepareFnSymbol; + fn.closeFnSymbol = closeFnSymbol; + fn.setLocation(location); + return fn; + } + + public ScalarFunction(ScalarFunction other) { + super(other); + if (other == null) { + return; + } + symbolName = other.symbolName; + prepareFnSymbol = other.prepareFnSymbol; + closeFnSymbol = other.closeFnSymbol; + dictFunction = other.dictFunction; + } + + @Override + public Function clone() { + return new ScalarFunction(this); + } + + public String getSymbolName() { + return symbolName; + } + + public String getPrepareFnSymbol() { + return prepareFnSymbol; + } + + public String getCloseFnSymbol() { + return closeFnSymbol; + } + + public DictFunction getDictFunction() { + return dictFunction; + } + + public void setDictFunction(DictFunction dictFunction) { + this.dictFunction = dictFunction; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Id.java b/fe/fe-common/src/main/java/org/apache/doris/common/Id.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/Id.java rename to fe/fe-common/src/main/java/org/apache/doris/common/Id.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/IdGenerator.java b/fe/fe-common/src/main/java/org/apache/doris/common/IdGenerator.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/IdGenerator.java rename to fe/fe-common/src/main/java/org/apache/doris/common/IdGenerator.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java b/fe/fe-common/src/main/java/org/apache/doris/common/TreeNode.java similarity index 79% rename from fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java rename to fe/fe-common/src/main/java/org/apache/doris/common/TreeNode.java index 8e14d6e10ae0e7..9d883ffe8e9e5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/TreeNode.java @@ -43,10 +43,6 @@ public void addChild(NodeType n) { children.add(n); } - public void addChildren(List n) { - children.addAll(n); - } - public boolean hasChild(int i) { return children.size() > i; } @@ -59,28 +55,6 @@ public ArrayList getChildren() { return children; } - public void clearChildren() { - children.clear(); - } - - public void removeNode(int i) { - if (children != null && i >= 0 && i < children.size()) { - children.remove(i); - } - } - - /** - * Count the total number of nodes in this tree. Leaf node will return 1. - * Non-leaf node will include all its children. - */ - public int numNodes() { - int numNodes = 1; - for (NodeType child : children) { - numNodes += child.numNodes(); - } - return numNodes; - } - /** * Add all nodes in the tree that satisfy 'predicate' to the list 'matches' * This node is checked first, followed by its children in order. If the node @@ -210,46 +184,6 @@ public static > boolean contains( return false; } - public boolean containsSubclass(Class cl) { - if (cl.isAssignableFrom(this.getClass())) { - return true; - } - for (NodeType child : children) { - if (child.containsSubclass(cl)) { - return true; - } - } - return false; - } - - /** - * Return 'this' or first child that is exactly of class 'cl'. - * Looks for matching children via depth-first, left-to-right traversal. - */ - public C findFirstOf(Class cl) { - if (this.getClass().equals(cl)) { - return (C) this; - } - for (NodeType child : children) { - NodeType result = child.findFirstOf(cl); - if (result != null) { - return (C) result; - } - } - return null; - } - - public interface ThrowingConsumer { - void accept(T t) throws AnalysisException; - } - - public void foreach(ThrowingConsumer> func) throws AnalysisException { - func.accept(this); - for (NodeType child : getChildren()) { - child.foreach(func); - } - } - /** anyMatch */ public boolean anyMatch(Predicate> func) { if (func.apply(this)) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/util/URI.java b/fe/fe-common/src/main/java/org/apache/doris/common/URI.java similarity index 98% rename from fe/fe-common/src/main/java/org/apache/doris/common/util/URI.java rename to fe/fe-common/src/main/java/org/apache/doris/common/URI.java index 71ba4a069d88e6..ab0184228bd557 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/util/URI.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/URI.java @@ -15,9 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.common.util; - -import org.apache.doris.common.AnalysisException; +package org.apache.doris.common; import com.google.common.base.Strings; import com.google.gson.annotations.SerializedName; diff --git a/fe/fe-common/src/main/java/org/apache/doris/persist/gson/GsonUtilsBase.java b/fe/fe-common/src/main/java/org/apache/doris/persist/gson/GsonUtilsBase.java new file mode 100644 index 00000000000000..a41065f77a92be --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/persist/gson/GsonUtilsBase.java @@ -0,0 +1,408 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist.gson; + +import org.apache.doris.common.io.Text; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Table; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import com.google.gson.ReflectionAccessFilter; +import com.google.gson.ToNumberPolicy; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.annotations.SerializedName; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import org.apache.commons.lang3.reflect.TypeUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class GsonUtilsBase { + + // the builder of GSON instance. + // Add any other adapters if necessary. + // + // ATTN: + // Since GsonBuilder.create() adds all registered factories to GSON in reverse order, if you + // need to ensure the search order of two RuntimeTypeAdapterFactory instances, be sure to + // register them in reverse priority order. + public static final GsonBuilder GSON_BUILDER = new GsonBuilder() + .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) + .addSerializationExclusionStrategy( + new HiddenAnnotationExclusionStrategy()).serializeSpecialFloatingPointValues() + .enableComplexMapKeySerialization() + .addReflectionAccessFilter(ReflectionAccessFilter.BLOCK_INACCESSIBLE_JAVA) + .registerTypeHierarchyAdapter(Table.class, new GuavaTableAdapter()) + .registerTypeHierarchyAdapter(Multimap.class, new GuavaMultimapAdapter()) + .registerTypeAdapterFactory(new PostProcessTypeAdapterFactory()) + .registerTypeAdapterFactory(new PreProcessTypeAdapterFactory()) + .registerTypeAdapter(ImmutableMap.class, new ImmutableMapDeserializer()) + .registerTypeAdapter(ImmutableList.class, new ImmutableListDeserializer()) + .registerTypeAdapter(AtomicBoolean.class, new AtomicBooleanAdapter()); + + + /* + * The exclusion strategy of GSON serialization. + * Any fields without "@SerializedName" annotation with be ignore with + * serializing and deserializing. + */ + public static class HiddenAnnotationExclusionStrategy implements ExclusionStrategy { + public boolean shouldSkipField(FieldAttributes f) { + return f.getAnnotation(SerializedName.class) == null; + } + + @Override + public boolean shouldSkipClass(Class clazz) { + return false; + } + } + + /* + * + * The json adapter for Guava Table. + * Current support: + * 1. HashBasedTable + * + * The RowKey, ColumnKey and Value classes in Table should also be serializable. + * + * What is Adapter and Why we should implement it? + * + * Adapter is mainly used to provide serialization and deserialization methods for some complex classes. + * Complex classes here usually refer to classes that are complex and cannot be modified. + * These classes mainly include third-party library classes or some inherited classes. + */ + private static class GuavaTableAdapter + implements JsonSerializer>, JsonDeserializer> { + /* + * serialize Table as: + * { + * "rowKeys": [ "rowKey1", "rowKey2", ...], + * "columnKeys": [ "colKey1", "colKey2", ...], + * "cells" : [[0, 0, value1], [0, 1, value2], ...] + * } + * + * the [0, 0] .. in cells are the indexes of rowKeys array and columnKeys array. + * This serialization method can reduce the size of json string because it + * replace the same row key + * and column key to integer. + */ + @Override + public JsonElement serialize(Table src, Type typeOfSrc, JsonSerializationContext context) { + JsonArray rowKeysJsonArray = new JsonArray(); + Map rowKeyToIndex = new HashMap<>(); + for (R rowKey : src.rowKeySet()) { + rowKeyToIndex.put(rowKey, rowKeyToIndex.size()); + rowKeysJsonArray.add(context.serialize(rowKey)); + } + JsonArray columnKeysJsonArray = new JsonArray(); + Map columnKeyToIndex = new HashMap<>(); + for (C columnKey : src.columnKeySet()) { + columnKeyToIndex.put(columnKey, columnKeyToIndex.size()); + columnKeysJsonArray.add(context.serialize(columnKey)); + } + JsonArray cellsJsonArray = new JsonArray(); + for (Table.Cell cell : src.cellSet()) { + int rowIndex = rowKeyToIndex.get(cell.getRowKey()); + int columnIndex = columnKeyToIndex.get(cell.getColumnKey()); + cellsJsonArray.add(rowIndex); + cellsJsonArray.add(columnIndex); + cellsJsonArray.add(context.serialize(cell.getValue())); + } + JsonObject tableJsonObject = new JsonObject(); + tableJsonObject.addProperty("clazz", src.getClass().getSimpleName()); + tableJsonObject.add("rowKeys", rowKeysJsonArray); + tableJsonObject.add("columnKeys", columnKeysJsonArray); + tableJsonObject.add("cells", cellsJsonArray); + return tableJsonObject; + } + + @Override + public Table deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) { + Type typeOfR; + Type typeOfC; + Type typeOfV; + { // CHECKSTYLE IGNORE THIS LINE + ParameterizedType parameterizedType = (ParameterizedType) typeOfT; + Type[] actualTypeArguments = parameterizedType.getActualTypeArguments(); + typeOfR = actualTypeArguments[0]; + typeOfC = actualTypeArguments[1]; + typeOfV = actualTypeArguments[2]; + } // CHECKSTYLE IGNORE THIS LINE + JsonObject tableJsonObject = json.getAsJsonObject(); + String tableClazz = tableJsonObject.get("clazz").getAsString(); + JsonArray rowKeysJsonArray = tableJsonObject.getAsJsonArray("rowKeys"); + Map rowIndexToKey = new HashMap<>(); + for (JsonElement jsonElement : rowKeysJsonArray) { + R rowKey = context.deserialize(jsonElement, typeOfR); + rowIndexToKey.put(rowIndexToKey.size(), rowKey); + } + JsonArray columnKeysJsonArray = tableJsonObject.getAsJsonArray("columnKeys"); + Map columnIndexToKey = new HashMap<>(); + for (JsonElement jsonElement : columnKeysJsonArray) { + C columnKey = context.deserialize(jsonElement, typeOfC); + columnIndexToKey.put(columnIndexToKey.size(), columnKey); + } + JsonArray cellsJsonArray = tableJsonObject.getAsJsonArray("cells"); + Table table = null; + switch (tableClazz) { + case "HashBasedTable": + table = HashBasedTable.create(); + break; + default: + Preconditions.checkState(false, "unknown guava table class: " + tableClazz); + break; + } + for (int i = 0; i < cellsJsonArray.size(); i = i + 3) { + // format is [rowIndex, columnIndex, value] + int rowIndex = cellsJsonArray.get(i).getAsInt(); + int columnIndex = cellsJsonArray.get(i + 1).getAsInt(); + R rowKey = rowIndexToKey.get(rowIndex); + C columnKey = columnIndexToKey.get(columnIndex); + V value = context.deserialize(cellsJsonArray.get(i + 2), typeOfV); + table.put(rowKey, columnKey, value); + } + return table; + } + } + + /* + * The json adapter for Guava Multimap. + * Current support: + * 1. ArrayListMultimap + * 2. HashMultimap + * 3. LinkedListMultimap + * 4. LinkedHashMultimap + * + * The key and value classes of multi map should also be json serializable. + */ + private static class GuavaMultimapAdapter + implements JsonSerializer>, JsonDeserializer> { + + private static final Type asMapReturnType = getAsMapMethod().getGenericReturnType(); + + private static Type asMapType(Type multimapType) { + return com.google.common.reflect.TypeToken.of(multimapType).resolveType(asMapReturnType).getType(); + } + + private static Method getAsMapMethod() { + try { + return Multimap.class.getDeclaredMethod("asMap"); + } catch (NoSuchMethodException e) { + throw new AssertionError(e); + } + } + + @Override + public JsonElement serialize(Multimap map, Type typeOfSrc, JsonSerializationContext context) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("clazz", map.getClass().getSimpleName()); + Map> asMap = map.asMap(); + Type type = asMapType(typeOfSrc); + JsonElement jsonElement = context.serialize(asMap, type); + jsonObject.add("map", jsonElement); + return jsonObject; + } + + @Override + public Multimap deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + JsonObject jsonObject = json.getAsJsonObject(); + String clazz = jsonObject.get("clazz").getAsString(); + + JsonElement mapElement = jsonObject.get("map"); + Map> asMap = context.deserialize(mapElement, asMapType(typeOfT)); + + Multimap map = null; + switch (clazz) { + case "ArrayListMultimap": + map = ArrayListMultimap.create(); + break; + case "HashMultimap": + map = HashMultimap.create(); + break; + case "LinkedListMultimap": + map = LinkedListMultimap.create(); + break; + case "LinkedHashMultimap": + map = LinkedHashMultimap.create(); + break; + default: + Preconditions.checkState(false, "unknown guava multi map class: " + clazz); + break; + } + + for (Map.Entry> entry : asMap.entrySet()) { + map.putAll(entry.getKey(), entry.getValue()); + } + return map; + } + } + + private static class AtomicBooleanAdapter + implements JsonSerializer, JsonDeserializer { + + @Override + public AtomicBoolean deserialize(JsonElement jsonElement, Type type, + JsonDeserializationContext jsonDeserializationContext) + throws JsonParseException { + JsonObject jsonObject = jsonElement.getAsJsonObject(); + boolean value = jsonObject.get("boolean").getAsBoolean(); + return new AtomicBoolean(value); + } + + @Override + public JsonElement serialize(AtomicBoolean atomicBoolean, Type type, + JsonSerializationContext jsonSerializationContext) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("boolean", atomicBoolean.get()); + return jsonObject; + } + } + + private static final class ImmutableMapDeserializer implements JsonDeserializer> { + @Override + public ImmutableMap deserialize(final JsonElement json, final Type type, + final JsonDeserializationContext context) throws JsonParseException { + final Type type2 = TypeUtils.parameterize(Map.class, ((ParameterizedType) type).getActualTypeArguments()); + final Map map = context.deserialize(json, type2); + return ImmutableMap.copyOf(map); + } + } + + private static final class ImmutableListDeserializer implements JsonDeserializer> { + @Override + public ImmutableList deserialize(final JsonElement json, final Type type, + final JsonDeserializationContext context) throws JsonParseException { + final Type type2 = TypeUtils.parameterize(List.class, ((ParameterizedType) type).getActualTypeArguments()); + final List list = context.deserialize(json, type2); + return ImmutableList.copyOf(list); + } + } + + public static class PreProcessTypeAdapterFactory implements TypeAdapterFactory { + + public PreProcessTypeAdapterFactory() { + } + + @Override + public TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter delegate = gson.getDelegateAdapter(this, type); + + return new TypeAdapter() { + public void write(JsonWriter out, T value) throws IOException { + if (value instanceof GsonPreProcessable) { + ((GsonPreProcessable) value).gsonPreProcess(); + } + delegate.write(out, value); + } + + public T read(JsonReader reader) throws IOException { + return delegate.read(reader); + } + }; + } + } + + public static class PostProcessTypeAdapterFactory implements TypeAdapterFactory { + + public PostProcessTypeAdapterFactory() { + } + + @Override + public TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter delegate = gson.getDelegateAdapter(this, type); + + return new TypeAdapter() { + public void write(JsonWriter out, T value) throws IOException { + delegate.write(out, value); + } + + public T read(JsonReader reader) throws IOException { + T obj = delegate.read(reader); + if (obj instanceof GsonPostProcessable) { + ((GsonPostProcessable) obj).gsonPostProcess(); + } + return obj; + } + }; + } + } + + // this instance is thread-safe. + public static final Gson GSON = GSON_BUILDER.create(); + + public static void toJsonCompressed(Gson gson, DataOutput out, Object src) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { + try (OutputStreamWriter writer = new OutputStreamWriter(gzipStream)) { + gson.toJson(src, writer); + } + } + Text text = new Text(byteStream.toByteArray()); + text.write(out); + } + + public static T fromJsonCompressed(Gson gson, DataInput in, Class clazz) throws IOException { + Text text = new Text(); + text.readFields(in); + + ByteArrayInputStream byteStream = new ByteArrayInputStream(text.getBytes()); + try (GZIPInputStream gzipStream = new GZIPInputStream(byteStream)) { + try (InputStreamReader reader = new InputStreamReader(gzipStream)) { + return gson.fromJson(reader, clazz); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 818cd47ac82cfd..53b61c17a2f734 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -3537,7 +3537,7 @@ public void buildOrDeleteTableInvertedIndices(Database db, OlapTable olapTable, List colList = indexMeta.getSchema(true); for (Column col : colList) { TColumn tColumn = col.toThrift(); - col.setIndexFlag(tColumn, olapTable); + col.setIndexFlag(tColumn, olapTable.getCopiedBfColumns()); } List indexList = indexMeta.getIndexes(); int schemaVersion = indexMeta.getSchemaVersion(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 46cff0a9657689..bd65bc18349bd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -20,7 +20,6 @@ package org.apache.doris.analysis; -import org.apache.doris.catalog.AggStateType; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -38,7 +37,6 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -181,16 +179,6 @@ public static void collectList(List input, Clas } } - public static void extractSlots(Expr root, Set slotIdSet) { - if (root instanceof SlotRef) { - slotIdSet.add(((SlotRef) root).getDesc().getId()); - return; - } - for (Expr child : root.getChildren()) { - extractSlots(child, slotIdSet); - } - } - /** * Accept a visitor and dispatch to the appropriate typed {@code visitXxx} method. * Each concrete subclass must override this to call the correct visitor method. @@ -249,29 +237,6 @@ public int hashCode() { return result; } - /** - * Gather conjuncts from this expr and return them in a list. - * A conjunct is an expr that returns a boolean, e.g., Predicates, function calls, - * SlotRefs, etc. Hence, this method is placed here and not in Predicate. - */ - public List getConjuncts() { - List list = Lists.newArrayList(); - if (this instanceof CompoundPredicate - && ((CompoundPredicate) this).getOp() == CompoundPredicate.Operator.AND) { - // TODO: we have to convert CompoundPredicate.AND to two expr trees for - // conjuncts because NULLs are handled differently for CompoundPredicate.AND - // and conjunct evaluation. This is not optimal for jitted exprs because it - // will result in two functions instead of one. Create a new CompoundPredicate - // Operator (i.e. CONJUNCT_AND) with the right NULL semantics and use that - // instead - list.addAll((getChild(0)).getConjuncts()); - list.addAll((getChild(1)).getConjuncts()); - } else { - list.add(this); - } - return list; - } - /** * Create a deep copy of 'this'. If sMap is non-null, * use it to substitute 'this' or its subnodes. @@ -286,10 +251,10 @@ public Expr clone(ExprSubstitutionMap sMap) { } } } - Expr result = (Expr) this.clone(); + Expr result = this.clone(); result.children = Lists.newArrayList(); for (Expr child : children) { - result.children.add(((Expr) child).clone(sMap)); + result.children.add(child.clone(sMap)); } return result; } @@ -306,6 +271,7 @@ public boolean isBound(SlotId slotId) { return true; } + @Deprecated public Map> getTableIdToColumnNames() { Map> tableIdToColumnNames = new HashMap>(); getTableIdToColumnNames(tableIdToColumnNames); @@ -362,31 +328,6 @@ public String toString() { return MoreObjects.toStringHelper(this.getClass()).add("type", type).toString(); } - /** - * If 'this' is a SlotRef or a Cast that wraps a SlotRef, returns that SlotRef. - * Otherwise returns null. - */ - public SlotRef unwrapSlotRef() { - if (this instanceof SlotRef) { - return (SlotRef) this; - } else if (this instanceof CastExpr && getChild(0) instanceof SlotRef) { - return (SlotRef) getChild(0); - } else { - return null; - } - } - - /** - * Returns the first child if this Expr is a CastExpr. Otherwise, returns 'this'. - */ - public Expr unwrapExpr(boolean implicitOnly) { - if (this instanceof CastExpr - && (!implicitOnly || ((CastExpr) this).isImplicit())) { - return children.get(0); - } - return this; - } - public String getStringValue() { return ""; } @@ -398,11 +339,6 @@ public boolean isNullable() { return nullable; } - public static AggStateType createAggStateType(String name, List typeList, - List nullableList, boolean resultNullable) { - return new AggStateType(name, resultNullable, typeList, nullableList); - } - // This is only for transactional insert operation, // to check it the given value in insert stmt is LiteralExpr. // And if we write "1" to a boolean column, there will be a cast(1 as boolean) expr, @@ -414,21 +350,4 @@ public boolean isLiteralOrCastExpr() { return this instanceof LiteralExpr; } } - - public boolean isNullLiteral() { - return this instanceof NullLiteral; - } - - public Set getInputSlotRef() { - Set slots = new HashSet<>(); - if (this instanceof SlotRef) { - slots.add((SlotRef) this); - return slots; - } else { - for (Expr expr : children) { - slots.addAll(expr.getInputSlotRef()); - } - } - return slots; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprId.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprId.java deleted file mode 100644 index 2c3e64941dd8b2..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprId.java +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// This file is copied from -// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/ExprId.java -// and modified by Doris - -package org.apache.doris.analysis; - -import org.apache.doris.common.Id; -import org.apache.doris.common.IdGenerator; - -public class ExprId extends Id { - // Construction only allowed via an IdGenerator. - protected ExprId(int id) { - super(id); - } - - public static IdGenerator createGenerator() { - return new IdGenerator() { - @Override - public ExprId getNextId() { - return new ExprId(nextId++); - } - }; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java index 97128ef9f656e6..8023f822334aef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToExternalSqlVisitor.java @@ -68,7 +68,7 @@ public String visitSlotRef(SlotRef expr, ToSqlParams context) { return expr.getCol(); } } else { - return ""; + return ""; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java index 3bbbbc902cc9c0..fbbe328ebb6af4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java @@ -175,7 +175,7 @@ public String visitSlotRef(SlotRef expr, ToSqlParams context) { && !ConnectContext.get().getState().isQuery() && ConnectContext.get().getSessionVariable() != null && expr.desc != null) { - return expr.getLabel() + "[#" + expr.desc.getId().asInt() + "]"; + return expr.getLabel() + "[#" + expr.getSlotId().asInt() + "]"; } else { return expr.getLabel(); } @@ -197,7 +197,7 @@ public String visitSlotRef(SlotRef expr, ToSqlParams context) { } return sb.toString(); } else { - return "" + sb; + return "" + sb; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java index 7161417cc3e8e8..5bb078364ed103 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java @@ -26,6 +26,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TAggregateExpr; import org.apache.doris.thrift.TBoolLiteral; import org.apache.doris.thrift.TCaseExpr; import org.apache.doris.thrift.TColumnRef; @@ -290,7 +291,7 @@ public Void visitPlaceHolderExpr(PlaceHolderExpr expr, TExprNode msg) { @Override public Void visitSlotRef(SlotRef expr, TExprNode msg) { msg.node_type = TExprNodeType.SLOT_REF; - msg.slot_ref = new TSlotRef(expr.getDesc().getId().asInt(), expr.getDesc().getParent().getId().asInt()); + msg.slot_ref = new TSlotRef(expr.getSlotId().asInt(), expr.getDesc().getParent().getId().asInt()); msg.slot_ref.setColUniqueId(expr.getDesc().getUniqueId()); msg.slot_ref.setIsVirtualSlot(expr.getDesc().getVirtualColumn() != null); msg.setLabel(expr.getLabel()); @@ -493,7 +494,7 @@ public Void visitFunctionCallExpr(FunctionCallExpr expr, TExprNode msg) { if (aggParams == null) { aggParams = expr.getFnParams(); } - msg.setAggExpr(aggParams.createTAggregateExpr(expr.isMergeAggFn())); + msg.setAggExpr(createTAggregateExpr(aggParams, expr.isMergeAggFn())); } else { msg.node_type = TExprNodeType.FUNCTION_CALL; } @@ -504,6 +505,20 @@ public Void visitFunctionCallExpr(FunctionCallExpr expr, TExprNode msg) { return null; } + private TAggregateExpr createTAggregateExpr(FunctionParams functionParams, boolean isMergeAggFn) { + List paramTypes = new ArrayList<>(); + if (functionParams.exprs() != null) { + for (Expr expr : functionParams.exprs()) { + TTypeDesc desc = expr.getType().toThrift(); + desc.setIsNullable(expr.isNullable()); + paramTypes.add(desc); + } + } + TAggregateExpr aggExpr = new TAggregateExpr(isMergeAggFn); + aggExpr.setParamTypes(paramTypes); + return aggExpr; + } + @Override public Void visitLambdaFunctionCallExpr(LambdaFunctionCallExpr expr, TExprNode msg) { msg.node_type = TExprNodeType.LAMBDA_FUNCTION_CALL_EXPR; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionParams.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionParams.java index 83c293eef91bb5..9ea4972309b22e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionParams.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionParams.java @@ -20,13 +20,9 @@ package org.apache.doris.analysis; -import org.apache.doris.thrift.TAggregateExpr; -import org.apache.doris.thrift.TTypeDesc; - import com.google.common.base.Preconditions; import com.google.gson.annotations.SerializedName; -import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -74,20 +70,6 @@ public FunctionParams clone(List children) { return new FunctionParams(isDistinct(), children); } - public TAggregateExpr createTAggregateExpr(boolean isMergeAggFn) { - List paramTypes = new ArrayList<>(); - if (exprs != null) { - for (Expr expr : exprs) { - TTypeDesc desc = expr.getType().toThrift(); - desc.setIsNullable(expr.isNullable()); - paramTypes.add(desc); - } - } - TAggregateExpr aggExpr = new TAggregateExpr(isMergeAggFn); - aggExpr.setParamTypes(paramTypes); - return aggExpr; - } - public boolean isStar() { return isStar; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 6c292e2569ecc3..f965980d5347e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -148,10 +148,6 @@ public void setColumn(Column column) { this.caption = column.getName(); } - public void setSrcColumn(Column column) { - this.column = column; - } - public boolean isAutoInc() { return isAutoInc; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/VirtualSlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/VirtualSlotRef.java index e31bef9ca6fe09..1a7f5d7c47fb38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/VirtualSlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/VirtualSlotRef.java @@ -38,9 +38,4 @@ public void getTableIdToColumnNames(Map> tableIdToColumnNames) public Expr clone() { return new VirtualSlotRef(this); } - - @Override - public String getExprName() { - return super.getExprName(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java index 57ec6be03f584d..56bc960cd1d9e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/AbstractJob.java @@ -23,6 +23,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.persist.gson.GsonUtilsBase; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -174,7 +175,7 @@ public long getRepoId() { public static AbstractJob read(DataInput in) throws IOException { String json = Text.readString(in); if (COMPRESSED_JOB_ID.equals(json)) { - return GsonUtils.fromJsonCompressed(in, AbstractJob.class); + return GsonUtilsBase.fromJsonCompressed(GsonUtils.GSON, in, AbstractJob.class); } else { return GsonUtils.GSON.fromJson(json, AbstractJob.class); } @@ -199,7 +200,7 @@ public void write(DataOutput out) throws IOException { if (!isDone() && ((type == JobType.BACKUP && Config.backup_job_compressed_serialization) || (type == JobType.RESTORE && Config.restore_job_compressed_serialization))) { Text.writeString(out, COMPRESSED_JOB_ID); - GsonUtils.toJsonCompressed(out, this); + GsonUtilsBase.toJsonCompressed(GsonUtils.GSON, out, this); } else { Text.writeString(out, GsonUtils.GSON.toJson(this)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 10490355a8e011..b55b0a2d0d639d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -41,6 +41,7 @@ import org.apache.doris.persist.BarrierLog; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.persist.gson.GsonUtilsBase; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; @@ -1212,7 +1213,7 @@ private String getBackupObjs() { public static BackupJob read(DataInput in) throws IOException { String json = Text.readString(in); if (AbstractJob.COMPRESSED_JOB_ID.equals(json)) { - return GsonUtils.fromJsonCompressed(in, BackupJob.class); + return GsonUtilsBase.fromJsonCompressed(GsonUtils.GSON, in, BackupJob.class); } else { return GsonUtils.GSON.fromJson(json, BackupJob.class); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 60142fa694f07a..be16c36d3edeef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -71,6 +71,7 @@ import org.apache.doris.persist.ColocatePersistInfo; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.persist.gson.GsonUtilsBase; import org.apache.doris.resource.Tag; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentBoundedBatchTask; @@ -2750,7 +2751,7 @@ protected void setTableStateToNormalAndUpdateProperties(Database db, boolean com public static RestoreJob read(DataInput in) throws IOException { String json = Text.readString(in); if (AbstractJob.COMPRESSED_JOB_ID.equals(json)) { - return GsonUtils.fromJsonCompressed(in, RestoreJob.class); + return GsonUtilsBase.fromJsonCompressed(GsonUtils.GSON, in, RestoreJob.class); } else { return GsonUtils.GSON.fromJson(json, RestoreJob.class); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java index 1f6c624c86db00..0ae4d209a10d8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java @@ -18,11 +18,8 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.Expr; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.gson.annotations.SerializedName; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.Arrays; @@ -34,15 +31,10 @@ * Internal representation of an alias function. */ public class AliasFunction extends Function { - private static final Logger LOG = LogManager.getLogger(AliasFunction.class); - - private static final String DIGITAL_MASKING = "digital_masking"; - @SerializedName("of") private Expr originFunction; @SerializedName("pm") private List parameters = new ArrayList<>(); - private List typeDefParams = new ArrayList<>(); @SerializedName("sv") private Map sessionVariables; @@ -62,7 +54,7 @@ public static AliasFunction createFunction(FunctionName functionName, Type[] arg public static AliasFunction createFunction(FunctionName functionName, Type[] argTypes, Type retType, boolean hasVarArgs, List parameters, Expr originFunction, Map sessionVariables) { AliasFunction aliasFunction = new AliasFunction(functionName, Arrays.asList(argTypes), retType, hasVarArgs); - aliasFunction.setBinaryType(TFunctionBinaryType.JAVA_UDF); + aliasFunction.setBinaryType(Function.BinaryType.JAVA_UDF); aliasFunction.setUserVisible(true); aliasFunction.originFunction = originFunction; aliasFunction.parameters = parameters; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index d3aa5aa5917089..0845bf3f756e8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -19,9 +19,7 @@ import org.apache.doris.analysis.DefaultValueExprDef; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -73,15 +71,11 @@ public class Column implements GsonPostProcessable { // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! private static final String COLUMN_ARRAY_CHILDREN = "item"; - private static final String COLUMN_STRUCT_CHILDREN = "field"; private static final String COLUMN_AGG_ARGUMENT_CHILDREN = "argument"; public static final int COLUMN_UNIQUE_ID_INIT_VALUE = -1; private static final String COLUMN_MAP_KEY = "key"; private static final String COLUMN_MAP_VALUE = "value"; - public static final Column UNSUPPORTED_COLUMN = new Column("unknown", Type.UNSUPPORTED, true, null, true, -1, - null, "invalid", true, null, -1, null); - @SerializedName(value = "name") private String name; @SerializedName(value = "type") @@ -127,7 +121,6 @@ public class Column implements GsonPostProcessable { // so the define expr in RollupJob must be analyzed. // In other cases, such as define expr in `MaterializedIndexMeta`, it may not be analyzed after being replayed. private Expr defineExpr; // use to define column in materialize view - private String defineName = null; @SerializedName(value = "visible") private boolean visible; @SerializedName(value = "defaultValueExprDef") @@ -277,8 +270,8 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType } this.sessionVariables = sessionVariables; - if (type.isAggStateType()) { - AggStateType aggState = (AggStateType) type; + if (this.type.isAggStateType()) { + AggStateType aggState = (AggStateType) this.type; for (int i = 0; i < aggState.getSubTypes().size(); i++) { Column c = new Column(COLUMN_AGG_ARGUMENT_CHILDREN, aggState.getSubTypes().get(i)); c.setIsAllowNull(aggState.getSubTypeNullables().get(i)); @@ -330,7 +323,6 @@ public Column(Column column) { this.children = column.getChildren(); this.uniqueId = column.getUniqueId(); this.defineExpr = column.getDefineExpr(); - this.defineName = column.getRealDefineName(); this.hasOnUpdateDefaultValue = column.hasOnUpdateDefaultValue; this.onUpdateDefaultValueExprDef = column.onUpdateDefaultValueExprDef; this.clusterKeyId = column.getClusterKeyId(); @@ -381,23 +373,10 @@ private void addChildrenColumn(Column column) { this.children.add(column); } - public void setDefineName(String defineName) { - this.defineName = defineName; - } - public String getDefineName() { - if (defineName != null) { - return defineName; - } return name; } - // In order for the copy constructor to get the real defineName value. - // getDefineName() cannot meet this requirement - public String getRealDefineName() { - return defineName; - } - public void setName(String newName) { this.name = newName; } @@ -410,15 +389,6 @@ public String getNonShadowName() { return removeNamePrefix(name); } - public String getDisplayName() { - if (defineExpr == null) { - return name; - } else { - return MaterializedIndexMeta.normalizeName( - defineExpr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); - } - } - public String getNameWithoutPrefix(String prefix) { if (isNameWithPrefix(prefix)) { return name.substring(prefix.length()); @@ -716,7 +686,7 @@ private void addChildren(OlapFile.ColumnPB.Builder builder) throws DdlException if (this.getChildren() != null) { List childrenColumns = this.getChildren(); for (Column c : childrenColumns) { - builder.addChildrenColumns(c.toPb(Sets.newHashSet(), Lists.newArrayList())); + builder.addChildrenColumns(c.toPb(Sets.newHashSet())); } } } @@ -812,7 +782,7 @@ public int getFieldLengthByType(TPrimitiveType type, int stringLength) throws Dd } } - public OlapFile.ColumnPB toPb(Set bfColumns, List indexes) throws DdlException { + public OlapFile.ColumnPB toPb(Set bfColumns) throws DdlException { OlapFile.ColumnPB.Builder builder = OlapFile.ColumnPB.newBuilder(); // when doing schema change, some modified column has a prefix in name. @@ -838,7 +808,7 @@ public OlapFile.ColumnPB toPb(Set bfColumns, List indexes) throws builder.setBeExecVersion(Config.be_exec_version); if (children != null) { for (Column column : children) { - builder.addChildrenColumns(column.toPb(Sets.newHashSet(), Lists.newArrayList())); + builder.addChildrenColumns(column.toPb(Sets.newHashSet())); } } } else { @@ -872,16 +842,16 @@ public OlapFile.ColumnPB toPb(Set bfColumns, List indexes) throws if (this.type.isArrayType()) { Column child = this.getChildren().get(0); - builder.addChildrenColumns(child.toPb(Sets.newHashSet(), Lists.newArrayList())); + builder.addChildrenColumns(child.toPb(Sets.newHashSet())); } else if (this.type.isMapType()) { Column k = this.getChildren().get(0); - builder.addChildrenColumns(k.toPb(Sets.newHashSet(), Lists.newArrayList())); + builder.addChildrenColumns(k.toPb(Sets.newHashSet())); Column v = this.getChildren().get(1); - builder.addChildrenColumns(v.toPb(Sets.newHashSet(), Lists.newArrayList())); + builder.addChildrenColumns(v.toPb(Sets.newHashSet())); } else if (this.type.isStructType()) { List childrenColumns = this.getChildren(); for (Column c : childrenColumns) { - builder.addChildrenColumns(c.toPb(Sets.newHashSet(), Lists.newArrayList())); + builder.addChildrenColumns(c.toPb(Sets.newHashSet())); } } else if (this.type.isVariantType()) { builder.setVariantMaxSubcolumnsCount(this.getVariantMaxSubcolumnsCount()); @@ -896,8 +866,7 @@ public OlapFile.ColumnPB toPb(Set bfColumns, List indexes) throws addChildren(builder); } - OlapFile.ColumnPB col = builder.build(); - return col; + return builder.build(); } // CLOUD_CODE_END @@ -1057,10 +1026,6 @@ public String toSql() { return toSql(false, false); } - public String toSql(boolean isUniqueTable) { - return toSql(isUniqueTable, false); - } - public String toSql(boolean isUniqueTable, boolean isCompatible) { StringBuilder sb = new StringBuilder(); sb.append("`").append(name).append("` "); @@ -1077,8 +1042,7 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { sb.append(" ").append(aggregationType.toSql()); } if (generatedColumnInfo != null) { - sb.append(" AS (").append(generatedColumnInfo.getExpr() - .accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE)).append(")"); + sb.append(" AS (").append(generatedColumnInfo.getExprSql()).append(")"); } if (isAllowNull) { sb.append(" NULL"); @@ -1090,18 +1054,18 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { } if (defaultValue != null && getDataType() != PrimitiveType.HLL && getDataType() != PrimitiveType.BITMAP) { if (defaultValueExprDef != null) { - sb.append(" DEFAULT ").append(defaultValue).append(""); + sb.append(" DEFAULT ").append(defaultValue); } else { sb.append(" DEFAULT \"").append(defaultValue).append("\""); } } if ((getDataType() == PrimitiveType.BITMAP) && defaultValue != null) { if (defaultValueExprDef != null) { - sb.append(" DEFAULT ").append(defaultValueExprDef.getExprName()).append(""); + sb.append(" DEFAULT ").append(defaultValueExprDef.getExprName()); } } if (hasOnUpdateDefaultValue) { - sb.append(" ON UPDATE ").append(defaultValue).append(""); + sb.append(" ON UPDATE ").append(defaultValue); } if (StringUtils.isNotBlank(comment)) { sb.append(" COMMENT \"").append(getComment(true)).append("\""); @@ -1196,9 +1160,6 @@ public String getSignatureString(Map typeStringMap) { case VARCHAR: sb.append(String.format(typeStringMap.get(dataType), getStrLen())); break; - case JSONB: - sb.append(type.toString()); - break; case DECIMALV2: case DECIMAL32: case DECIMAL64: @@ -1206,6 +1167,7 @@ public String getSignatureString(Map typeStringMap) { case DECIMAL256: sb.append(String.format(typeStringMap.get(dataType), getPrecision(), getScale())); break; + case JSONB: case ARRAY: case MAP: case STRUCT: @@ -1239,8 +1201,7 @@ public long getAutoIncInitValue() { return this.autoIncInitValue; } - public void setIndexFlag(TColumn tColumn, OlapTable olapTable) { - Set bfColumns = olapTable.getCopiedBfColumns(); + public void setIndexFlag(TColumn tColumn, Set bfColumns) { if (bfColumns != null && bfColumns.contains(tColumn.getColumnName())) { tColumn.setIsBloomFilterColumn(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index e8622cf840832c..ba6d99599e867d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4110,7 +4110,7 @@ public static void getCreateTableLikeStmt(CreateTableLikeInfo createTableLikeInf } // There MUST BE 2 space in front of each column description line // sqlalchemy requires this to parse SHOW CREATE TABLE stmt. - if (table.isManagedTable()) { + if (table instanceof OlapTable) { sb.append(" ").append( column.toSql(((OlapTable) table).getKeysType() == KeysType.UNIQUE_KEYS, true)); } else { @@ -5943,9 +5943,6 @@ private void renameColumn(Database db, OlapTable table, String colName, expr.collect(SlotRef.class, slots); for (SlotRef slot : slots) { String name = slot.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); - if (slot.getColumn() != null) { - name = slot.getColumn().getName(); - } if (!isReplay && name.equals(colName)) { throw new DdlException("Column[" + colName + "] have materialized view index"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java index c599a3f9eb2ab6..8709eb5b6de8ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java @@ -22,7 +22,6 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.catalog.Function.NullableMode; -import org.apache.doris.thrift.TFunctionBinaryType; import java.util.List; import java.util.stream.Collectors; @@ -71,7 +70,7 @@ public static String toSql(ScalarFunction fn, boolean ifNotExists) { sb.append(",\n \"CLOSE_FN\"=").append("\"" + fn.getCloseFnSymbol() + "\""); } - if (fn.getBinaryType() == TFunctionBinaryType.JAVA_UDF) { + if (fn.getBinaryType() == Function.BinaryType.JAVA_UDF) { sb.append(",\n \"FILE\"=") .append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\""); boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE; @@ -106,7 +105,7 @@ public static String toSql(AggregateFunction fn, boolean ifNotExists) { } sb.append(" PROPERTIES ("); - if (fn.getBinaryType() != TFunctionBinaryType.JAVA_UDF) { + if (fn.getBinaryType() != Function.BinaryType.JAVA_UDF) { sb.append("\n \"INIT_FN\"=\"" + fn.getInitFnSymbol() + "\",") .append("\n \"UPDATE_FN\"=\"" + fn.getUpdateFnSymbol() + "\",") .append("\n \"MERGE_FN\"=\"" + fn.getMergeFnSymbol() + "\","); @@ -121,7 +120,7 @@ public static String toSql(AggregateFunction fn, boolean ifNotExists) { sb.append("\n \"SYMBOL\"=\"" + fn.getSymbolName() + "\","); } - if (fn.getBinaryType() == TFunctionBinaryType.JAVA_UDF) { + if (fn.getBinaryType() == Function.BinaryType.JAVA_UDF) { sb.append("\n \"FILE\"=") .append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\","); boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java index 4a33d93fa78c45..4df3fe3585f445 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToThriftConverter.java @@ -18,6 +18,7 @@ package org.apache.doris.catalog; import org.apache.doris.thrift.TAggregateFunction; +import org.apache.doris.thrift.TDictFunction; import org.apache.doris.thrift.TFunction; import org.apache.doris.thrift.TFunctionBinaryType; import org.apache.doris.thrift.TFunctionName; @@ -52,20 +53,23 @@ public static TFunction toThrift(ScalarFunction fn, Type realReturnType, Type[] Boolean[] realArgTypeNullables) { TFunction tfn = toThriftBase(fn, realReturnType, realArgTypes, realArgTypeNullables); tfn.setScalarFn(new TScalarFunction()); - if (fn.getBinaryType() == TFunctionBinaryType.JAVA_UDF || fn.getBinaryType() == TFunctionBinaryType.RPC - || fn.getBinaryType() == TFunctionBinaryType.PYTHON_UDF) { + if (fn.getBinaryType() == Function.BinaryType.JAVA_UDF || fn.getBinaryType() == Function.BinaryType.RPC + || fn.getBinaryType() == Function.BinaryType.PYTHON_UDF) { tfn.getScalarFn().setSymbol(fn.getSymbolName()); } else { tfn.getScalarFn().setSymbol(""); } - if (fn.getBinaryType() == TFunctionBinaryType.PYTHON_UDF) { + if (fn.getBinaryType() == Function.BinaryType.PYTHON_UDF) { if (!Strings.isNullOrEmpty(fn.getFunctionCode())) { tfn.setFunctionCode(fn.getFunctionCode()); } tfn.setRuntimeVersion(fn.getRuntimeVersion()); } if (fn.getDictFunction() != null) { - tfn.setDictFunction(fn.getDictFunction()); + TDictFunction tDictFunction = new TDictFunction(); + tDictFunction.setDictionaryId(fn.getDictFunction().getId()); + tDictFunction.setVersionId(fn.getDictFunction().getVersion()); + tfn.setDictFunction(tDictFunction); } return tfn; } @@ -105,7 +109,7 @@ public static TFunction toThrift(AggregateFunction fn, Type realReturnType, Type tfn.setAggregateFn(aggFn); // Set runtime_version and function_code for Python UDAF - if (fn.getBinaryType() == TFunctionBinaryType.PYTHON_UDF) { + if (fn.getBinaryType() == Function.BinaryType.PYTHON_UDF) { if (!Strings.isNullOrEmpty(fn.getFunctionCode())) { tfn.setFunctionCode(fn.getFunctionCode()); } @@ -123,7 +127,7 @@ private static TFunction toThriftBase(Function fn, Type realReturnType, Type[] r tName.setDbName(fn.getFunctionName().getDb()); tName.setFunctionName(fn.getFunctionName().getFunction()); tfn.setName(tName); - tfn.setBinaryType(fn.getBinaryType()); + tfn.setBinaryType(TFunctionBinaryType.valueOf(fn.getBinaryType().name())); if (fn.getLocation() != null) { tfn.setHdfsLocation(fn.getLocation().getLocation()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java index 6a75179d151501..c9ef0e81f14857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java @@ -29,7 +29,6 @@ import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdf; import org.apache.doris.nereids.trees.expressions.functions.udf.PythonUdtf; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -195,22 +194,22 @@ private static void translateToNereidsImpl(String dbName, Function function) { AliasUdf.translateToNereidsFunction(dbName, ((AliasFunction) function)); } else if (function instanceof ScalarFunction) { if (function.isUDTFunction()) { - if (function.getBinaryType() == TFunctionBinaryType.JAVA_UDF) { + if (function.getBinaryType() == Function.BinaryType.JAVA_UDF) { JavaUdtf.translateToNereidsFunction(dbName, ((ScalarFunction) function)); - } else if (function.getBinaryType() == TFunctionBinaryType.PYTHON_UDF) { + } else if (function.getBinaryType() == Function.BinaryType.PYTHON_UDF) { PythonUdtf.translateToNereidsFunction(dbName, ((ScalarFunction) function)); } } else { - if (function.getBinaryType() == TFunctionBinaryType.JAVA_UDF) { + if (function.getBinaryType() == Function.BinaryType.JAVA_UDF) { JavaUdf.translateToNereidsFunction(dbName, ((ScalarFunction) function)); - } else if (function.getBinaryType() == TFunctionBinaryType.PYTHON_UDF) { + } else if (function.getBinaryType() == Function.BinaryType.PYTHON_UDF) { PythonUdf.translateToNereidsFunction(dbName, (ScalarFunction) function); } } } else if (function instanceof AggregateFunction) { - if (function.getBinaryType() == TFunctionBinaryType.JAVA_UDF) { + if (function.getBinaryType() == Function.BinaryType.JAVA_UDF) { JavaUdaf.translateToNereidsFunction(dbName, ((AggregateFunction) function)); - } else if (function.getBinaryType() == TFunctionBinaryType.PYTHON_UDF) { + } else if (function.getBinaryType() == Function.BinaryType.PYTHON_UDF) { PythonUdaf.translateToNereidsFunction(dbName, ((AggregateFunction) function)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java index eff55fc8a47a7f..1005d893ac92cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java @@ -18,11 +18,8 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ToSqlParams; import com.google.gson.annotations.SerializedName; -import jline.internal.Nullable; /**GeneratedColumnInfo*/ public class GeneratedColumnInfo { @@ -43,20 +40,17 @@ public enum GeneratedColumnType { e.g. this is column d generated column info expr is c+1, expandExprForLoad is a+b+1 expandExprForLoad is used in streamload, routineload, mysqlload, etc */ + @Deprecated @SerializedName(value = "efl") private Expr expandExprForLoad; /** constructor */ - public GeneratedColumnInfo(@Nullable String exprSql, Expr expr) { + public GeneratedColumnInfo(String exprSql, Expr expr) { this(exprSql, expr, null); } - public GeneratedColumnInfo(@Nullable String exprSql, Expr expr, Expr expandExprForLoad) { - if (exprSql != null) { - this.exprSql = exprSql; - } else { - this.exprSql = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); - } + public GeneratedColumnInfo(String exprSql, Expr expr, Expr expandExprForLoad) { + this.exprSql = exprSql; this.expr = expr; this.expandExprForLoad = expandExprForLoad; this.type = GeneratedColumnType.STORED; @@ -69,12 +63,4 @@ public String getExprSql() { public Expr getExpr() { return expr; } - - public Expr getExpandExprForLoad() { - return expandExprForLoad; - } - - public void setExpandExprForLoad(Expr expandExprForLoad) { - this.expandExprForLoad = expandExprForLoad; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index ba1b8d3c715739..1e98164801303a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3211,7 +3211,7 @@ public void getColumnDesc(long selectedIndexId, List columnsDesc, List< // continue; // } TColumn tColumn = col.toThrift(); - col.setIndexFlag(tColumn, this); + col.setIndexFlag(tColumn, this.getCopiedBfColumns()); if (columnsDesc != null) { columnsDesc.add(tColumn); } @@ -3232,7 +3232,7 @@ public void getColumnDesc(long selectedIndexId, List columnsDesc, List< if (selectedIndexId != -1) { for (Column col : this.getSchemaByIndexId(selectedIndexId, true)) { TColumn tColumn = col.toThrift(); - col.setIndexFlag(tColumn, this); + col.setIndexFlag(tColumn, this.getCopiedBfColumns()); if (columnsDesc != null) { columnsDesc.add(tColumn); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 89e6c637a3a134..5ed7eaddca1b7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -131,7 +131,7 @@ public String getDisplayPartitionColumns() { if (index != 0) { sb.append(", "); } - sb.append(c.getDisplayName()); + sb.append(c.getName()); index++; } return sb.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index a5dcb9d12eb21d..9526b407aed031 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -379,7 +379,7 @@ public String toSql() { int i = 0; for (LiteralExpr expr : keys) { Object value = null; - if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) { + if (expr == MaxLiteral.MAX_VALUE || expr instanceof NullLiteral) { value = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE); sb.append(value); } else { @@ -428,7 +428,7 @@ public static String toString(List keys) { int i = 0; for (LiteralExpr expr : keys) { Object value = null; - if (expr == MaxLiteral.MAX_VALUE || expr.isNullLiteral()) { + if (expr == MaxLiteral.MAX_VALUE || expr instanceof NullLiteral) { value = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE); } else { value = expr.getRealValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java deleted file mode 100644 index cafe90f2e519c9..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarFunction.java +++ /dev/null @@ -1,213 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.catalog; - -import org.apache.doris.common.util.URI; -import org.apache.doris.thrift.TDictFunction; -import org.apache.doris.thrift.TFunctionBinaryType; - -import com.google.gson.annotations.SerializedName; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -// import org.apache.doris.thrift.TSymbolType; - -/** - * Internal representation of a scalar function. - */ -public class ScalarFunction extends Function { - private static final Logger LOG = LogManager.getLogger(ScalarFunction.class); - // The name inside the binary at location_ that contains this particular - // function. e.g. org.example.MyUdf.class. - @SerializedName("sn") - private String symbolName; - @SerializedName("pfs") - private String prepareFnSymbol; - @SerializedName("cfs") - private String closeFnSymbol; - - TDictFunction dictFunction = null; - - // Only used for serialization - protected ScalarFunction() { - } - - public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, - boolean userVisible) { - this(fnName, argTypes, retType, hasVarArgs, TFunctionBinaryType.BUILTIN, userVisible, true); - } - - public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, - boolean userVisible, boolean isVec) { - this(fnName, argTypes, retType, hasVarArgs, TFunctionBinaryType.BUILTIN, userVisible, isVec); - } - - public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, - TFunctionBinaryType binaryType, boolean userVisible, boolean isVec) { - super(0, fnName, argTypes, retType, hasVarArgs, binaryType, userVisible, isVec, - NullableMode.DEPEND_ON_ARGUMENT); - } - - /** - * nerieds custom scalar function - */ - public ScalarFunction(FunctionName fnName, List argTypes, Type retType, boolean hasVarArgs, String symbolName, - TFunctionBinaryType binaryType, boolean userVisible, boolean isVec, NullableMode nullableMode) { - super(0, fnName, argTypes, retType, hasVarArgs, binaryType, userVisible, isVec, nullableMode); - this.symbolName = symbolName; - } - - public ScalarFunction(FunctionName fnName, List argTypes, - Type retType, URI location, String symbolName, String initFnSymbol, - String closeFnSymbol) { - super(fnName, argTypes, retType, false); - setLocation(location); - setSymbolName(symbolName); - setPrepareFnSymbol(initFnSymbol); - setCloseFnSymbol(closeFnSymbol); - } - - /** - * Creates a builtin scalar function. This is a helper that wraps a few steps - * into one call. - */ - public static ScalarFunction createBuiltin(String name, Type retType, - ArrayList argTypes, boolean hasVarArgs, - String symbol, String prepareFnSymbol, String closeFnSymbol, - boolean userVisible) { - return createBuiltin(name, retType, NullableMode.DEPEND_ON_ARGUMENT, argTypes, hasVarArgs, - symbol, prepareFnSymbol, closeFnSymbol, userVisible); - } - - public static ScalarFunction createBuiltin( - String name, Type retType, NullableMode nullableMode, - ArrayList argTypes, boolean hasVarArgs, - String symbol, String prepareFnSymbol, String closeFnSymbol, boolean userVisible) { - ScalarFunction fn = new ScalarFunction( - new FunctionName(name), argTypes, retType, hasVarArgs, userVisible); - fn.symbolName = symbol; - fn.prepareFnSymbol = prepareFnSymbol; - fn.closeFnSymbol = closeFnSymbol; - fn.nullableMode = nullableMode; - return fn; - } - - public static ScalarFunction createBuiltinOperator( - String name, ArrayList argTypes, Type retType) { - return createBuiltinOperator(name, argTypes, retType, NullableMode.DEPEND_ON_ARGUMENT); - } - - /** - * Creates a builtin scalar operator function. This is a helper that wraps a few - * steps - * into one call. - * TODO: this needs to be kept in sync with what generates the be operator - * implementations. (gen_functions.py). Is there a better way to coordinate - * this. - */ - public static ScalarFunction createBuiltinOperator( - String name, ArrayList argTypes, Type retType, NullableMode nullableMode) { - return createBuiltinOperator(name, null, argTypes, retType, nullableMode); - } - - public static ScalarFunction createBuiltinOperator( - String name, String symbol, ArrayList argTypes, Type retType) { - return createBuiltinOperator(name, symbol, argTypes, retType, NullableMode.DEPEND_ON_ARGUMENT); - } - - public static ScalarFunction createBuiltinOperator( - String name, String symbol, ArrayList argTypes, Type retType, NullableMode nullableMode) { - return createBuiltin(name, symbol, argTypes, false, retType, false, nullableMode); - } - - public static ScalarFunction createBuiltin( - String name, String symbol, ArrayList argTypes, - boolean hasVarArgs, Type retType, boolean userVisible, NullableMode nullableMode) { - ScalarFunction fn = new ScalarFunction( - new FunctionName(name), argTypes, retType, hasVarArgs, userVisible); - fn.symbolName = symbol; - fn.nullableMode = nullableMode; - return fn; - } - - public static ScalarFunction createUdf( - TFunctionBinaryType binaryType, - FunctionName name, Type[] args, - Type returnType, boolean isVariadic, - URI location, String symbol, String prepareFnSymbol, String closeFnSymbol) { - ScalarFunction fn = new ScalarFunction(name, Arrays.asList(args), returnType, isVariadic, binaryType, - true, false); - fn.symbolName = symbol; - fn.prepareFnSymbol = prepareFnSymbol; - fn.closeFnSymbol = closeFnSymbol; - fn.setLocation(location); - return fn; - } - - public ScalarFunction(ScalarFunction other) { - super(other); - if (other == null) { - return; - } - symbolName = other.symbolName; - prepareFnSymbol = other.prepareFnSymbol; - closeFnSymbol = other.closeFnSymbol; - } - - @Override - public Function clone() { - return new ScalarFunction(this); - } - - public void setSymbolName(String s) { - symbolName = s; - } - - public void setPrepareFnSymbol(String s) { - prepareFnSymbol = s; - } - - public void setCloseFnSymbol(String s) { - closeFnSymbol = s; - } - - public String getSymbolName() { - return symbolName; - } - - public String getPrepareFnSymbol() { - return prepareFnSymbol; - } - - public String getCloseFnSymbol() { - return closeFnSymbol; - } - - public TDictFunction getDictFunction() { - return dictFunction; - } - - public void setDictFunction(TDictFunction dictFunction) { - this.dictFunction = dictFunction; - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index 1c1c0385c5b736..62d134c88b3cc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -344,7 +344,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.setSortColNum(dataSortInfo.getColNum()); for (int i = 0; i < schemaColumns.size(); i++) { Column column = schemaColumns.get(i); - schemaBuilder.addColumn(column.toPb(bfColumns, indexes)); + schemaBuilder.addColumn(column.toPb(bfColumns)); } Map columnMap = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index b32dd168ffcdef..66090e65b6dc98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -86,7 +86,7 @@ public static ProcResult createResult(List schema, Set bfColumns } String extraStr = StringUtils.join(extras, ","); - List rowList = Lists.newArrayList(column.getDisplayName(), + List rowList = Lists.newArrayList(column.getName(), column.getOriginType().hideVersionForVersionColumn(true), column.isAllowNull() ? "Yes" : "No", ((Boolean) column.isKey()).toString(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 516d6942478f17..83cbbe3e42dbfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -102,13 +102,13 @@ public static void checkTimeUnit(String timeUnit, PartitionInfo partitionInfo) t && (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString()))) { ErrorReport.reportDdlException(DynamicPartitionProperty.TIME_UNIT + " could not be " + TimeUnit.HOUR + " when type of partition column " - + partitionColumn.getDisplayName() + " is " + PrimitiveType.DATE + " or " + PrimitiveType.DATEV2); + + partitionColumn.getName() + " is " + PrimitiveType.DATE + " or " + PrimitiveType.DATEV2); } else if (PrimitiveType.getIntegerTypes().contains(partitionColumn.getDataType()) && timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { // The partition column's type is INT, not support HOUR ErrorReport.reportDdlException(DynamicPartitionProperty.TIME_UNIT + " could not be " + TimeUnit.HOUR.toString() + " when type of partition column " - + partitionColumn.getDisplayName() + " is Integer"); + + partitionColumn.getName() + " is Integer"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java index 36c1a3a725818b..877f84b6b9e05f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java @@ -257,7 +257,7 @@ private List collectConjunctsToPushDown(List conjunctsList, List pushDownConjuncts = new ArrayList<>(); for (Expr p : conjunctsList) { if (shouldPushDownConjunct(jdbcType, p)) { - List individualConjuncts = p.getConjuncts(); + List individualConjuncts = getConjuncts(p); for (Expr individualConjunct : individualConjuncts) { Expr newp = JdbcFunctionPushDownRule.processFunctions(jdbcType, individualConjunct, errors, functionRules); @@ -272,6 +272,30 @@ private List collectConjunctsToPushDown(List conjunctsList, List getConjuncts(Expr root) { + List list = Lists.newArrayList(); + if (root instanceof CompoundPredicate + && ((CompoundPredicate) root).getOp() == CompoundPredicate.Operator.AND) { + // TODO: we have to convert CompoundPredicate.AND to two expr trees for + // conjuncts because NULLs are handled differently for CompoundPredicate.AND + // and conjunct evaluation. This is not optimal for jitted exprs because it + // will result in two functions instead of one. Create a new CompoundPredicate + // Operator (i.e. CONJUNCT_AND) with the right NULL semantics and use that + // instead + list.addAll(getConjuncts(root.getChild(0))); + list.addAll(getConjuncts(root.getChild(1))); + } else { + list.add(root); + } + return list; + } + private void createJdbcColumns() { columns.clear(); for (SlotDescriptor slot : desc.getSlots()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorPredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorPredicateConverter.java index ed0b3dc2121a59..be8c73be2c91de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorPredicateConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorPredicateConverter.java @@ -57,7 +57,7 @@ public class TrinoConnectorPredicateConverter { private static final Logger LOG = LogManager.getLogger(TrinoConnectorPredicateConverter.class); - private static final String EPOCH_DATE = "1970-01-01"; + private static final DateLiteral EPOCH_DATE = new DateLiteral(1970, 1, 1); private static final String GMT = "GMT"; private final Map trinoConnectorColumnHandleMap; @@ -278,7 +278,7 @@ private Object convertLiteralToDomainValues(Class type, LiteralE case "VarcharType": return Slices.utf8Slice((String) literalExpr.getRealValue()); case "DateType": - return ((DateLiteral) literalExpr).daynr() - new DateLiteral(EPOCH_DATE).daynr(); + return ((DateLiteral) literalExpr).daynr() - EPOCH_DATE.daynr(); case "ShortTimestampType": { DateLiteral dateLiteral = (DateLiteral) literalExpr; return dateLiteral.unixTimestamp(TimeZone.getTimeZone(GMT)) * 1000 diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/operations/HDFSFileOperations.java b/fe/fe-core/src/main/java/org/apache/doris/fs/operations/HDFSFileOperations.java index 44563317bc27bb..38641cdca073b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/operations/HDFSFileOperations.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/operations/HDFSFileOperations.java @@ -18,8 +18,8 @@ package org.apache.doris.fs.operations; import org.apache.doris.backup.Status; +import org.apache.doris.common.URI; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.URI; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java index 2ee3c15693130a..63e5a38abb13fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java @@ -19,10 +19,10 @@ import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status; +import org.apache.doris.common.URI; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.HadoopAuthenticator; import org.apache.doris.common.util.S3Util; -import org.apache.doris.common.util.URI; import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.io.DorisInputFile; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java index fc486d43c5d233..06bc891eb72474 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java @@ -84,7 +84,7 @@ private Map buildColumnInfo(Column column) { columnInfo.put("column_uid", String.valueOf(column.getUniqueId())); columnInfo.put("type", primitiveType.toString()); columnInfo.put("comment", column.getComment()); - columnInfo.put("name", column.getDisplayName()); + columnInfo.put("name", column.getName()); Optional aggregationType = Optional.ofNullable(column.getAggregationType()); columnInfo.put("aggregation_type", aggregationType.isPresent() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index f16384872586b6..dafae086ab3cb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -38,9 +38,11 @@ import org.apache.doris.analysis.SearchPredicate; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TryCastExpr; +import org.apache.doris.catalog.AggStateType; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.Function.BinaryType; import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; @@ -109,8 +111,6 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TDictFunction; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -475,7 +475,7 @@ public Expr visitWindowFunction(WindowFunction function, PlanTranslatorContext c function.hasVarArguments(), null, "", "", null, "", null, "", null, false, - isAnalyticFunction, false, TFunctionBinaryType.BUILTIN, + isAnalyticFunction, BinaryType.BUILTIN, true, true, nullableMode ); @@ -612,13 +612,13 @@ public Expr visitDictGet(DictGet dictGet, PlanTranslatorContext context) { org.apache.doris.catalog.ScalarFunction catalogFunction = new org.apache.doris.catalog.ScalarFunction( new FunctionName(dictGet.getName()), argTypes, signature.returnType.toCatalogDataType(), - dictGet.hasVarArguments(), "", TFunctionBinaryType.BUILTIN, true, true, + dictGet.hasVarArguments(), "", Function.BinaryType.BUILTIN, true, true, NullableMode.ALWAYS_NOT_NULLABLE); // set special fields - TDictFunction dictFunction = new TDictFunction(); - dictFunction.setDictionaryId(dictionary.getId()); - dictFunction.setVersionId(dictionary.getVersion()); + org.apache.doris.catalog.ScalarFunction.DictFunction dictFunction + = new org.apache.doris.catalog.ScalarFunction.DictFunction( + dictionary.getId(), dictionary.getVersion()); catalogFunction.setDictFunction(dictFunction); // create catalog FunctionCallExpr without analyze again @@ -639,13 +639,13 @@ public Expr visitDictGetMany(DictGetMany dictGetMany, PlanTranslatorContext cont org.apache.doris.catalog.ScalarFunction catalogFunction = new org.apache.doris.catalog.ScalarFunction( new FunctionName(dictGetMany.getName()), argTypes, signature.returnType.toCatalogDataType(), - dictGetMany.hasVarArguments(), "", TFunctionBinaryType.BUILTIN, true, true, + dictGetMany.hasVarArguments(), "", Function.BinaryType.BUILTIN, true, true, NullableMode.ALWAYS_NOT_NULLABLE); // set special fields - TDictFunction dictFunction = new TDictFunction(); - dictFunction.setDictionaryId(dictionary.getId()); - dictFunction.setVersionId(dictionary.getVersion()); + org.apache.doris.catalog.ScalarFunction.DictFunction dictFunction + = new org.apache.doris.catalog.ScalarFunction.DictFunction( + dictionary.getId(), dictionary.getVersion()); catalogFunction.setDictFunction(dictFunction); // create catalog FunctionCallExpr without analyze again @@ -703,7 +703,7 @@ public Expr visitScalarFunction(ScalarFunction function, PlanTranslatorContext c org.apache.doris.catalog.ScalarFunction catalogFunction = new org.apache.doris.catalog.ScalarFunction( new FunctionName(function.getName()), argTypes, function.getDataType().toCatalogDataType(), function.hasVarArguments(), - "", TFunctionBinaryType.BUILTIN, true, true, nullableMode); + "", Function.BinaryType.BUILTIN, true, true, nullableMode); // create catalog FunctionCallExpr without analyze again return new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments), function.nullable()); @@ -748,7 +748,7 @@ public Expr visitTableGeneratingFunction(TableGeneratingFunction function, org.apache.doris.catalog.ScalarFunction catalogFunction = new org.apache.doris.catalog.ScalarFunction( new FunctionName(function.getName()), argTypes, function.getDataType().toCatalogDataType(), function.hasVarArguments(), - "", TFunctionBinaryType.BUILTIN, true, true, nullableMode); + "", Function.BinaryType.BUILTIN, true, true, nullableMode); // create catalog FunctionCallExpr without analyze again return new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments), function.nullable()); @@ -814,11 +814,10 @@ private FunctionCallExpr convertToStateCombinator(String name, FunctionCallExpr List arguments = Arrays.asList(aggFunction.getArgs()); org.apache.doris.catalog.ScalarFunction fn = new org.apache.doris.catalog.ScalarFunction( new FunctionName(name), arguments, - Expr.createAggStateType(aggFunction.getFunctionName().getFunction(), - argTypes, argNullables, returnNullable), + new AggStateType(aggFunction.getFunctionName().getFunction(), returnNullable, argTypes, argNullables), aggFunction.hasVarArgs(), aggFunction.isUserVisible()); fn.setNullableMode(NullableMode.ALWAYS_NOT_NULLABLE); - fn.setBinaryType(TFunctionBinaryType.AGG_STATE); + fn.setBinaryType(Function.BinaryType.AGG_STATE); return new FunctionCallExpr(fn, new FunctionParams(fnCall.getChildren()), false); } @@ -836,7 +835,7 @@ private FunctionCallExpr convertToMergeCombinator(String name, FunctionCallExpr Function aggFunction = fnCall.getFn(); aggFunction.setName(new FunctionName(name)); aggFunction.setArgs(Arrays.asList(fnCall.getChildren().get(0).getType())); - aggFunction.setBinaryType(TFunctionBinaryType.AGG_STATE); + aggFunction.setBinaryType(Function.BinaryType.AGG_STATE); return fnCall; } @@ -854,7 +853,7 @@ private FunctionCallExpr convertToUnionCombinator(String name, FunctionCallExpr Function aggFunction = fnCall.getFn(); aggFunction.setName(new FunctionName(name)); aggFunction.setArgs(Arrays.asList(fnCall.getChildren().get(0).getType())); - aggFunction.setBinaryType(TFunctionBinaryType.AGG_STATE); + aggFunction.setBinaryType(Function.BinaryType.AGG_STATE); aggFunction.setNullableMode(NullableMode.ALWAYS_NOT_NULLABLE); aggFunction.setReturnType(fnCall.getChildren().get(0).getType()); fnCall.setType(fnCall.getChildren().get(0).getType()); @@ -900,8 +899,8 @@ public Expr visitAggregateFunction(AggregateFunction function, PlanTranslatorCon new FunctionName(function.getName()), argTypes, function.getDataType().toCatalogDataType(), function.getIntermediateTypes().toCatalogDataType(), - function.hasVarArguments(), null, "", "", null, "", null, "", null, false, false, false, - TFunctionBinaryType.BUILTIN, true, true, + function.hasVarArguments(), null, "", "", null, "", null, "", null, false, false, + BinaryType.BUILTIN, true, true, function.nullable() ? NullableMode.ALWAYS_NULLABLE : NullableMode.ALWAYS_NOT_NULLABLE); return new FunctionCallExpr(catalogFunction, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 48bf546ef6ba19..0795eafb326346 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -2206,11 +2206,11 @@ public PlanFragment visitPhysicalProject(PhysicalProject project List conjuncts = inputPlanNode.getConjuncts(); Set requiredSlotIdSet = Sets.newHashSet(); for (Expr expr : allProjectionExprs) { - Expr.extractSlots(expr, requiredSlotIdSet); + extractSlots(expr, requiredSlotIdSet); } Set requiredByProjectSlotIdSet = Sets.newHashSet(requiredSlotIdSet); for (Expr expr : conjuncts) { - Expr.extractSlots(expr, requiredSlotIdSet); + extractSlots(expr, requiredSlotIdSet); } // For hash join node, use vSrcToOutputSMap to describe the expression calculation, use // vIntermediateTupleDescList as input, and set vOutputTupleDesc as the final output. @@ -2228,13 +2228,13 @@ public PlanFragment visitPhysicalProject(PhysicalProject project Set requiredOtherConjunctsSlotIdSet = Sets.newHashSet(); List otherConjuncts = ((HashJoinNode) joinNode).getOtherJoinConjuncts(); for (Expr expr : otherConjuncts) { - Expr.extractSlots(expr, requiredOtherConjunctsSlotIdSet); + extractSlots(expr, requiredOtherConjunctsSlotIdSet); } if (!((HashJoinNode) joinNode).getEqJoinConjuncts().isEmpty() && !((HashJoinNode) joinNode).getMarkJoinConjuncts().isEmpty()) { List markConjuncts = ((HashJoinNode) joinNode).getMarkJoinConjuncts(); for (Expr expr : markConjuncts) { - Expr.extractSlots(expr, requiredOtherConjunctsSlotIdSet); + extractSlots(expr, requiredOtherConjunctsSlotIdSet); } } requiredOtherConjunctsSlotIdSet.forEach(e -> requiredExprIds.add(context.findExprId(e))); @@ -2265,7 +2265,7 @@ public PlanFragment visitPhysicalProject(PhysicalProject project // slots used by expandConjuncts must be added to TableFunctionNode's output slot ids List expandConjuncts = tableFunctionNode.getExpandConjuncts(); for (Expr expr : expandConjuncts) { - Expr.extractSlots(expr, requiredSlotIdSet); + extractSlots(expr, requiredSlotIdSet); } tableFunctionNode.setOutputSlotIds(Lists.newArrayList(requiredSlotIdSet)); } @@ -2297,6 +2297,16 @@ public PlanFragment visitPhysicalProject(PhysicalProject project return inputFragment; } + private static void extractSlots(Expr root, Set slotIdSet) { + if (root instanceof SlotRef) { + slotIdSet.add(((SlotRef) root).getDesc().getId()); + return; + } + for (Expr child : root.getChildren()) { + extractSlots(child, slotIdSet); + } + } + @Override public PlanFragment visitPhysicalRecursiveUnion(PhysicalRecursiveUnion recursiveCte, PlanTranslatorContext context) { @@ -2546,7 +2556,7 @@ public PlanFragment visitPhysicalTopN(PhysicalTopN topN, PlanTra TopnFilter filter = context.getTopnFilterContext().getTopnFilter(topN); List> targets = new ArrayList<>(); for (Entry entry : filter.legacyTargets.entrySet()) { - Set inputSlots = entry.getValue().getInputSlotRef(); + Set inputSlots = getInputSlotRef(entry.getValue()); if (inputSlots.size() != 1) { LOG.warn("topn filter targets error: " + inputSlots); } else { @@ -2596,6 +2606,19 @@ public PlanFragment visitPhysicalTopN(PhysicalTopN topN, PlanTra return inputFragment; } + private Set getInputSlotRef(Expr root) { + Set slots = new HashSet<>(); + if (root instanceof SlotRef) { + slots.add((SlotRef) root); + return slots; + } else { + for (Expr expr : root.getChildren()) { + slots.addAll(getInputSlotRef(expr)); + } + } + return slots; + } + @Override public PlanFragment visitPhysicalDeferMaterializeTopN(PhysicalDeferMaterializeTopN topN, PlanTranslatorContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0a34b8af2962e9..39891d7b7dc1c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -18,13 +18,11 @@ package org.apache.doris.nereids.rules.expression.rules; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprId; import org.apache.doris.analysis.ExprToThriftVisitor; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.Config; -import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.TimeUtils; @@ -146,7 +144,7 @@ public static boolean isEnableFoldByBe(ExpressionMatchingContext ctx /** foldByBE */ public static Expression foldByBE(ExpressionMatchingContext context) { - IdGenerator idGenerator = ExprId.createGenerator(); + Integer idGenerator = 0; Expression root = context.expr; Map constMap = Maps.newHashMap(); @@ -202,10 +200,10 @@ private static Expression replace( } private static void collectConst(Expression expr, Map constMap, - Map tExprMap, IdGenerator idGenerator) { + Map tExprMap, Integer id) { if (expr.isConstant() && !expr.isLiteral() && !expr.anyMatch(e -> shouldSkipFold((Expression) e))) { - String id = idGenerator.getNextId().toString(); - constMap.put(id, expr); + id++; + constMap.put(String.valueOf(id), expr); Expr staleExpr; try { staleExpr = ExpressionTranslator.translate(expr, null); @@ -218,11 +216,11 @@ private static void collectConst(Expression expr, Map constM LOG.warn("expression {} translate to legacy expr failed. ", expr); return; } - tExprMap.put(id, ExprToThriftVisitor.treeToThrift(staleExpr)); + tExprMap.put(String.valueOf(id), ExprToThriftVisitor.treeToThrift(staleExpr)); } else { for (int i = 0; i < expr.children().size(); i++) { final Expression child = expr.children().get(i); - collectConst(child, constMap, tExprMap, idGenerator); + collectConst(child, constMap, tExprMap, id); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java index 9fa2b2e66eb727..60466766e96f22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java @@ -23,7 +23,7 @@ import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; -import org.apache.doris.common.util.URI; +import org.apache.doris.common.URI; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -32,7 +32,6 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -47,7 +46,7 @@ public class JavaUdaf extends AggregateFunction implements ExplicitlyCastableSignature, Udf { private final String dbName; private final long functionId; - private final TFunctionBinaryType binaryType; + private final Function.BinaryType binaryType; private final FunctionSignature signature; private final DataType intermediateType; private final NullableMode nullableMode; @@ -67,7 +66,7 @@ public class JavaUdaf extends AggregateFunction implements ExplicitlyCastableSig /** * Constructor of UDAF */ - public JavaUdaf(String name, long functionId, String dbName, TFunctionBinaryType binaryType, + private JavaUdaf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, DataType intermediateType, NullableMode nullableMode, String objectFile, String symbol, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java index 7582c65f67d88b..cd540c261401de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java @@ -23,7 +23,7 @@ import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; -import org.apache.doris.common.util.URI; +import org.apache.doris.common.URI; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -32,7 +32,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -47,7 +46,7 @@ public class JavaUdf extends ScalarFunction implements ExplicitlyCastableSignature, Udf { private final String dbName; private final long functionId; - private final TFunctionBinaryType binaryType; + private final Function.BinaryType binaryType; private final FunctionSignature signature; private final NullableMode nullableMode; private final String objectFile; @@ -61,7 +60,7 @@ public class JavaUdf extends ScalarFunction implements ExplicitlyCastableSignatu /** * Constructor of UDF */ - public JavaUdf(String name, long functionId, String dbName, TFunctionBinaryType binaryType, + public JavaUdf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, Expression... args) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java index 3bc113df51279d..786832f79dc6d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java @@ -23,7 +23,7 @@ import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; -import org.apache.doris.common.util.URI; +import org.apache.doris.common.URI; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -32,7 +32,6 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -47,7 +46,7 @@ public class JavaUdtf extends TableGeneratingFunction implements ExplicitlyCastableSignature, Udf { private final String dbName; private final long functionId; - private final TFunctionBinaryType binaryType; + private final Function.BinaryType binaryType; private final FunctionSignature signature; private final NullableMode nullableMode; private final String objectFile; @@ -61,7 +60,7 @@ public class JavaUdtf extends TableGeneratingFunction implements ExplicitlyCasta /** * Constructor of UDTF */ - public JavaUdtf(String name, long functionId, String dbName, TFunctionBinaryType binaryType, + private JavaUdtf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, Expression... args) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java index 1e78297975ef34..e5a55d381eee6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java @@ -23,7 +23,7 @@ import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; -import org.apache.doris.common.util.URI; +import org.apache.doris.common.URI; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -32,7 +32,6 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -47,7 +46,7 @@ public class PythonUdaf extends AggregateFunction implements ExplicitlyCastableSignature, Udf { private final String dbName; private final long functionId; - private final TFunctionBinaryType binaryType; + private final Function.BinaryType binaryType; private final FunctionSignature signature; private final DataType intermediateType; private final NullableMode nullableMode; @@ -69,7 +68,7 @@ public class PythonUdaf extends AggregateFunction implements ExplicitlyCastableS /** * Constructor of UDAF */ - public PythonUdaf(String name, long functionId, String dbName, TFunctionBinaryType binaryType, + public PythonUdaf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, DataType intermediateType, NullableMode nullableMode, String objectFile, String symbol, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java index b505db5aceb3b5..325247cf3038f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java @@ -23,7 +23,7 @@ import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; -import org.apache.doris.common.util.URI; +import org.apache.doris.common.URI; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -32,7 +32,6 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -47,7 +46,7 @@ public class PythonUdf extends ScalarFunction implements ExplicitlyCastableSignature, Udf { private final String dbName; private final long functionId; - private final TFunctionBinaryType binaryType; + private final Function.BinaryType binaryType; private final FunctionSignature signature; private final NullableMode nullableMode; private final String objectFile; @@ -63,7 +62,7 @@ public class PythonUdf extends ScalarFunction implements ExplicitlyCastableSigna /** * Constructor of UDF */ - public PythonUdf(String name, long functionId, String dbName, TFunctionBinaryType binaryType, + public PythonUdf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java index 24ea3e0d6641a8..a0e03052e5f5e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java @@ -23,7 +23,7 @@ import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; -import org.apache.doris.common.util.URI; +import org.apache.doris.common.URI; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -32,7 +32,6 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -47,7 +46,7 @@ public class PythonUdtf extends TableGeneratingFunction implements ExplicitlyCastableSignature, Udf { private final String dbName; private final long functionId; - private final TFunctionBinaryType binaryType; + private final Function.BinaryType binaryType; private final FunctionSignature signature; private final NullableMode nullableMode; private final String objectFile; @@ -63,7 +62,7 @@ public class PythonUdtf extends TableGeneratingFunction implements ExplicitlyCas /** * Constructor of Python UDTF */ - public PythonUdtf(String name, long functionId, String dbName, TFunctionBinaryType binaryType, + public PythonUdtf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index 3a59c4cea2f57b..e6404617295d65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -44,8 +44,8 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.URI; import org.apache.doris.common.plugin.CloudPluginDownloader; -import org.apache.doris.common.util.URI; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.CascadesContext; @@ -82,7 +82,6 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContextUtil; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.thrift.TFunctionBinaryType; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -166,7 +165,7 @@ public class CreateFunctionCommand extends Command implements ForwardWithSync { private final Map properties; private final List parameters; private final Expression originFunction; - private TFunctionBinaryType binaryType = TFunctionBinaryType.JAVA_UDF; + private Function.BinaryType binaryType = Function.BinaryType.JAVA_UDF; // needed item set after analyzed private String userFile; private String originalUserFile; // Keep original jar name for BE @@ -320,10 +319,10 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { userFile = properties.getOrDefault(FILE_KEY, properties.get(OBJECT_FILE_KEY)); originalUserFile = userFile; // Keep original jar name for BE // Convert userFile to realUrl only for FE checksum calculation - if (!Strings.isNullOrEmpty(userFile) && binaryType != TFunctionBinaryType.RPC) { + if (!Strings.isNullOrEmpty(userFile) && binaryType != Function.BinaryType.RPC) { userFile = getRealUrl(userFile); } - if (!Strings.isNullOrEmpty(userFile) && binaryType != TFunctionBinaryType.RPC) { + if (!Strings.isNullOrEmpty(userFile) && binaryType != Function.BinaryType.RPC) { try { computeObjectChecksum(); } catch (IOException | NoSuchAlgorithmException e) { @@ -334,7 +333,7 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { throw new AnalysisException("library's checksum is not equal with input, checksum=" + checksum); } } - if (binaryType == TFunctionBinaryType.JAVA_UDF) { + if (binaryType == Function.BinaryType.JAVA_UDF) { FunctionUtil.checkEnableJavaUdf(); // always_nullable the default value is true, equal null means true @@ -348,7 +347,7 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { isStaticLoad = true; } extractExpirationTime(); - } else if (binaryType == TFunctionBinaryType.PYTHON_UDF) { + } else if (binaryType == Function.BinaryType.PYTHON_UDF) { FunctionUtil.checkEnablePythonUdf(); // always_nullable the default value is true, equal null means true @@ -457,9 +456,9 @@ private void analyzeUdtf() throws AnalysisException { if (!returnType.isArrayType()) { throw new AnalysisException("JAVA_UDTF OR PYTHON_UDTF return type must be array type"); } - if (binaryType == TFunctionBinaryType.JAVA_UDF) { + if (binaryType == Function.BinaryType.JAVA_UDF) { analyzeJavaUdf(symbol); - } else if (binaryType == TFunctionBinaryType.PYTHON_UDF) { + } else if (binaryType == Function.BinaryType.PYTHON_UDF) { analyzePythonUdtf(symbol); } URI location; @@ -494,19 +493,19 @@ private void analyzeUdaf() throws AnalysisException { .hasVarArgs(argsDef.isVariadic()).intermediateType(intermediateType.toCatalogDataType()) .location(location); String initFnSymbol = properties.get(INIT_KEY); - if (initFnSymbol == null && !(binaryType == TFunctionBinaryType.JAVA_UDF - || binaryType == TFunctionBinaryType.PYTHON_UDF - || binaryType == TFunctionBinaryType.RPC)) { + if (initFnSymbol == null && !(binaryType == Function.BinaryType.JAVA_UDF + || binaryType == Function.BinaryType.PYTHON_UDF + || binaryType == Function.BinaryType.RPC)) { throw new AnalysisException("No 'init_fn' in properties"); } String updateFnSymbol = properties.get(UPDATE_KEY); - if (updateFnSymbol == null && !(binaryType == TFunctionBinaryType.JAVA_UDF - || binaryType == TFunctionBinaryType.PYTHON_UDF)) { + if (updateFnSymbol == null && !(binaryType == Function.BinaryType.JAVA_UDF + || binaryType == Function.BinaryType.PYTHON_UDF)) { throw new AnalysisException("No 'update_fn' in properties"); } String mergeFnSymbol = properties.get(MERGE_KEY); - if (mergeFnSymbol == null && !(binaryType == TFunctionBinaryType.JAVA_UDF - || binaryType == TFunctionBinaryType.PYTHON_UDF)) { + if (mergeFnSymbol == null && !(binaryType == Function.BinaryType.JAVA_UDF + || binaryType == Function.BinaryType.PYTHON_UDF)) { throw new AnalysisException("No 'merge_fn' in properties"); } String serializeFnSymbol = properties.get(SERIALIZE_KEY); @@ -514,7 +513,7 @@ private void analyzeUdaf() throws AnalysisException { String getValueFnSymbol = properties.get(GET_VALUE_KEY); String removeFnSymbol = properties.get(REMOVE_KEY); String symbol = properties.get(SYMBOL_KEY); - if (binaryType == TFunctionBinaryType.RPC && !userFile.contains("://")) { + if (binaryType == Function.BinaryType.RPC && !userFile.contains("://")) { if (initFnSymbol != null) { checkRPCUdf(initFnSymbol); } @@ -532,12 +531,12 @@ private void analyzeUdaf() throws AnalysisException { if (removeFnSymbol != null) { checkRPCUdf(removeFnSymbol); } - } else if (binaryType == TFunctionBinaryType.JAVA_UDF) { + } else if (binaryType == Function.BinaryType.JAVA_UDF) { if (Strings.isNullOrEmpty(symbol)) { throw new AnalysisException("No 'symbol' in properties of java-udaf"); } analyzeJavaUdaf(symbol); - } else if (binaryType == TFunctionBinaryType.PYTHON_UDF) { + } else if (binaryType == Function.BinaryType.PYTHON_UDF) { analyzePythonUdaf(symbol); } function = builder.initFnSymbol(initFnSymbol).updateFnSymbol(updateFnSymbol).mergeFnSymbol(mergeFnSymbol) @@ -562,14 +561,14 @@ private void analyzeUdf() throws AnalysisException { String closeFnSymbol = properties.get(CLOSE_SYMBOL_KEY); // TODO(yangzhg) support check function in FE when function service behind load balancer // the format for load balance can ref https://github.com/apache/incubator-brpc/blob/master/docs/en/client.md#connect-to-a-cluster - if (binaryType == TFunctionBinaryType.RPC && !userFile.contains("://")) { + if (binaryType == Function.BinaryType.RPC && !userFile.contains("://")) { if (StringUtils.isNotBlank(prepareFnSymbol) || StringUtils.isNotBlank(closeFnSymbol)) { throw new AnalysisException("prepare and close in RPC UDF are not supported."); } checkRPCUdf(symbol); - } else if (binaryType == TFunctionBinaryType.JAVA_UDF) { + } else if (binaryType == Function.BinaryType.JAVA_UDF) { analyzeJavaUdf(symbol); - } else if (binaryType == TFunctionBinaryType.PYTHON_UDF) { + } else if (binaryType == Function.BinaryType.PYTHON_UDF) { analyzePythonUdf(symbol); } URI location; @@ -1037,10 +1036,10 @@ private Types.PGenericType convertToPParameterType(Type arg) throws AnalysisExce return typeBuilder.build(); } - private TFunctionBinaryType getFunctionBinaryType(String type) { - TFunctionBinaryType binaryType = null; + private Function.BinaryType getFunctionBinaryType(String type) { + Function.BinaryType binaryType = null; try { - binaryType = TFunctionBinaryType.valueOf(type); + binaryType = Function.BinaryType.valueOf(type); } catch (IllegalArgumentException e) { // ignore enum Exception } @@ -1258,7 +1257,7 @@ private Expr makeFunctionCallExpr(Expression expression, String name, boolean ha org.apache.doris.catalog.ScalarFunction catalogFunction = new org.apache.doris.catalog.ScalarFunction( new FunctionName(name), argTypes, expression.getDataType().toCatalogDataType(), hasVarArguments, - "", TFunctionBinaryType.BUILTIN, true, true, nullableMode); + "", Function.BinaryType.BUILTIN, true, true, nullableMode); // create catalog FunctionCallExpr without analyze again return new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments), expression.nullable()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java index 9ac8ff9c1db61e..b59930964efeec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.util; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableSet; @@ -47,207 +46,6 @@ public class DateUtils { "%Y-%m-%d %T", "%Y%m%d", "%Y%m"); private static final WeekFields weekFields = WeekFields.of(DayOfWeek.SUNDAY, 7); - /** - * get the length of day of week string - */ - public static int dayOfWeekLength(int dayOfWeek) { - if (dayOfWeek == 1 || dayOfWeek == 7 || dayOfWeek == 5) { - return 6; - } else if (dayOfWeek == 2) { - return 7; - } else if (dayOfWeek == 6 || dayOfWeek == 4) { - return 8; - } else { - return 10; - } - } - - /** - * get the length of month string - */ - public static int monthLength(long month) { - if (month == 5) { - return 3; - } else if (month == 6 || month == 7) { - return 4; - } else if (month == 4 || month == 3) { - return 5; - } else if (month == 8) { - return 6; - } else if (month == 10 || month == 1) { - return 7; - } else if (month == 2 || month == 11 || month == 12) { - return 8; - } else { - return 9; - } - } - - /** - * format builder with length check. - * for some pattern, the length of result string is not fixed, - * so we need to check the length of result string to avoid parse error. - */ - public static DateTimeFormatter dateTimeFormatterChecklength(String pattern, DateTimeV2Literal date) - throws AnalysisException { - DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - boolean escaped = false; - int length = 0; - for (int i = 0; i < pattern.length(); i++) { - char character = pattern.charAt(i); - if (length > 100) { - throw new AnalysisException("Date format string is too long"); - } - if (escaped) { - switch (character) { - case 'a': // %a Abbreviated weekday name (Sun..Sat) - builder.appendText(ChronoField.DAY_OF_WEEK, TextStyle.SHORT); - length += 3; - break; - case 'b': // %b Abbreviated month name (Jan..Dec) - builder.appendText(ChronoField.MONTH_OF_YEAR, TextStyle.SHORT); - length += 3; - break; - case 'c': // %c Month, numeric (0..12) - builder.appendValue(ChronoField.MONTH_OF_YEAR); - if (date.getMonth() < 10) { - length += 1; - } else { - length += 2; - } - break; - case 'd': // %d Day of the month, numeric (00..31) - builder.appendValue(ChronoField.DAY_OF_MONTH, 2); - length += 2; - break; - case 'e': // %e Day of the month, numeric (0..31) - builder.appendValue(ChronoField.DAY_OF_MONTH); - if (date.getDay() < 10) { - length += 1; - } else { - length += 2; - } - break; - case 'H': // %H Hour (00..23) - builder.appendValue(ChronoField.HOUR_OF_DAY, 2); - length += 2; - break; - case 'h': // %h Hour (01..12) - case 'I': // %I Hour (01..12) - builder.appendValue(ChronoField.CLOCK_HOUR_OF_AMPM, 2); - length += 2; - break; - case 'i': // %i Minutes, numeric (00..59) - builder.appendValue(ChronoField.MINUTE_OF_HOUR, 2); - length += 2; - break; - case 'j': // %j Day of year (001..366) - builder.appendValue(ChronoField.DAY_OF_YEAR, 3); - length += 3; - break; - case 'k': // %k Hour (0..23) - builder.appendValue(ChronoField.HOUR_OF_DAY); - length += 2; - break; - case 'l': // %l Hour (1..12) - builder.appendValue(ChronoField.CLOCK_HOUR_OF_AMPM); - if ((date.getHour() % 24 + 11) % 12 + 1 >= 10) { - length += 2; - } else { - length += 1; - } - break; - case 'M': // %M Month name (January..December) - builder.appendText(ChronoField.MONTH_OF_YEAR, TextStyle.FULL); - length += monthLength(date.getMonth()); - break; - case 'm': // %m Month, numeric (00..12） - builder.appendValue(ChronoField.MONTH_OF_YEAR, 2); - length += 2; - break; - case 'p': // %p AM or PM - builder.appendText(ChronoField.AMPM_OF_DAY); - length += 2; - break; - case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or PM) - builder.appendValue(ChronoField.CLOCK_HOUR_OF_AMPM, 2) - .appendLiteral(':') - .appendValue(ChronoField.MINUTE_OF_HOUR, 2) - .appendLiteral(':') - .appendValue(ChronoField.SECOND_OF_MINUTE, 2) - .appendLiteral(' ') - .appendText(ChronoField.AMPM_OF_DAY, TextStyle.FULL) - .toFormatter(); - length += 11; - break; - case 'S': // %S Seconds (00..59) - case 's': // %s Seconds (00..59) - builder.appendValue(ChronoField.SECOND_OF_MINUTE, 2); - length += 2; - break; - case 'T': // %T Time, 24-hour (HH:mm:ss) - builder.appendPattern("HH:mm:ss"); - length += 8; - break; - case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X - builder.appendValue(weekFields.weekOfWeekBasedYear(), 2); - length += 2; - break; - case 'v': // %v Week (01..53), where Monday is the first day of the week; used with %x - length += 2; - builder.appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 2); - break; - case 'W': // %W Weekday name (Sunday..Saturday) - builder.appendText(ChronoField.DAY_OF_WEEK, TextStyle.FULL); - length += dayOfWeekLength(date.getDayOfWeek()); - break; - case 'x': // %x Year for the week where Monday is the first day of the week, - builder.appendValue(IsoFields.WEEK_BASED_YEAR, 4); - length += 4; - break; - case 'X': - builder.appendValue(weekFields.weekBasedYear(), 4, 10, SignStyle.EXCEEDS_PAD); - length += 4; - break; - case 'Y': // %Y Year, numeric, four digits - // %X Year for the week, where Sunday is the first day of the week, - // numeric, four digits; used with %v - builder.appendValue(ChronoField.YEAR, 4); - length += 4; - break; - case 'y': // %y Year, numeric (two digits) - builder.appendValueReduced(ChronoField.YEAR, 2, 2, 1970); - length += 2; - break; - case 'f': - builder.appendValue(ChronoField.MICRO_OF_SECOND, 6); - length += 6; - break; - // TODO(Gabriel): support microseconds in date literal - case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) - case 'U': // %U Week (00..53), where Sunday is the first day of the week - case 'u': // %u Week (00..53), where Monday is the first day of the week - case 'w': // %w Day of the week (0=Sunday..6=Saturday) - throw new AnalysisException(String.format("%%%s not supported in date format string", - character)); - case '%': // %% A literal "%" character - builder.appendLiteral('%'); - break; - default: // % The literal character represented by - builder.appendLiteral(character); - length += 1; - break; - } - escaped = false; - } else if (character == '%') { - escaped = true; - } else { - builder.appendLiteral(character); - } - } - return builder.toFormatter(Locale.US).withResolverStyle(ResolverStyle.STRICT); - } - /** * format builder. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 15250aecc8a471..30140635249d61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -127,7 +127,6 @@ import org.apache.doris.cloud.load.CloudBrokerLoadJob; import org.apache.doris.cloud.load.CopyJob; import org.apache.doris.common.Config; -import org.apache.doris.common.io.Text; import org.apache.doris.common.util.RangeUtils; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalDatabase; @@ -221,56 +220,11 @@ import org.apache.doris.system.HeartbeatResponse; import org.apache.doris.transaction.TxnCommitAttachment; -import com.google.common.base.Preconditions; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.LinkedHashMultimap; -import com.google.common.collect.LinkedListMultimap; -import com.google.common.collect.Multimap; import com.google.common.collect.Range; -import com.google.common.collect.Table; import com.google.gson.ExclusionStrategy; import com.google.gson.FieldAttributes; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; -import com.google.gson.ReflectionAccessFilter; -import com.google.gson.ToNumberPolicy; -import com.google.gson.TypeAdapter; -import com.google.gson.TypeAdapterFactory; -import com.google.gson.annotations.SerializedName; -import com.google.gson.reflect.TypeToken; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonWriter; -import org.apache.commons.lang3.reflect.TypeUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.lang.reflect.Method; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; /* * Some utilities about Gson. @@ -614,16 +568,7 @@ public class GsonUtils { // Since GsonBuilder.create() adds all registered factories to GSON in reverse order, if you // need to ensure the search order of two RuntimeTypeAdapterFactory instances, be sure to // register them in reverse priority order. - private static final GsonBuilder GSON_BUILDER = new GsonBuilder() - .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) - .addSerializationExclusionStrategy( - new HiddenAnnotationExclusionStrategy()).serializeSpecialFloatingPointValues() - .enableComplexMapKeySerialization() - .addReflectionAccessFilter(ReflectionAccessFilter.BLOCK_INACCESSIBLE_JAVA) - .registerTypeHierarchyAdapter(Table.class, new GuavaTableAdapter()) - .registerTypeHierarchyAdapter(Multimap.class, new GuavaMultimapAdapter()) - .registerTypeAdapterFactory(new PostProcessTypeAdapterFactory()) - .registerTypeAdapterFactory(new PreProcessTypeAdapterFactory()) + private static final GsonBuilder GSON_BUILDER = GsonUtilsBase.GSON_BUILDER .registerTypeAdapterFactory(exprAdapterFactory) .registerTypeAdapterFactory(columnTypeAdapterFactory) .registerTypeAdapterFactory(distributionInfoTypeAdapterFactory) @@ -649,9 +594,6 @@ public class GsonUtils { .registerTypeAdapterFactory(jobBackupTypeAdapterFactory) .registerTypeAdapterFactory(loadJobTypeAdapterFactory) .registerTypeAdapterFactory(partitionItemTypeAdapterFactory) - .registerTypeAdapter(ImmutableMap.class, new ImmutableMapDeserializer()) - .registerTypeAdapter(ImmutableList.class, new ImmutableListDeserializer()) - .registerTypeAdapter(AtomicBoolean.class, new AtomicBooleanAdapter()) .registerTypeAdapter(PartitionKey.class, new PartitionKey.PartitionKeySerializer()) .registerTypeAdapter(Range.class, new RangeUtils.RangeSerializer()).setExclusionStrategies( new ExclusionStrategy() { @@ -669,321 +611,10 @@ public boolean shouldSkipClass(Class clazz) { } }); - // this instance is thread-safe. public static final Gson GSON = GSON_BUILDER.create(); // ATTN: the order between creating GSON and GSON_PRETTY_PRINTING is very important. private static final GsonBuilder GSON_BUILDER_PRETTY_PRINTING = GSON_BUILDER.setPrettyPrinting(); public static final Gson GSON_PRETTY_PRINTING = GSON_BUILDER_PRETTY_PRINTING.create(); - - /* - * The exclusion strategy of GSON serialization. - * Any fields without "@SerializedName" annotation with be ignore with - * serializing and deserializing. - */ - public static class HiddenAnnotationExclusionStrategy implements ExclusionStrategy { - public boolean shouldSkipField(FieldAttributes f) { - return f.getAnnotation(SerializedName.class) == null; - } - - @Override - public boolean shouldSkipClass(Class clazz) { - return false; - } - } - - /* - * - * The json adapter for Guava Table. - * Current support: - * 1. HashBasedTable - * - * The RowKey, ColumnKey and Value classes in Table should also be serializable. - * - * What is Adapter and Why we should implement it? - * - * Adapter is mainly used to provide serialization and deserialization methods for some complex classes. - * Complex classes here usually refer to classes that are complex and cannot be modified. - * These classes mainly include third-party library classes or some inherited classes. - */ - private static class GuavaTableAdapter - implements JsonSerializer>, JsonDeserializer> { - /* - * serialize Table as: - * { - * "rowKeys": [ "rowKey1", "rowKey2", ...], - * "columnKeys": [ "colKey1", "colKey2", ...], - * "cells" : [[0, 0, value1], [0, 1, value2], ...] - * } - * - * the [0, 0] .. in cells are the indexes of rowKeys array and columnKeys array. - * This serialization method can reduce the size of json string because it - * replace the same row key - * and column key to integer. - */ - @Override - public JsonElement serialize(Table src, Type typeOfSrc, JsonSerializationContext context) { - JsonArray rowKeysJsonArray = new JsonArray(); - Map rowKeyToIndex = new HashMap<>(); - for (R rowKey : src.rowKeySet()) { - rowKeyToIndex.put(rowKey, rowKeyToIndex.size()); - rowKeysJsonArray.add(context.serialize(rowKey)); - } - JsonArray columnKeysJsonArray = new JsonArray(); - Map columnKeyToIndex = new HashMap<>(); - for (C columnKey : src.columnKeySet()) { - columnKeyToIndex.put(columnKey, columnKeyToIndex.size()); - columnKeysJsonArray.add(context.serialize(columnKey)); - } - JsonArray cellsJsonArray = new JsonArray(); - for (Table.Cell cell : src.cellSet()) { - int rowIndex = rowKeyToIndex.get(cell.getRowKey()); - int columnIndex = columnKeyToIndex.get(cell.getColumnKey()); - cellsJsonArray.add(rowIndex); - cellsJsonArray.add(columnIndex); - cellsJsonArray.add(context.serialize(cell.getValue())); - } - JsonObject tableJsonObject = new JsonObject(); - tableJsonObject.addProperty("clazz", src.getClass().getSimpleName()); - tableJsonObject.add("rowKeys", rowKeysJsonArray); - tableJsonObject.add("columnKeys", columnKeysJsonArray); - tableJsonObject.add("cells", cellsJsonArray); - return tableJsonObject; - } - - @Override - public Table deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) { - Type typeOfR; - Type typeOfC; - Type typeOfV; - { // CHECKSTYLE IGNORE THIS LINE - ParameterizedType parameterizedType = (ParameterizedType) typeOfT; - Type[] actualTypeArguments = parameterizedType.getActualTypeArguments(); - typeOfR = actualTypeArguments[0]; - typeOfC = actualTypeArguments[1]; - typeOfV = actualTypeArguments[2]; - } // CHECKSTYLE IGNORE THIS LINE - JsonObject tableJsonObject = json.getAsJsonObject(); - String tableClazz = tableJsonObject.get("clazz").getAsString(); - JsonArray rowKeysJsonArray = tableJsonObject.getAsJsonArray("rowKeys"); - Map rowIndexToKey = new HashMap<>(); - for (JsonElement jsonElement : rowKeysJsonArray) { - R rowKey = context.deserialize(jsonElement, typeOfR); - rowIndexToKey.put(rowIndexToKey.size(), rowKey); - } - JsonArray columnKeysJsonArray = tableJsonObject.getAsJsonArray("columnKeys"); - Map columnIndexToKey = new HashMap<>(); - for (JsonElement jsonElement : columnKeysJsonArray) { - C columnKey = context.deserialize(jsonElement, typeOfC); - columnIndexToKey.put(columnIndexToKey.size(), columnKey); - } - JsonArray cellsJsonArray = tableJsonObject.getAsJsonArray("cells"); - Table table = null; - switch (tableClazz) { - case "HashBasedTable": - table = HashBasedTable.create(); - break; - default: - Preconditions.checkState(false, "unknown guava table class: " + tableClazz); - break; - } - for (int i = 0; i < cellsJsonArray.size(); i = i + 3) { - // format is [rowIndex, columnIndex, value] - int rowIndex = cellsJsonArray.get(i).getAsInt(); - int columnIndex = cellsJsonArray.get(i + 1).getAsInt(); - R rowKey = rowIndexToKey.get(rowIndex); - C columnKey = columnIndexToKey.get(columnIndex); - V value = context.deserialize(cellsJsonArray.get(i + 2), typeOfV); - table.put(rowKey, columnKey, value); - } - return table; - } - } - - /* - * The json adapter for Guava Multimap. - * Current support: - * 1. ArrayListMultimap - * 2. HashMultimap - * 3. LinkedListMultimap - * 4. LinkedHashMultimap - * - * The key and value classes of multi map should also be json serializable. - */ - private static class GuavaMultimapAdapter - implements JsonSerializer>, JsonDeserializer> { - - private static final Type asMapReturnType = getAsMapMethod().getGenericReturnType(); - - private static Type asMapType(Type multimapType) { - return com.google.common.reflect.TypeToken.of(multimapType).resolveType(asMapReturnType).getType(); - } - - private static Method getAsMapMethod() { - try { - return Multimap.class.getDeclaredMethod("asMap"); - } catch (NoSuchMethodException e) { - throw new AssertionError(e); - } - } - - @Override - public JsonElement serialize(Multimap map, Type typeOfSrc, JsonSerializationContext context) { - JsonObject jsonObject = new JsonObject(); - jsonObject.addProperty("clazz", map.getClass().getSimpleName()); - Map> asMap = map.asMap(); - Type type = asMapType(typeOfSrc); - JsonElement jsonElement = context.serialize(asMap, type); - jsonObject.add("map", jsonElement); - return jsonObject; - } - - @Override - public Multimap deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) - throws JsonParseException { - JsonObject jsonObject = json.getAsJsonObject(); - String clazz = jsonObject.get("clazz").getAsString(); - - JsonElement mapElement = jsonObject.get("map"); - Map> asMap = context.deserialize(mapElement, asMapType(typeOfT)); - - Multimap map = null; - switch (clazz) { - case "ArrayListMultimap": - map = ArrayListMultimap.create(); - break; - case "HashMultimap": - map = HashMultimap.create(); - break; - case "LinkedListMultimap": - map = LinkedListMultimap.create(); - break; - case "LinkedHashMultimap": - map = LinkedHashMultimap.create(); - break; - default: - Preconditions.checkState(false, "unknown guava multi map class: " + clazz); - break; - } - - for (Map.Entry> entry : asMap.entrySet()) { - map.putAll(entry.getKey(), entry.getValue()); - } - return map; - } - } - - private static class AtomicBooleanAdapter - implements JsonSerializer, JsonDeserializer { - - @Override - public AtomicBoolean deserialize(JsonElement jsonElement, Type type, - JsonDeserializationContext jsonDeserializationContext) - throws JsonParseException { - JsonObject jsonObject = jsonElement.getAsJsonObject(); - boolean value = jsonObject.get("boolean").getAsBoolean(); - return new AtomicBoolean(value); - } - - @Override - public JsonElement serialize(AtomicBoolean atomicBoolean, Type type, - JsonSerializationContext jsonSerializationContext) { - JsonObject jsonObject = new JsonObject(); - jsonObject.addProperty("boolean", atomicBoolean.get()); - return jsonObject; - } - } - - public static final class ImmutableMapDeserializer implements JsonDeserializer> { - @Override - public ImmutableMap deserialize(final JsonElement json, final Type type, - final JsonDeserializationContext context) throws JsonParseException { - final Type type2 = TypeUtils.parameterize(Map.class, ((ParameterizedType) type).getActualTypeArguments()); - final Map map = context.deserialize(json, type2); - return ImmutableMap.copyOf(map); - } - } - - public static final class ImmutableListDeserializer implements JsonDeserializer> { - @Override - public ImmutableList deserialize(final JsonElement json, final Type type, - final JsonDeserializationContext context) throws JsonParseException { - final Type type2 = TypeUtils.parameterize(List.class, ((ParameterizedType) type).getActualTypeArguments()); - final List list = context.deserialize(json, type2); - return ImmutableList.copyOf(list); - } - } - - public static class PreProcessTypeAdapterFactory implements TypeAdapterFactory { - - public PreProcessTypeAdapterFactory() { - } - - @Override - public TypeAdapter create(Gson gson, TypeToken type) { - TypeAdapter delegate = gson.getDelegateAdapter(this, type); - - return new TypeAdapter() { - public void write(JsonWriter out, T value) throws IOException { - if (value instanceof GsonPreProcessable) { - ((GsonPreProcessable) value).gsonPreProcess(); - } - delegate.write(out, value); - } - - public T read(JsonReader reader) throws IOException { - return delegate.read(reader); - } - }; - } - } - - public static class PostProcessTypeAdapterFactory implements TypeAdapterFactory { - - public PostProcessTypeAdapterFactory() { - } - - @Override - public TypeAdapter create(Gson gson, TypeToken type) { - TypeAdapter delegate = gson.getDelegateAdapter(this, type); - - return new TypeAdapter() { - public void write(JsonWriter out, T value) throws IOException { - delegate.write(out, value); - } - - public T read(JsonReader reader) throws IOException { - T obj = delegate.read(reader); - if (obj instanceof GsonPostProcessable) { - ((GsonPostProcessable) obj).gsonPostProcess(); - } - return obj; - } - }; - } - } - - public static void toJsonCompressed(DataOutput out, Object src) throws IOException { - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - try (GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { - try (OutputStreamWriter writer = new OutputStreamWriter(gzipStream)) { - GsonUtils.GSON.toJson(src, writer); - } - } - Text text = new Text(byteStream.toByteArray()); - text.write(out); - } - - public static T fromJsonCompressed(DataInput in, Class clazz) throws IOException { - Text text = new Text(); - text.readFields(in); - - ByteArrayInputStream byteStream = new ByteArrayInputStream(text.getBytes()); - try (GZIPInputStream gzipStream = new GZIPInputStream(byteStream)) { - try (InputStreamReader reader = new InputStreamReader(gzipStream)) { - return GsonUtils.GSON.fromJson(reader, clazz); - } - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 699a4c31d499df..6ef037f61e34c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -418,7 +418,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) throws An if (column.getName().startsWith(Column.SHADOW_NAME_PREFIX)) { tColumn.setColumnName(column.getNonShadowName()); } - column.setIndexFlag(tColumn, table); + column.setIndexFlag(tColumn, table.getCopiedBfColumns()); columnsDesc.add(tColumn); } List indexes = indexMeta.getIndexes(); @@ -704,7 +704,7 @@ public static void setPartitionKeys(TOlapTablePartition tPartition, PartitionIte List tExprNodes = new ArrayList<>(); for (int i = 0; i < partColNum; i++) { LiteralExpr literalExpr = partitionKey.getKeys().get(i); - if (literalExpr.isNullLiteral()) { + if (literalExpr instanceof NullLiteral) { tExprNodes.add(ExprToThriftVisitor.treeToThrift( NullLiteral.create(literalExpr.getType())).getNodes().get(0)); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index bc7b91d5036354..b8f4ae4973b162 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -24,11 +24,8 @@ import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ExprToThriftVisitor; import org.apache.doris.analysis.JoinOperator; -import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.ToSqlParams; -import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.TreeNode; import org.apache.doris.nereids.trees.plans.distribute.NereidsSpecifyInstances; import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanSource; @@ -41,7 +38,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Suppliers; -import com.google.common.collect.Lists; import org.apache.commons.codec.binary.Hex; import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.LogManager; @@ -255,14 +251,6 @@ public void setOutputExprs(List outputExprs) { this.outputExprs = Expr.cloneList(outputExprs, null); } - public void resetOutputExprs(TupleDescriptor tupleDescriptor) { - this.outputExprs = Lists.newArrayList(); - for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) { - SlotRef slotRef = new SlotRef(slotDescriptor); - outputExprs.add(slotRef); - } - } - public ArrayList getOutputExprs() { return outputExprs; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 499b6086e66c38..446d25e47075f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -21,6 +21,7 @@ package org.apache.doris.planner; import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.CompoundPredicate; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.InPredicate; @@ -300,7 +301,7 @@ public static ColumnRanges expressionToRanges(Expr expr, return ColumnRanges.createFailure(); } - if (!(inPredicate.getChild(0).unwrapExpr(false) instanceof SlotRef)) { + if (!(unwrapCast(inPredicate.getChild(0)) instanceof SlotRef)) { // If child(0) of the in predicate is not a SlotRef, // then other children of in predicate should not be used as a condition for partition prune. return ColumnRanges.createFailure(); @@ -339,6 +340,16 @@ public static ColumnRanges expressionToRanges(Expr expr, } } + /** + * Returns the first child if this Expr is a CastExpr. Otherwise, returns 'this'. + */ + private static Expr unwrapCast(Expr root) { + if (root instanceof CastExpr) { + return root.getChild(0); + } + return root; + } + private PartitionColumnFilter createPartitionFilter(SlotDescriptor desc, List conjuncts, PartitionInfo partitionsInfo) { PartitionColumnFilter partitionColumnFilter = null; @@ -395,7 +406,7 @@ private PartitionColumnFilter createPartitionFilter(SlotDescriptor desc, List mockLiteralExpr() throws AnalysisException { .add(new StringLiteral("trino connector varchar test")) .add(new StringLiteral("trino connector varbinary test")) - .add(new DateLiteral("1969-12-31", Type.DATEV2)) - .add(new DateLiteral("1970-01-01 00:00:01.000001", Type.DATETIMEV2)) + .add(new DateLiteral(1969, 12, 31, Type.DATEV2)) + .add(new DateLiteral(1970, 1, 1, 0, 0, 1, 1, Type.DATETIMEV2)) // .add(new DateLiteral("1970-01-01 00:00:00.000000", Type.DATETIMEV2)) - .add(new DateLiteral("1970-01-01 00:00:01.000001", Type.DATETIMEV2)) - .add(new DateLiteral("1970-01-01 08:00:01.000001", Type.DATETIMEV2)) + .add(new DateLiteral(1970, 1, 1, 0, 0, 1, 1, Type.DATETIMEV2)) + .add(new DateLiteral(1970, 1, 1, 8, 0, 1, 1, Type.DATETIMEV2)) .build(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java index 58dce041fb6d0b..5e99b5e6e2c2e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java @@ -32,7 +32,7 @@ public class MTMVUtilTest { @Test public void testGetExprTimeSec() throws AnalysisException { - LiteralExpr expr = new DateLiteral("2020-01-01"); + LiteralExpr expr = new DateLiteral(2020, 1, 1); long exprTimeSec = MTMVUtil.getExprTimeSec(expr, Optional.empty()); Assert.assertEquals(1577808000L, exprTimeSec); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java index c2421155d1a62f..b35fb459795ea5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/gson/GsonDerivedClassSerializationTest.java @@ -19,8 +19,8 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.persist.gson.GsonUtils.HiddenAnnotationExclusionStrategy; -import org.apache.doris.persist.gson.GsonUtils.PostProcessTypeAdapterFactory; +import org.apache.doris.persist.gson.GsonUtilsBase.HiddenAnnotationExclusionStrategy; +import org.apache.doris.persist.gson.GsonUtilsBase.PostProcessTypeAdapterFactory; import com.google.common.collect.Maps; import com.google.gson.Gson; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PlanNodeTest.java index a065a186e7e8d6..99acecb0e22b20 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlanNodeTest.java @@ -66,12 +66,22 @@ public void testGetPointQueryProjectListFlattensIntermediateProjection() { Assertions.assertEquals(1, pointQueryProjectList.size()); Set slotIds = new HashSet<>(); - Expr.extractSlots(pointQueryProjectList.get(0), slotIds); + extractSlots(pointQueryProjectList.get(0), slotIds); Assertions.assertEquals(Sets.newHashSet(scanSlotA.getId(), scanSlotB.getId()), slotIds); Assertions.assertFalse(slotIds.contains(projectedSlotA.getId())); Assertions.assertFalse(slotIds.contains(projectedExprSlot.getId())); } + static void extractSlots(Expr root, Set slotIdSet) { + if (root instanceof SlotRef) { + slotIdSet.add(((SlotRef) root).getDesc().getId()); + return; + } + for (Expr child : root.getChildren()) { + extractSlots(child, slotIdSet); + } + } + private static SlotDescriptor createColumnSlot(TupleDescriptor tupleDescriptor, int slotId, String columnName) { SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(slotId), tupleDescriptor);