From cffeb4c9f63d926c7c06d0131a0b7065769a2883 Mon Sep 17 00:00:00 2001 From: SreeramaYeshwanthGowd Date: Sat, 1 Aug 2026 01:58:29 +0530 Subject: [PATCH 1/2] [SPARK-45900][SQL][PYTHON] Add xxh3_64 and xxh3_128 functions --- .../reference/pyspark.sql/functions.rst | 2 + .../pyspark/sql/connect/functions/builtin.py | 14 + python/pyspark/sql/functions/__init__.py | 2 + python/pyspark/sql/functions/builtin.py | 65 +++ .../org/apache/spark/sql/functions.scala | 24 + .../spark/sql/catalyst/expressions/XXH3.java | 421 ++++++++++++++++++ .../catalyst/analysis/FunctionRegistry.scala | 2 + .../spark/sql/catalyst/expressions/hash.scala | 72 +++ .../expressions/HashExpressionsSuite.scala | 11 + .../sql/catalyst/expressions/XXH3Suite.scala | 141 ++++++ .../spark/sql/PlanGenerationTestSuite.scala | 8 + .../explain-results/function_xxh3_128.explain | 2 + .../explain-results/function_xxh3_64.explain | 2 + .../queries/function_xxh3_128.json | 85 ++++ .../queries/function_xxh3_128.proto.bin | Bin 0 -> 722 bytes .../query-tests/queries/function_xxh3_64.json | 85 ++++ .../queries/function_xxh3_64.proto.bin | Bin 0 -> 720 bytes .../sql-functions/sql-expression-schema.md | 2 + .../analyzer-results/misc-functions.sql.out | 42 ++ .../sql-tests/inputs/misc-functions.sql | 9 + .../sql-tests/results/misc-functions.sql.out | 48 ++ 21 files changed, 1037 insertions(+) create mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XXH3Suite.scala create mode 100644 sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_128.explain create mode 100644 sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_64.explain create mode 100644 sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_128.json create mode 100644 sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_128.proto.bin create mode 100644 sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_64.json create mode 100644 sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_64.proto.bin diff --git a/python/docs/source/reference/pyspark.sql/functions.rst b/python/docs/source/reference/pyspark.sql/functions.rst index 7ce413fde0bb9..5ffc9e9d74cc3 100644 --- a/python/docs/source/reference/pyspark.sql/functions.rst +++ b/python/docs/source/reference/pyspark.sql/functions.rst @@ -349,6 +349,8 @@ Hash Functions sha sha1 sha2 + xxh3_128 + xxh3_64 xxhash64 diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py index acf1ee6b069fa..ab73390c30dbf 100644 --- a/python/pyspark/sql/connect/functions/builtin.py +++ b/python/pyspark/sql/connect/functions/builtin.py @@ -4674,6 +4674,20 @@ def md5(col: "ColumnOrName") -> Column: md5.__doc__ = pysparkfuncs.md5.__doc__ +def xxh3_64(col: "ColumnOrName") -> Column: + return _invoke_function_over_columns("xxh3_64", col) + + +xxh3_64.__doc__ = pysparkfuncs.xxh3_64.__doc__ + + +def xxh3_128(col: "ColumnOrName") -> Column: + return _invoke_function_over_columns("xxh3_128", col) + + +xxh3_128.__doc__ = pysparkfuncs.xxh3_128.__doc__ + + def sha1(col: "ColumnOrName") -> Column: return _invoke_function_over_columns("sha1", col) diff --git a/python/pyspark/sql/functions/__init__.py b/python/pyspark/sql/functions/__init__.py index 7780bae38ee94..70fbbae134b6b 100644 --- a/python/pyspark/sql/functions/__init__.py +++ b/python/pyspark/sql/functions/__init__.py @@ -292,6 +292,8 @@ "sha", "sha1", "sha2", + "xxh3_128", + "xxh3_64", "xxhash64", # Collection Functions "aggregate", diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 3cd42dc2e0e28..4d999e5cb4a93 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -14697,6 +14697,71 @@ def md5(col: "ColumnOrName") -> Column: return _invoke_function_over_columns("md5", col) +@_try_remote_functions +def xxh3_64(col: "ColumnOrName") -> Column: + """Returns a 64-bit hash value of the argument using the XXH3 algorithm. + + Unlike :func:`xxhash64`, which hashes one or more columns structurally, this hashes the raw + bytes of a single value with seed 0, so its result is byte compatible with the reference XXH3. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. A column of string or binary type. + + Returns + ------- + :class:`~pyspark.sql.Column` + Returns a column that evaluates to a long. + + See Also + -------- + :meth:`pyspark.sql.functions.xxh3_128` + :meth:`pyspark.sql.functions.xxhash64` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark',)], ['a']) + >>> df.select(sf.xxh3_64('a').alias('h')).collect() + [Row(h=80997306238743657)] + """ + return _invoke_function_over_columns("xxh3_64", col) + + +@_try_remote_functions +def xxh3_128(col: "ColumnOrName") -> Column: + """Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. A column of string or binary type. + + Returns + ------- + :class:`~pyspark.sql.Column` + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.xxh3_64` + :meth:`pyspark.sql.functions.md5` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark',)], ['a']) + >>> df.select(sf.xxh3_128('a').alias('h')).collect() + [Row(h='7d57dd84c60c86ca1f4e82ab91a12b5e')] + """ + return _invoke_function_over_columns("xxh3_128", col) + + @_try_remote_functions def sha1(col: "ColumnOrName") -> Column: """Returns the hex string result of SHA-1. diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 8b8cd98960e0b..5e2105e4c7f02 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -6551,6 +6551,30 @@ object functions { @scala.annotation.varargs def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) + /** + * Returns a 64-bit hash value of the argument using the XXH3 algorithm. + * + * @param col + * the column to hash. A column of string or binary type. + * @group hash_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a long. + */ + def xxh3_64(col: Column): Column = Column.fn("xxh3_64", col) + + /** + * Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. + * + * @param col + * the column to hash. A column of string or binary type. + * @group hash_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. + */ + def xxh3_128(col: Column): Column = Column.fn("xxh3_128", col) + /** * Returns null if the condition is true, and throws an exception otherwise. * diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java new file mode 100644 index 0000000000000..d2637adfd5818 --- /dev/null +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions; + +import java.util.HexFormat; + +import org.apache.spark.unsafe.types.UTF8String; + +/** + * A Java port of the XXH3 64-bit and 128-bit hash functions (from the reference implementation at + * https://github.com/Cyan4973/xxHash, as specified in doc/xxhash_spec.md). The output is byte + * compatible with the reference implementation, so it matches `xxhsum` and other XXH3 tools. + * + *

All arithmetic is on 64-bit lanes; Java's signed {@code long} is used as an unsigned 64-bit + * integer throughout ({@code >>>} for logical shift, {@link Long#rotateLeft}, and + * {@link #unsignedMultiplyHigh} for the high half of 64x64 products). + */ +public final class XXH3 { + + private XXH3() {} + + private static final long PRIME32_1 = 0x9E3779B1L; + private static final long PRIME32_2 = 0x85EBCA77L; + private static final long PRIME32_3 = 0xC2B2AE3DL; + private static final long PRIME64_1 = 0x9E3779B185EBCA87L; + private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL; + private static final long PRIME64_3 = 0x165667B19E3779F9L; + private static final long PRIME64_4 = 0x85EBCA77C2B2AE63L; + private static final long PRIME64_5 = 0x27D4EB2F165667C5L; + private static final long PRIME_MX1 = 0x165667919E3779F9L; + private static final long PRIME_MX2 = 0x9FB21C651E98DF25L; + + private static final int SECRET_SIZE = 192; + private static final int SECRET_SIZE_MIN = 136; + private static final int STRIPE_LEN = 64; + private static final int SECRET_MERGEACCS_START = 11; + private static final int SECRET_LASTACC_START = 7; + private static final int NB_STRIPES_PER_BLOCK = (SECRET_SIZE - STRIPE_LEN) / 8; + private static final int BLOCK_LEN = STRIPE_LEN * NB_STRIPES_PER_BLOCK; + + // The default 192-byte secret from the reference implementation. + private static final byte[] SECRET = HexFormat.of().parseHex( + "b8fe6c3923a44bbe7c01812cf721ad1c" + + "ded46de9839097db7240a4a4b7b3671f" + + "cb79e64eccc0e578825ad07dccff7221" + + "b8084674f743248ee03590e6813a264c" + + "3c2852bb91c300cb88d0658b1b532ea3" + + "71644897a20df94e3819ef46a9deacd8" + + "a8fa763fe39c343ff9dcbbc7c70b4f1d" + + "8a51e04bcdb45931c89f7ec9d9787364" + + "eac5ac8334d3ebc3c581a0fffa1363eb" + + "170ddd51b7f0da49d316552629d4689e" + + "2b16be587d47a1fc8ff8b8d17ad031ce" + + "45cb3a8f95160428afd7fbcabb4b407e"); + + // ---- little-endian reads / writes ---- + + private static long readLE64(byte[] data, int offset) { + return (data[offset] & 0xFFL) + | ((data[offset + 1] & 0xFFL) << 8) + | ((data[offset + 2] & 0xFFL) << 16) + | ((data[offset + 3] & 0xFFL) << 24) + | ((data[offset + 4] & 0xFFL) << 32) + | ((data[offset + 5] & 0xFFL) << 40) + | ((data[offset + 6] & 0xFFL) << 48) + | ((data[offset + 7] & 0xFFL) << 56); + } + + // Reads 4 little-endian bytes as an unsigned 32-bit value (in the low 32 bits of the result). + private static long readLE32(byte[] data, int offset) { + return (data[offset] & 0xFFL) + | ((data[offset + 1] & 0xFFL) << 8) + | ((data[offset + 2] & 0xFFL) << 16) + | ((data[offset + 3] & 0xFFL) << 24); + } + + private static void writeLE64(byte[] data, int offset, long value) { + for (int i = 0; i < 8; i++) { + data[offset + i] = (byte) (value >>> (8 * i)); + } + } + + // ---- mixing helpers ---- + + private static long xxh64Avalanche(long h) { + h ^= h >>> 33; + h *= PRIME64_2; + h ^= h >>> 29; + h *= PRIME64_3; + h ^= h >>> 32; + return h; + } + + private static long xxh3Avalanche(long h) { + h ^= h >>> 37; + h *= PRIME_MX1; + h ^= h >>> 32; + return h; + } + + // Unsigned high 64 bits of the 128-bit product a*b (Math.multiplyHigh is signed). + private static long unsignedMultiplyHigh(long a, long b) { + return Math.multiplyHigh(a, b) + ((a >> 63) & b) + ((b >> 63) & a); + } + + // Low 64 bits XOR high 64 bits of the 128-bit product a*b. + private static long mul128Fold64(long a, long b) { + return (a * b) ^ unsignedMultiplyHigh(a, b); + } + + private static long mix16B(byte[] input, int inOff, int secOff, long seed) { + long lo = readLE64(input, inOff) ^ (readLE64(SECRET, secOff) + seed); + long hi = readLE64(input, inOff + 8) ^ (readLE64(SECRET, secOff + 8) - seed); + return mul128Fold64(lo, hi); + } + + // ---- XXH3 64-bit ---- + + public static long hash64(byte[] input, long seed) { + int len = input.length; + if (len <= 16) { + if (len > 8) { + return len9to16(input, len, seed); + } else if (len >= 4) { + return len4to8(input, len, seed); + } else if (len > 0) { + return len1to3(input, len, seed); + } + return xxh64Avalanche(seed ^ readLE64(SECRET, 56) ^ readLE64(SECRET, 64)); + } else if (len <= 128) { + return len17to128(input, len, seed); + } else if (len <= 240) { + return len129to240(input, len, seed); + } + return hashLong(input, len, seed); + } + + private static long len1to3(byte[] input, int len, long seed) { + long combined = ((input[0] & 0xFFL) << 16) + | ((input[len >> 1] & 0xFFL) << 24) + | (input[len - 1] & 0xFFL) + | ((long) len << 8); + long flip = (readLE32(SECRET, 0) ^ readLE32(SECRET, 4)) + seed; + return xxh64Avalanche(combined ^ flip); + } + + private static long len4to8(byte[] input, int len, long seed) { + seed ^= ((long) Integer.reverseBytes((int) seed) & 0xFFFFFFFFL) << 32; + long in1 = readLE32(input, 0); + long in2 = readLE32(input, len - 4); + long combined = in2 | (in1 << 32); + long flip = (readLE64(SECRET, 8) ^ readLE64(SECRET, 16)) - seed; + long x = combined ^ flip; + x ^= Long.rotateLeft(x, 49) ^ Long.rotateLeft(x, 24); + x *= PRIME_MX2; + x ^= (x >>> 35) + len; + x *= PRIME_MX2; + x ^= x >>> 28; + return x; + } + + private static long len9to16(byte[] input, int len, long seed) { + long flip1 = (readLE64(SECRET, 24) ^ readLE64(SECRET, 32)) + seed; + long flip2 = (readLE64(SECRET, 40) ^ readLE64(SECRET, 48)) - seed; + long in1 = readLE64(input, 0) ^ flip1; + long in2 = readLE64(input, len - 8) ^ flip2; + long acc = len + Long.reverseBytes(in1) + in2 + mul128Fold64(in1, in2); + return xxh3Avalanche(acc); + } + + private static long len17to128(byte[] input, int len, long seed) { + long acc = (long) len * PRIME64_1; + for (int i = (len - 1) >> 5; i >= 0; i--) { + acc += mix16B(input, 16 * i, 32 * i, seed); + acc += mix16B(input, len - 16 * (i + 1), 32 * i + 16, seed); + } + return xxh3Avalanche(acc); + } + + private static long len129to240(byte[] input, int len, long seed) { + long acc = (long) len * PRIME64_1; + int nbRounds = len / 16; + for (int i = 0; i < 8; i++) { + acc += mix16B(input, 16 * i, 16 * i, seed); + } + acc = xxh3Avalanche(acc); + for (int i = 8; i < nbRounds; i++) { + acc += mix16B(input, 16 * i, 16 * (i - 8) + 3, seed); + } + acc += mix16B(input, len - 16, SECRET_SIZE_MIN - 17, seed); + return xxh3Avalanche(acc); + } + + // ---- long input (> 240 bytes) ---- + + private static byte[] customSecret(long seed) { + if (seed == 0) { + return SECRET; + } + byte[] secret = new byte[SECRET_SIZE]; + for (int i = 0; i < SECRET_SIZE / 16; i++) { + writeLE64(secret, 16 * i, readLE64(SECRET, 16 * i) + seed); + writeLE64(secret, 16 * i + 8, readLE64(SECRET, 16 * i + 8) - seed); + } + return secret; + } + + private static void accumulate512( + long[] acc, byte[] input, int inOff, byte[] secret, int secOff) { + for (int i = 0; i < 8; i++) { + long data = readLE64(input, inOff + 8 * i); + long key = data ^ readLE64(secret, secOff + 8 * i); + acc[i ^ 1] += data; + acc[i] += (key & 0xFFFFFFFFL) * (key >>> 32); + } + } + + private static void scrambleAcc(long[] acc, byte[] secret, int secOff) { + for (int i = 0; i < 8; i++) { + acc[i] ^= acc[i] >>> 47; + acc[i] ^= readLE64(secret, secOff + 8 * i); + acc[i] *= PRIME32_1; + } + } + + private static long mergeAccs(long[] acc, byte[] secret, int secOff, long start) { + long result = start; + for (int i = 0; i < 4; i++) { + long a0 = acc[2 * i] ^ readLE64(secret, secOff + 16 * i); + long a1 = acc[2 * i + 1] ^ readLE64(secret, secOff + 16 * i + 8); + result += mul128Fold64(a0, a1); + } + return xxh3Avalanche(result); + } + + private static long[] hashLongAccumulate(byte[] input, int len, byte[] secret) { + long[] acc = {PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3, PRIME64_4, PRIME32_2, PRIME64_5, + PRIME32_1}; + int nbBlocks = (len - 1) / BLOCK_LEN; + for (int n = 0; n < nbBlocks; n++) { + for (int s = 0; s < NB_STRIPES_PER_BLOCK; s++) { + accumulate512(acc, input, n * BLOCK_LEN + s * STRIPE_LEN, secret, s * 8); + } + scrambleAcc(acc, secret, SECRET_SIZE - STRIPE_LEN); + } + int nbStripes = ((len - 1) - BLOCK_LEN * nbBlocks) / STRIPE_LEN; + for (int s = 0; s < nbStripes; s++) { + accumulate512(acc, input, nbBlocks * BLOCK_LEN + s * STRIPE_LEN, secret, s * 8); + } + accumulate512( + acc, input, len - STRIPE_LEN, secret, SECRET_SIZE - STRIPE_LEN - SECRET_LASTACC_START); + return acc; + } + + private static long hashLong(byte[] input, int len, long seed) { + byte[] secret = customSecret(seed); + long[] acc = hashLongAccumulate(input, len, secret); + return mergeAccs(acc, secret, SECRET_MERGEACCS_START, (long) len * PRIME64_1); + } + + /** Hashes the input with the default seed 0. */ + public static long hash64(byte[] input) { + return hash64(input, 0L); + } + + // ---- XXH3 128-bit ---- + + private static long mult32to64(long a, long b) { + return (a & 0xFFFFFFFFL) * (b & 0xFFFFFFFFL); + } + + private static void mix32B(long[] acc, byte[] input, int in1, int in2, int secOff, long seed) { + acc[0] += mix16B(input, in1, secOff, seed); + acc[0] ^= readLE64(input, in2) + readLE64(input, in2 + 8); + acc[1] += mix16B(input, in2, secOff + 16, seed); + acc[1] ^= readLE64(input, in1) + readLE64(input, in1 + 8); + } + + private static long[] finalize128(long[] acc, int len, long seed) { + long low = xxh3Avalanche(acc[0] + acc[1]); + long high = -xxh3Avalanche( + acc[0] * PRIME64_1 + acc[1] * PRIME64_4 + ((long) len - seed) * PRIME64_2); + return new long[] {low, high}; + } + + /** Returns the XXH3 128-bit hash as {@code {low64, high64}}. */ + public static long[] hash128(byte[] input, long seed) { + int len = input.length; + if (len <= 16) { + if (len > 8) { + return len9to16128(input, len, seed); + } else if (len >= 4) { + return len4to8128(input, len, seed); + } else if (len > 0) { + return len1to3128(input, len, seed); + } + long low = xxh64Avalanche(seed ^ readLE64(SECRET, 64) ^ readLE64(SECRET, 72)); + long high = xxh64Avalanche(seed ^ readLE64(SECRET, 80) ^ readLE64(SECRET, 88)); + return new long[] {low, high}; + } else if (len <= 128) { + return len17to128128(input, len, seed); + } else if (len <= 240) { + return len129to240128(input, len, seed); + } + return hashLong128(input, len, seed); + } + + private static long[] len1to3128(byte[] input, int len, long seed) { + int c1 = input[0] & 0xFF; + int c2 = input[len >> 1] & 0xFF; + int c3 = input[len - 1] & 0xFF; + int combinedl = (c1 << 16) | (c2 << 24) | c3 | (len << 8); + int combinedh = Integer.rotateLeft(Integer.reverseBytes(combinedl), 13); + long bitflipl = (readLE32(SECRET, 0) ^ readLE32(SECRET, 4)) + seed; + long bitfliph = (readLE32(SECRET, 8) ^ readLE32(SECRET, 12)) - seed; + long low = xxh64Avalanche((combinedl & 0xFFFFFFFFL) ^ bitflipl); + long high = xxh64Avalanche((combinedh & 0xFFFFFFFFL) ^ bitfliph); + return new long[] {low, high}; + } + + private static long[] len4to8128(byte[] input, int len, long seed) { + seed ^= ((long) Integer.reverseBytes((int) seed) & 0xFFFFFFFFL) << 32; + long inputLo = readLE32(input, 0); + long inputHi = readLE32(input, len - 4); + long input64 = inputLo | (inputHi << 32); + long keyed = input64 ^ ((readLE64(SECRET, 16) ^ readLE64(SECRET, 24)) + seed); + long mul = PRIME64_1 + ((long) len << 2); + long lo = keyed * mul; + long hi = unsignedMultiplyHigh(keyed, mul); + hi += lo << 1; + lo ^= hi >>> 3; + lo ^= lo >>> 35; + lo *= PRIME_MX2; + lo ^= lo >>> 28; + return new long[] {lo, xxh3Avalanche(hi)}; + } + + private static long[] len9to16128(byte[] input, int len, long seed) { + long bitflipl = (readLE64(SECRET, 32) ^ readLE64(SECRET, 40)) - seed; + long bitfliph = (readLE64(SECRET, 48) ^ readLE64(SECRET, 56)) + seed; + long inputLo = readLE64(input, 0); + long inputHi = readLE64(input, len - 8); + long m0 = inputLo ^ inputHi ^ bitflipl; + long lo = m0 * PRIME64_1; + long hi = unsignedMultiplyHigh(m0, PRIME64_1); + lo += (long) (len - 1) << 54; + inputHi ^= bitfliph; + hi += inputHi + mult32to64(inputHi, PRIME32_2 - 1); + lo ^= Long.reverseBytes(hi); + long h2lo = lo * PRIME64_2; + long h2hi = unsignedMultiplyHigh(lo, PRIME64_2) + hi * PRIME64_2; + return new long[] {xxh3Avalanche(h2lo), xxh3Avalanche(h2hi)}; + } + + private static long[] len17to128128(byte[] input, int len, long seed) { + long[] acc = {(long) len * PRIME64_1, 0L}; + int i = (len - 1) / 32; + do { + mix32B(acc, input, 16 * i, len - 16 * (i + 1), 32 * i, seed); + } while (i-- != 0); + return finalize128(acc, len, seed); + } + + private static long[] len129to240128(byte[] input, int len, long seed) { + long[] acc = {(long) len * PRIME64_1, 0L}; + for (int i = 0; i < 4; i++) { + mix32B(acc, input, 32 * i, 32 * i + 16, 32 * i, seed); + } + acc[0] = xxh3Avalanche(acc[0]); + acc[1] = xxh3Avalanche(acc[1]); + for (int i = 4; i < (len >> 5); i++) { + mix32B(acc, input, 32 * i, 32 * i + 16, (i - 4) * 32 + 3, seed); + } + mix32B(acc, input, len - 16, len - 32, 103, -seed); + return finalize128(acc, len, seed); + } + + private static long[] hashLong128(byte[] input, int len, long seed) { + byte[] secret = customSecret(seed); + long[] acc = hashLongAccumulate(input, len, secret); + long low = mergeAccs(acc, secret, SECRET_MERGEACCS_START, (long) len * PRIME64_1); + long high = mergeAccs(acc, secret, SECRET_SIZE - STRIPE_LEN - SECRET_MERGEACCS_START, + ~((long) len * PRIME64_2)); + return new long[] {low, high}; + } + + /** Returns the XXH3 128-bit hash as a 32-character lowercase hex string (default seed 0). */ + public static UTF8String hash128Hex(byte[] input) { + return hash128Hex(input, 0L); + } + + /** Returns the XXH3 128-bit hash as a 32-character lowercase hex string (canonical big-endian). */ + public static UTF8String hash128Hex(byte[] input, long seed) { + long[] h = hash128(input, seed); + byte[] out = new byte[16]; + writeBE64(out, 0, h[1]); + writeBE64(out, 8, h[0]); + return UTF8String.fromString(HexFormat.of().formatHex(out)); + } + + private static void writeBE64(byte[] out, int offset, long value) { + for (int i = 0; i < 8; i++) { + out[offset + i] = (byte) (value >>> (56 - 8 * i)); + } + } +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 5b39aa06f1e41..a3c4d7dcb4649 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -846,6 +846,8 @@ object FunctionRegistry { expression[Uuid]("uuid"), expression[Murmur3Hash]("hash"), expression[XxHash64]("xxhash64"), + expression[Xxh364]("xxh3_64"), + expression[Xxh3128]("xxh3_128"), expression[Sha1]("sha", true), expression[Sha1]("sha1"), expression[Sha2]("sha2"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala index 085586041b07e..a3cc265dda7ed 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala @@ -248,6 +248,78 @@ case class Crc32(child: Expression) override protected def withNewChildInternal(newChild: Expression): Crc32 = copy(child = newChild) } +@ExpressionDescription( + usage = "_FUNC_(expr) - Returns a 64-bit hash value of the argument using the XXH3 algorithm.", + arguments = """ + Arguments: + * expr - The expression to compute the XXH3 hash of. + An expression that evaluates to a binary. + """, + examples = """ + Examples: + > SELECT _FUNC_('Spark'); + 80997306238743657 + """, + since = "4.3.0", + group = "hash_funcs") +case class Xxh364(child: Expression) + extends UnaryExpression with ImplicitCastInputTypes { + override def nullIntolerant: Boolean = true + + override def dataType: DataType = LongType + + override def inputTypes: Seq[DataType] = Seq(BinaryType) + + override def contextIndependentFoldable: Boolean = child.contextIndependentFoldable + + protected override def nullSafeEval(input: Any): Any = + XXH3.hash64(input.asInstanceOf[Array[Byte]]) + + override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val cls = classOf[XXH3].getName + nullSafeCodeGen(ctx, ev, value => s"${ev.value} = $cls.hash64($value);") + } + + override def prettyName: String = "xxh3_64" + + override protected def withNewChildInternal(newChild: Expression): Xxh364 = copy(child = newChild) +} + +@ExpressionDescription( + usage = "_FUNC_(expr) - Returns a 128-bit XXH3 hash of the argument as a hex string.", + arguments = """ + Arguments: + * expr - The expression to compute the XXH3 hash of. + An expression that evaluates to a binary. + """, + examples = """ + Examples: + > SELECT _FUNC_('Spark'); + 7d57dd84c60c86ca1f4e82ab91a12b5e + """, + since = "4.3.0", + group = "hash_funcs") +case class Xxh3128(child: Expression) + extends UnaryExpression with ImplicitCastInputTypes with DefaultStringProducingExpression { + override def nullIntolerant: Boolean = true + + override def inputTypes: Seq[DataType] = Seq(BinaryType) + + override def contextIndependentFoldable: Boolean = child.contextIndependentFoldable + + protected override def nullSafeEval(input: Any): Any = + XXH3.hash128Hex(input.asInstanceOf[Array[Byte]]) + + override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val cls = classOf[XXH3].getName + defineCodeGen(ctx, ev, c => s"$cls.hash128Hex($c)") + } + + override def prettyName: String = "xxh3_128" + + override protected def withNewChildInternal(newChild: Expression): Xxh3128 = + copy(child = newChild) +} /** * A function that calculates hash value for a group of expressions. Note that the `seed` argument diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala index 02846b560ad00..d03db29257580 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala @@ -92,6 +92,17 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkConsistencyBetweenInterpretedAndCodegen(Crc32, BinaryType) } + test("xxh3_64 and xxh3_128") { + // Concrete values match the reference implementation; XXH3Suite covers the full vector set. + checkEvaluation(Xxh364(Literal("Spark".getBytes(StandardCharsets.UTF_8))), 80997306238743657L) + checkEvaluation(Xxh3128(Literal("Spark".getBytes(StandardCharsets.UTF_8))), + "7d57dd84c60c86ca1f4e82ab91a12b5e") + checkEvaluation(Xxh364(Literal.create(null, BinaryType)), null) + checkEvaluation(Xxh3128(Literal.create(null, BinaryType)), null) + checkConsistencyBetweenInterpretedAndCodegen(Xxh364, BinaryType) + checkConsistencyBetweenInterpretedAndCodegen(Xxh3128, BinaryType) + } + def checkHiveHash(input: Any, dataType: DataType, expected: Long): Unit = { // Note : All expected hashes need to be computed using Hive 1.2.1 val actual = HiveHashFunction.hash( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XXH3Suite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XXH3Suite.scala new file mode 100644 index 0000000000000..1929e9bbe1727 --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XXH3Suite.scala @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import org.apache.spark.SparkFunSuite + +/** + * Validates the [[XXH3]] port against the reference implementation's known-answer vectors + * (github.com/Cyan4973/xxHash). Inputs are prefixes of the same pseudo-random buffer the reference + * test harness uses (tests/sanity_test.c); expected values were produced with the reference + * `xxhash` library and cover every length branch (0, 1-3, 4-8, 9-16, 17-128, 129-240, and the + * long path) with both a zero and a non-zero seed. + */ +class XXH3Suite extends SparkFunSuite { + + private val buffer: Array[Byte] = { + val buf = new Array[Byte](4200) + var byteGen = 0x9E3779B1L + for (i <- buf.indices) { + buf(i) = (byteGen >>> 56).toByte + byteGen *= 0x9E3779B185EBCA8DL + } + buf + } + + private def input(len: Int): Array[Byte] = java.util.Arrays.copyOf(buffer, len) + + // 64-bit: (length, seed, expected) + private val vectors64: Seq[(Int, Long, Long)] = Seq( + (0, 0L, 0x2d06800538d394c2L), + (1, 0L, 0xc44bdff4074eecdbL), + (2, 0L, 0x7a9978044cb8a8bbL), + (3, 0L, 0x54247382a8d6b94dL), + (4, 0L, 0xe5dc74bc51848a51L), + (5, 0L, 0xe4243f00720306bbL), + (7, 0L, 0x9941e0007f555e50L), + (8, 0L, 0x24ccc9acaa9f65e4L), + (9, 0L, 0x14d5001c15dd3f2bL), + (12, 0L, 0xa713daf0dfbb77e7L), + (16, 0L, 0x981b17d36c7498c9L), + (17, 0L, 0x796f5acd3a60f862L), + (32, 0L, 0x9feaddbdbf57eed3L), + (64, 0L, 0x9cb48487720ec49dL), + (100, 0L, 0x93cd95432b7d483fL), + (128, 0L, 0xfcff24126754d861L), + (129, 0L, 0x98f1b0a679a2ca29L), + (160, 0L, 0x9d03a319ed4cbd2bL), + (200, 0L, 0xbddca58935d7c038L), + (240, 0L, 0x81c3c2b67f568ccfL), + (241, 0L, 0xc5a639ecd2030e5eL), + (256, 0L, 0x55de574ad89d0ac5L), + (512, 0L, 0x617e49599013cb6bL), + (1024, 0L, 0xdd85c9b5c1109c5cL), + (2048, 0L, 0xdd59e2c3a5f038e0L), + (4096, 0L, 0xe91206429d1f48f9L), + (0, 0x9e3779b185ebca8dL, 0xa8a6b918b2f0364aL), + (8, 0x9e3779b185ebca8dL, 0x8f973410999b8f6bL), + (16, 0x9e3779b185ebca8dL, 0x663f29333b4db6b1L), + (64, 0x9e3779b185ebca8dL, 0x4fe8895db9b8c077L), + (240, 0x9e3779b185ebca8dL, 0xcc0f58c27ef3d8eeL), + (256, 0x9e3779b185ebca8dL, 0x4d30234b7a3aa61cL), + (1024, 0x9e3779b185ebca8dL, 0xef368a8a2ebabaefL) + ) + + // 128-bit: (length, seed, expected canonical hex) + private val vectors128: Seq[(Int, Long, String)] = Seq( + (0, 0L, "99aa06d3014798d86001c324468d497f"), + (1, 0L, "a6cd5e9392000f6ac44bdff4074eecdb"), + (2, 0L, "76750c3c7bf956687a9978044cb8a8bb"), + (3, 0L, "20efc49ff02422ea54247382a8d6b94d"), + (4, 0L, "970d585ac632bf8e2e7d8d6876a39fe9"), + (5, 0L, "62ed587687606b4e057c7ed2c01fa1d1"), + (7, 0L, "dd9b6039f79ec416081c22dd284a2f0a"), + (8, 0L, "47a7f080d82bb45664c69cab4bb21dc5"), + (9, 0L, "564ef6078950d457ed7ccbc501eb7501"), + (12, 0L, "6e3efd8fc7802b18061a192713f69ad9"), + (16, 0L, "c68c368ecf8a9c05562980258a998629"), + (17, 0L, "955fa78643ed3669abbc12d11973d7db"), + (32, 0L, "98fc6458710dc2e8278410a17595e3f9"), + (64, 0L, "6d90e81a9b0fd622efdb6a44690721a9"), + (100, 0L, "9b50b05817ab158e5fcbc2e3295f2476"), + (128, 0L, "39992220e045260aebb15e34a7fb5ab1"), + (129, 0L, "03815fc91f1b30b686c9e3bc8f0a3b5c"), + (160, 0L, "ba5d218964b622ad737126c8d7c09cee"), + (200, 0L, "e76ff4780fe18439eb060f1bb3126f5a"), + (240, 0L, "aa4202daa2769dc85c9aae94c8ebe5a0"), + (241, 0L, "99a80ecf0ecfc647c5a639ecd2030e5e"), + (256, 0L, "8b1c66091423d28855de574ad89d0ac5"), + (512, 0L, "18d2d110dcc9bca1617e49599013cb6b"), + (1024, 0L, "0d30d24071c64c57dd85c9b5c1109c5c"), + (2048, 0L, "f736557fd47073a5dd59e2c3a5f038e0"), + (4096, 0L, "b9cfaea2ca5626a4e91206429d1f48f9"), + (0, 0x9e3779b185ebca8dL, "00feaa732a3ce25ea986dfc5d7605bfe"), + (8, 0x9e3779b185ebca8dL, "f50cec145bcd5c5a7b29471dc729b5ff"), + (16, 0x9e3779b185ebca8dL, "6ffcb80cd33085c80346d13a7a5498c7"), + (64, 0x9e3779b185ebca8dL, "37b738968d40bda59405ba2affa95ceb"), + (240, 0x9e3779b185ebca8dL, "29d2133d6ea58c5b604e98db085c1864"), + (256, 0x9e3779b185ebca8dL, "aaa57235b92d5e7c4d30234b7a3aa61c"), + (1024, 0x9e3779b185ebca8dL, "17600efe2b493a18ef368a8a2ebabaef") + ) + + test("reference test buffer generation") { + val expectedFirst24 = Seq(0x00, 0x52, 0x92, 0x9B, 0xB7, 0x32, 0xA3, 0x24, 0x2D, 0x00, 0xAF, + 0x95, 0x0E, 0xEC, 0xB8, 0x93, 0xE3, 0xDF, 0xEF, 0x93, 0xAA, 0xD6, 0xCD, 0x2A) + expectedFirst24.zipWithIndex.foreach { case (b, i) => + assert((buffer(i) & 0xFF) == b, + s"buffer($i): expected 0x${b.toHexString} got 0x${(buffer(i) & 0xFF).toHexString}") + } + } + + test("XXH3 64-bit against reference vectors") { + vectors64.foreach { case (len, seed, expected) => + val actual = XXH3.hash64(input(len), seed) + assert(actual == expected, s"len=$len seed=0x${seed.toHexString}: " + + s"expected 0x${expected.toHexString} but got 0x${actual.toHexString}") + } + } + + test("XXH3 128-bit against reference vectors") { + vectors128.foreach { case (len, seed, expected) => + val actual = XXH3.hash128Hex(input(len), seed).toString + assert(actual == expected, s"len=$len seed=0x${seed.toHexString}: " + + s"expected $expected but got $actual") + } + } +} diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala index 068460293f15c..02d6576d539a0 100644 --- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala +++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala @@ -1711,6 +1711,14 @@ class PlanGenerationTestSuite extends ConnectFunSuite with Logging { fn.crc32(fn.col("g").cast("binary")) } + functionTest("xxh3_64") { + fn.xxh3_64(fn.col("g").cast("binary")) + } + + functionTest("xxh3_128") { + fn.xxh3_128(fn.col("g").cast("binary")) + } + functionTest("hash") { fn.hash(fn.col("b"), fn.col("id")) } diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_128.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_128.explain new file mode 100644 index 0000000000000..189ed0c540e9c --- /dev/null +++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_128.explain @@ -0,0 +1,2 @@ +Project [xxh3_128(cast(g#0 as binary)) AS xxh3_128(CAST(g AS BINARY))#0] ++- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_64.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_64.explain new file mode 100644 index 0000000000000..38bb03544c003 --- /dev/null +++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_64.explain @@ -0,0 +1,2 @@ +Project [xxh3_64(cast(g#0 as binary)) AS xxh3_64(CAST(g AS BINARY))#0L] ++- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_128.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_128.json new file mode 100644 index 0000000000000..27c099a4146fe --- /dev/null +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_128.json @@ -0,0 +1,85 @@ +{ + "common": { + "planId": "1" + }, + "project": { + "input": { + "common": { + "planId": "0" + }, + "localRelation": { + "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e" + } + }, + "expressions": [{ + "unresolvedFunction": { + "functionName": "xxh3_128", + "arguments": [{ + "cast": { + "expr": { + "unresolvedAttribute": { + "unparsedIdentifier": "g" + }, + "common": { + "origin": { + "jvmOrigin": { + "stackTrace": [{ + "classLoaderName": "app", + "declaringClass": "org.apache.spark.sql.functions$", + "methodName": "col", + "fileName": "functions.scala" + }, { + "classLoaderName": "app", + "declaringClass": "org.apache.spark.sql.PlanGenerationTestSuite", + "methodName": "~~trimmed~anonfun~~", + "fileName": "PlanGenerationTestSuite.scala" + }] + } + } + } + }, + "type": { + "binary": { + } + } + }, + "common": { + "origin": { + "jvmOrigin": { + "stackTrace": [{ + "classLoaderName": "app", + "declaringClass": "org.apache.spark.sql.Column", + "methodName": "cast", + "fileName": "Column.scala" + }, { + "classLoaderName": "app", + "declaringClass": "org.apache.spark.sql.PlanGenerationTestSuite", + "methodName": "~~trimmed~anonfun~~", + "fileName": "PlanGenerationTestSuite.scala" + }] + } + } + } + }], + "isInternal": false + }, + "common": { + "origin": { + "jvmOrigin": { + "stackTrace": [{ + "classLoaderName": "app", + "declaringClass": "org.apache.spark.sql.functions$", + "methodName": "xxh3_128", + "fileName": "functions.scala" + }, { + "classLoaderName": "app", + "declaringClass": "org.apache.spark.sql.PlanGenerationTestSuite", + "methodName": "~~trimmed~anonfun~~", + "fileName": "PlanGenerationTestSuite.scala" + }] + } + } + } + }] + } +} \ No newline at end of file diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_128.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xxh3_128.proto.bin new file mode 100644 index 0000000000000000000000000000000000000000..34feea650f4e4e9816690751e1b6255ab0f1c716 GIT binary patch literal 722 zcmd;L5@3`%&B`^2k&8)yA*!2EsDrV%q^LBx#3nPvDk(EPGp|G^(F#N+S*7HcCgr5+ zq*xJ9VW*R7l~`1iSZM>)XQz{9m77>#1Jsk5m##xdtDR0d$atVqJE_eqQooqEI4UYK zjN=WBEQDS&S?y)y+Qulv%*B{KiE%9>*D6M#<%~`?T+E3D1xoVyMd^Bp1&PTSsd~i) ziAC9Z#f3R~X{C9|C7Jno#VT6N$@w`({BQxi;^f4fM5lC!ZXK+;19B4c+*9*XixNR* zgrpXi1ea!(q-qJ*)d3xuo12)XQz{9m77>#1Jsk5m##xdtDR0d$atVqJE@H3WF;iOCtMdc_5a zMcI1Ag*kd@rFqFEnfZCeDq76R`8h`XZ~?vI)zbnHe(u62w;s~Mdfhzwy^R1I7x0{}N6`-}hp literal 0 HcmV?d00001 diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 99285d126acdf..b7dd8b5fa12e9 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -460,6 +460,8 @@ | org.apache.spark.sql.catalyst.expressions.WindowTime | window_time | SELECT a, window.start as start, window.end as end, window_time(window), cnt FROM (SELECT a, window, count(*) as cnt FROM VALUES ('A1', '2021-01-01 00:00:00'), ('A1', '2021-01-01 00:04:30'), ('A1', '2021-01-01 00:06:00'), ('A2', '2021-01-01 00:01:00') AS tab(a, b) GROUP by a, window(b, '5 minutes') ORDER BY a, window.start) | struct | | org.apache.spark.sql.catalyst.expressions.XmlToStructs | from_xml | SELECT from_xml('

10.8

', 'a INT, b DOUBLE') | struct10.8

):struct> | | org.apache.spark.sql.catalyst.expressions.XxHash64 | xxhash64 | SELECT xxhash64('Spark', array(123), 2) | struct | +| org.apache.spark.sql.catalyst.expressions.Xxh3128 | xxh3_128 | SELECT xxh3_128('Spark') | struct | +| org.apache.spark.sql.catalyst.expressions.Xxh364 | xxh3_64 | SELECT xxh3_64('Spark') | struct | | org.apache.spark.sql.catalyst.expressions.Year | year | SELECT year('2016-07-30') | struct | | org.apache.spark.sql.catalyst.expressions.ZeroIfNull | zeroifnull | SELECT zeroifnull(NULL) | struct | | org.apache.spark.sql.catalyst.expressions.ZipWith | zip_with | SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)) | struct>> | diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/misc-functions.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/misc-functions.sql.out index a470d33d98931..08d901f949f8f 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/misc-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/misc-functions.sql.out @@ -296,3 +296,45 @@ SELECT hmac('key', 'message', 'SHA-3') -- !query analysis Project [hmac(cast(key as binary), cast(message as binary), SHA-3) AS hmac(key, message, SHA-3)#x] +- OneRowRelation + + +-- !query +SELECT xxh3_64('Spark') +-- !query analysis +Project [xxh3_64(cast(Spark as binary)) AS xxh3_64(Spark)#xL] ++- OneRowRelation + + +-- !query +SELECT xxh3_64(CAST('Spark' AS BINARY)) +-- !query analysis +Project [xxh3_64(cast(Spark as binary)) AS xxh3_64(CAST(Spark AS BINARY))#xL] ++- OneRowRelation + + +-- !query +SELECT xxh3_128('Spark') +-- !query analysis +Project [xxh3_128(cast(Spark as binary)) AS xxh3_128(Spark)#x] ++- OneRowRelation + + +-- !query +SELECT xxh3_128(CAST('Spark' AS BINARY)) +-- !query analysis +Project [xxh3_128(cast(Spark as binary)) AS xxh3_128(CAST(Spark AS BINARY))#x] ++- OneRowRelation + + +-- !query +SELECT xxh3_64(CAST(NULL AS STRING)) +-- !query analysis +Project [xxh3_64(cast(cast(null as string) as binary)) AS xxh3_64(CAST(NULL AS STRING))#xL] ++- OneRowRelation + + +-- !query +SELECT xxh3_128(CAST(NULL AS BINARY)) +-- !query analysis +Project [xxh3_128(cast(null as binary)) AS xxh3_128(CAST(NULL AS BINARY))#x] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql index e0b7b6ac3f88f..232f16963c84a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/misc-functions.sql @@ -58,3 +58,12 @@ SELECT hmac('key', CAST(NULL AS BINARY)); SELECT hmac('key', 'message', CAST(NULL AS STRING)); -- Unsupported algorithm. SELECT hmac('key', 'message', 'SHA-3'); + +-- xxh3_64 and xxh3_128 +SELECT xxh3_64('Spark'); +SELECT xxh3_64(CAST('Spark' AS BINARY)); +SELECT xxh3_128('Spark'); +SELECT xxh3_128(CAST('Spark' AS BINARY)); +-- Null propagation. +SELECT xxh3_64(CAST(NULL AS STRING)); +SELECT xxh3_128(CAST(NULL AS BINARY)); diff --git a/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out index 9373dc63289aa..5d9438d3a5234 100644 --- a/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/misc-functions.sql.out @@ -402,3 +402,51 @@ org.apache.spark.SparkRuntimeException "parameter" : "`algorithm`" } } + + +-- !query +SELECT xxh3_64('Spark') +-- !query schema +struct +-- !query output +80997306238743657 + + +-- !query +SELECT xxh3_64(CAST('Spark' AS BINARY)) +-- !query schema +struct +-- !query output +80997306238743657 + + +-- !query +SELECT xxh3_128('Spark') +-- !query schema +struct +-- !query output +7d57dd84c60c86ca1f4e82ab91a12b5e + + +-- !query +SELECT xxh3_128(CAST('Spark' AS BINARY)) +-- !query schema +struct +-- !query output +7d57dd84c60c86ca1f4e82ab91a12b5e + + +-- !query +SELECT xxh3_64(CAST(NULL AS STRING)) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT xxh3_128(CAST(NULL AS BINARY)) +-- !query schema +struct +-- !query output +NULL From 2cc11206d2464bdb538e3b1e3389b890ecd6e304 Mon Sep 17 00:00:00 2001 From: SreeramaYeshwanthGowd Date: Sat, 1 Aug 2026 09:24:02 +0530 Subject: [PATCH 2/2] [SPARK-45900][SQL] Wrap over-length Javadoc comment in XXH3.java --- .../java/org/apache/spark/sql/catalyst/expressions/XXH3.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java index d2637adfd5818..942fa2e95b1d4 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/XXH3.java @@ -404,7 +404,9 @@ public static UTF8String hash128Hex(byte[] input) { return hash128Hex(input, 0L); } - /** Returns the XXH3 128-bit hash as a 32-character lowercase hex string (canonical big-endian). */ + /** + * Returns the XXH3 128-bit hash as a 32-character lowercase hex string (canonical big-endian). + */ public static UTF8String hash128Hex(byte[] input, long seed) { long[] h = hash128(input, seed); byte[] out = new byte[16];