All arithmetic is on 64-bit lanes; Java's signed {@code long} is used as an unsigned 64-bit
+ * integer throughout ({@code >>>} for logical shift, {@link Long#rotateLeft}, and
+ * {@link #unsignedMultiplyHigh} for the high half of 64x64 products).
+ */
+public final class XXH3 {
+
+ private XXH3() {}
+
+ private static final long PRIME32_1 = 0x9E3779B1L;
+ private static final long PRIME32_2 = 0x85EBCA77L;
+ private static final long PRIME32_3 = 0xC2B2AE3DL;
+ private static final long PRIME64_1 = 0x9E3779B185EBCA87L;
+ private static final long PRIME64_2 = 0xC2B2AE3D27D4EB4FL;
+ private static final long PRIME64_3 = 0x165667B19E3779F9L;
+ private static final long PRIME64_4 = 0x85EBCA77C2B2AE63L;
+ private static final long PRIME64_5 = 0x27D4EB2F165667C5L;
+ private static final long PRIME_MX1 = 0x165667919E3779F9L;
+ private static final long PRIME_MX2 = 0x9FB21C651E98DF25L;
+
+ private static final int SECRET_SIZE = 192;
+ private static final int SECRET_SIZE_MIN = 136;
+ private static final int STRIPE_LEN = 64;
+ private static final int SECRET_MERGEACCS_START = 11;
+ private static final int SECRET_LASTACC_START = 7;
+ private static final int NB_STRIPES_PER_BLOCK = (SECRET_SIZE - STRIPE_LEN) / 8;
+ private static final int BLOCK_LEN = STRIPE_LEN * NB_STRIPES_PER_BLOCK;
+
+ // The default 192-byte secret from the reference implementation.
+ private static final byte[] SECRET = HexFormat.of().parseHex(
+ "b8fe6c3923a44bbe7c01812cf721ad1c" +
+ "ded46de9839097db7240a4a4b7b3671f" +
+ "cb79e64eccc0e578825ad07dccff7221" +
+ "b8084674f743248ee03590e6813a264c" +
+ "3c2852bb91c300cb88d0658b1b532ea3" +
+ "71644897a20df94e3819ef46a9deacd8" +
+ "a8fa763fe39c343ff9dcbbc7c70b4f1d" +
+ "8a51e04bcdb45931c89f7ec9d9787364" +
+ "eac5ac8334d3ebc3c581a0fffa1363eb" +
+ "170ddd51b7f0da49d316552629d4689e" +
+ "2b16be587d47a1fc8ff8b8d17ad031ce" +
+ "45cb3a8f95160428afd7fbcabb4b407e");
+
+ // ---- little-endian reads / writes ----
+
+ private static long readLE64(byte[] data, int offset) {
+ return (data[offset] & 0xFFL)
+ | ((data[offset + 1] & 0xFFL) << 8)
+ | ((data[offset + 2] & 0xFFL) << 16)
+ | ((data[offset + 3] & 0xFFL) << 24)
+ | ((data[offset + 4] & 0xFFL) << 32)
+ | ((data[offset + 5] & 0xFFL) << 40)
+ | ((data[offset + 6] & 0xFFL) << 48)
+ | ((data[offset + 7] & 0xFFL) << 56);
+ }
+
+ // Reads 4 little-endian bytes as an unsigned 32-bit value (in the low 32 bits of the result).
+ private static long readLE32(byte[] data, int offset) {
+ return (data[offset] & 0xFFL)
+ | ((data[offset + 1] & 0xFFL) << 8)
+ | ((data[offset + 2] & 0xFFL) << 16)
+ | ((data[offset + 3] & 0xFFL) << 24);
+ }
+
+ private static void writeLE64(byte[] data, int offset, long value) {
+ for (int i = 0; i < 8; i++) {
+ data[offset + i] = (byte) (value >>> (8 * i));
+ }
+ }
+
+ // ---- mixing helpers ----
+
+ private static long xxh64Avalanche(long h) {
+ h ^= h >>> 33;
+ h *= PRIME64_2;
+ h ^= h >>> 29;
+ h *= PRIME64_3;
+ h ^= h >>> 32;
+ return h;
+ }
+
+ private static long xxh3Avalanche(long h) {
+ h ^= h >>> 37;
+ h *= PRIME_MX1;
+ h ^= h >>> 32;
+ return h;
+ }
+
+ // Unsigned high 64 bits of the 128-bit product a*b (Math.multiplyHigh is signed).
+ private static long unsignedMultiplyHigh(long a, long b) {
+ return Math.multiplyHigh(a, b) + ((a >> 63) & b) + ((b >> 63) & a);
+ }
+
+ // Low 64 bits XOR high 64 bits of the 128-bit product a*b.
+ private static long mul128Fold64(long a, long b) {
+ return (a * b) ^ unsignedMultiplyHigh(a, b);
+ }
+
+ private static long mix16B(byte[] input, int inOff, int secOff, long seed) {
+ long lo = readLE64(input, inOff) ^ (readLE64(SECRET, secOff) + seed);
+ long hi = readLE64(input, inOff + 8) ^ (readLE64(SECRET, secOff + 8) - seed);
+ return mul128Fold64(lo, hi);
+ }
+
+ // ---- XXH3 64-bit ----
+
+ public static long hash64(byte[] input, long seed) {
+ int len = input.length;
+ if (len <= 16) {
+ if (len > 8) {
+ return len9to16(input, len, seed);
+ } else if (len >= 4) {
+ return len4to8(input, len, seed);
+ } else if (len > 0) {
+ return len1to3(input, len, seed);
+ }
+ return xxh64Avalanche(seed ^ readLE64(SECRET, 56) ^ readLE64(SECRET, 64));
+ } else if (len <= 128) {
+ return len17to128(input, len, seed);
+ } else if (len <= 240) {
+ return len129to240(input, len, seed);
+ }
+ return hashLong(input, len, seed);
+ }
+
+ private static long len1to3(byte[] input, int len, long seed) {
+ long combined = ((input[0] & 0xFFL) << 16)
+ | ((input[len >> 1] & 0xFFL) << 24)
+ | (input[len - 1] & 0xFFL)
+ | ((long) len << 8);
+ long flip = (readLE32(SECRET, 0) ^ readLE32(SECRET, 4)) + seed;
+ return xxh64Avalanche(combined ^ flip);
+ }
+
+ private static long len4to8(byte[] input, int len, long seed) {
+ seed ^= ((long) Integer.reverseBytes((int) seed) & 0xFFFFFFFFL) << 32;
+ long in1 = readLE32(input, 0);
+ long in2 = readLE32(input, len - 4);
+ long combined = in2 | (in1 << 32);
+ long flip = (readLE64(SECRET, 8) ^ readLE64(SECRET, 16)) - seed;
+ long x = combined ^ flip;
+ x ^= Long.rotateLeft(x, 49) ^ Long.rotateLeft(x, 24);
+ x *= PRIME_MX2;
+ x ^= (x >>> 35) + len;
+ x *= PRIME_MX2;
+ x ^= x >>> 28;
+ return x;
+ }
+
+ private static long len9to16(byte[] input, int len, long seed) {
+ long flip1 = (readLE64(SECRET, 24) ^ readLE64(SECRET, 32)) + seed;
+ long flip2 = (readLE64(SECRET, 40) ^ readLE64(SECRET, 48)) - seed;
+ long in1 = readLE64(input, 0) ^ flip1;
+ long in2 = readLE64(input, len - 8) ^ flip2;
+ long acc = len + Long.reverseBytes(in1) + in2 + mul128Fold64(in1, in2);
+ return xxh3Avalanche(acc);
+ }
+
+ private static long len17to128(byte[] input, int len, long seed) {
+ long acc = (long) len * PRIME64_1;
+ for (int i = (len - 1) >> 5; i >= 0; i--) {
+ acc += mix16B(input, 16 * i, 32 * i, seed);
+ acc += mix16B(input, len - 16 * (i + 1), 32 * i + 16, seed);
+ }
+ return xxh3Avalanche(acc);
+ }
+
+ private static long len129to240(byte[] input, int len, long seed) {
+ long acc = (long) len * PRIME64_1;
+ int nbRounds = len / 16;
+ for (int i = 0; i < 8; i++) {
+ acc += mix16B(input, 16 * i, 16 * i, seed);
+ }
+ acc = xxh3Avalanche(acc);
+ for (int i = 8; i < nbRounds; i++) {
+ acc += mix16B(input, 16 * i, 16 * (i - 8) + 3, seed);
+ }
+ acc += mix16B(input, len - 16, SECRET_SIZE_MIN - 17, seed);
+ return xxh3Avalanche(acc);
+ }
+
+ // ---- long input (> 240 bytes) ----
+
+ private static byte[] customSecret(long seed) {
+ if (seed == 0) {
+ return SECRET;
+ }
+ byte[] secret = new byte[SECRET_SIZE];
+ for (int i = 0; i < SECRET_SIZE / 16; i++) {
+ writeLE64(secret, 16 * i, readLE64(SECRET, 16 * i) + seed);
+ writeLE64(secret, 16 * i + 8, readLE64(SECRET, 16 * i + 8) - seed);
+ }
+ return secret;
+ }
+
+ private static void accumulate512(
+ long[] acc, byte[] input, int inOff, byte[] secret, int secOff) {
+ for (int i = 0; i < 8; i++) {
+ long data = readLE64(input, inOff + 8 * i);
+ long key = data ^ readLE64(secret, secOff + 8 * i);
+ acc[i ^ 1] += data;
+ acc[i] += (key & 0xFFFFFFFFL) * (key >>> 32);
+ }
+ }
+
+ private static void scrambleAcc(long[] acc, byte[] secret, int secOff) {
+ for (int i = 0; i < 8; i++) {
+ acc[i] ^= acc[i] >>> 47;
+ acc[i] ^= readLE64(secret, secOff + 8 * i);
+ acc[i] *= PRIME32_1;
+ }
+ }
+
+ private static long mergeAccs(long[] acc, byte[] secret, int secOff, long start) {
+ long result = start;
+ for (int i = 0; i < 4; i++) {
+ long a0 = acc[2 * i] ^ readLE64(secret, secOff + 16 * i);
+ long a1 = acc[2 * i + 1] ^ readLE64(secret, secOff + 16 * i + 8);
+ result += mul128Fold64(a0, a1);
+ }
+ return xxh3Avalanche(result);
+ }
+
+ private static long[] hashLongAccumulate(byte[] input, int len, byte[] secret) {
+ long[] acc = {PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3, PRIME64_4, PRIME32_2, PRIME64_5,
+ PRIME32_1};
+ int nbBlocks = (len - 1) / BLOCK_LEN;
+ for (int n = 0; n < nbBlocks; n++) {
+ for (int s = 0; s < NB_STRIPES_PER_BLOCK; s++) {
+ accumulate512(acc, input, n * BLOCK_LEN + s * STRIPE_LEN, secret, s * 8);
+ }
+ scrambleAcc(acc, secret, SECRET_SIZE - STRIPE_LEN);
+ }
+ int nbStripes = ((len - 1) - BLOCK_LEN * nbBlocks) / STRIPE_LEN;
+ for (int s = 0; s < nbStripes; s++) {
+ accumulate512(acc, input, nbBlocks * BLOCK_LEN + s * STRIPE_LEN, secret, s * 8);
+ }
+ accumulate512(
+ acc, input, len - STRIPE_LEN, secret, SECRET_SIZE - STRIPE_LEN - SECRET_LASTACC_START);
+ return acc;
+ }
+
+ private static long hashLong(byte[] input, int len, long seed) {
+ byte[] secret = customSecret(seed);
+ long[] acc = hashLongAccumulate(input, len, secret);
+ return mergeAccs(acc, secret, SECRET_MERGEACCS_START, (long) len * PRIME64_1);
+ }
+
+ /** Hashes the input with the default seed 0. */
+ public static long hash64(byte[] input) {
+ return hash64(input, 0L);
+ }
+
+ // ---- XXH3 128-bit ----
+
+ private static long mult32to64(long a, long b) {
+ return (a & 0xFFFFFFFFL) * (b & 0xFFFFFFFFL);
+ }
+
+ private static void mix32B(long[] acc, byte[] input, int in1, int in2, int secOff, long seed) {
+ acc[0] += mix16B(input, in1, secOff, seed);
+ acc[0] ^= readLE64(input, in2) + readLE64(input, in2 + 8);
+ acc[1] += mix16B(input, in2, secOff + 16, seed);
+ acc[1] ^= readLE64(input, in1) + readLE64(input, in1 + 8);
+ }
+
+ private static long[] finalize128(long[] acc, int len, long seed) {
+ long low = xxh3Avalanche(acc[0] + acc[1]);
+ long high = -xxh3Avalanche(
+ acc[0] * PRIME64_1 + acc[1] * PRIME64_4 + ((long) len - seed) * PRIME64_2);
+ return new long[] {low, high};
+ }
+
+ /** Returns the XXH3 128-bit hash as {@code {low64, high64}}. */
+ public static long[] hash128(byte[] input, long seed) {
+ int len = input.length;
+ if (len <= 16) {
+ if (len > 8) {
+ return len9to16128(input, len, seed);
+ } else if (len >= 4) {
+ return len4to8128(input, len, seed);
+ } else if (len > 0) {
+ return len1to3128(input, len, seed);
+ }
+ long low = xxh64Avalanche(seed ^ readLE64(SECRET, 64) ^ readLE64(SECRET, 72));
+ long high = xxh64Avalanche(seed ^ readLE64(SECRET, 80) ^ readLE64(SECRET, 88));
+ return new long[] {low, high};
+ } else if (len <= 128) {
+ return len17to128128(input, len, seed);
+ } else if (len <= 240) {
+ return len129to240128(input, len, seed);
+ }
+ return hashLong128(input, len, seed);
+ }
+
+ private static long[] len1to3128(byte[] input, int len, long seed) {
+ int c1 = input[0] & 0xFF;
+ int c2 = input[len >> 1] & 0xFF;
+ int c3 = input[len - 1] & 0xFF;
+ int combinedl = (c1 << 16) | (c2 << 24) | c3 | (len << 8);
+ int combinedh = Integer.rotateLeft(Integer.reverseBytes(combinedl), 13);
+ long bitflipl = (readLE32(SECRET, 0) ^ readLE32(SECRET, 4)) + seed;
+ long bitfliph = (readLE32(SECRET, 8) ^ readLE32(SECRET, 12)) - seed;
+ long low = xxh64Avalanche((combinedl & 0xFFFFFFFFL) ^ bitflipl);
+ long high = xxh64Avalanche((combinedh & 0xFFFFFFFFL) ^ bitfliph);
+ return new long[] {low, high};
+ }
+
+ private static long[] len4to8128(byte[] input, int len, long seed) {
+ seed ^= ((long) Integer.reverseBytes((int) seed) & 0xFFFFFFFFL) << 32;
+ long inputLo = readLE32(input, 0);
+ long inputHi = readLE32(input, len - 4);
+ long input64 = inputLo | (inputHi << 32);
+ long keyed = input64 ^ ((readLE64(SECRET, 16) ^ readLE64(SECRET, 24)) + seed);
+ long mul = PRIME64_1 + ((long) len << 2);
+ long lo = keyed * mul;
+ long hi = unsignedMultiplyHigh(keyed, mul);
+ hi += lo << 1;
+ lo ^= hi >>> 3;
+ lo ^= lo >>> 35;
+ lo *= PRIME_MX2;
+ lo ^= lo >>> 28;
+ return new long[] {lo, xxh3Avalanche(hi)};
+ }
+
+ private static long[] len9to16128(byte[] input, int len, long seed) {
+ long bitflipl = (readLE64(SECRET, 32) ^ readLE64(SECRET, 40)) - seed;
+ long bitfliph = (readLE64(SECRET, 48) ^ readLE64(SECRET, 56)) + seed;
+ long inputLo = readLE64(input, 0);
+ long inputHi = readLE64(input, len - 8);
+ long m0 = inputLo ^ inputHi ^ bitflipl;
+ long lo = m0 * PRIME64_1;
+ long hi = unsignedMultiplyHigh(m0, PRIME64_1);
+ lo += (long) (len - 1) << 54;
+ inputHi ^= bitfliph;
+ hi += inputHi + mult32to64(inputHi, PRIME32_2 - 1);
+ lo ^= Long.reverseBytes(hi);
+ long h2lo = lo * PRIME64_2;
+ long h2hi = unsignedMultiplyHigh(lo, PRIME64_2) + hi * PRIME64_2;
+ return new long[] {xxh3Avalanche(h2lo), xxh3Avalanche(h2hi)};
+ }
+
+ private static long[] len17to128128(byte[] input, int len, long seed) {
+ long[] acc = {(long) len * PRIME64_1, 0L};
+ int i = (len - 1) / 32;
+ do {
+ mix32B(acc, input, 16 * i, len - 16 * (i + 1), 32 * i, seed);
+ } while (i-- != 0);
+ return finalize128(acc, len, seed);
+ }
+
+ private static long[] len129to240128(byte[] input, int len, long seed) {
+ long[] acc = {(long) len * PRIME64_1, 0L};
+ for (int i = 0; i < 4; i++) {
+ mix32B(acc, input, 32 * i, 32 * i + 16, 32 * i, seed);
+ }
+ acc[0] = xxh3Avalanche(acc[0]);
+ acc[1] = xxh3Avalanche(acc[1]);
+ for (int i = 4; i < (len >> 5); i++) {
+ mix32B(acc, input, 32 * i, 32 * i + 16, (i - 4) * 32 + 3, seed);
+ }
+ mix32B(acc, input, len - 16, len - 32, 103, -seed);
+ return finalize128(acc, len, seed);
+ }
+
+ private static long[] hashLong128(byte[] input, int len, long seed) {
+ byte[] secret = customSecret(seed);
+ long[] acc = hashLongAccumulate(input, len, secret);
+ long low = mergeAccs(acc, secret, SECRET_MERGEACCS_START, (long) len * PRIME64_1);
+ long high = mergeAccs(acc, secret, SECRET_SIZE - STRIPE_LEN - SECRET_MERGEACCS_START,
+ ~((long) len * PRIME64_2));
+ return new long[] {low, high};
+ }
+
+ /** Returns the XXH3 128-bit hash as a 32-character lowercase hex string (default seed 0). */
+ public static UTF8String hash128Hex(byte[] input) {
+ return hash128Hex(input, 0L);
+ }
+
+ /**
+ * Returns the XXH3 128-bit hash as a 32-character lowercase hex string (canonical big-endian).
+ */
+ public static UTF8String hash128Hex(byte[] input, long seed) {
+ long[] h = hash128(input, seed);
+ byte[] out = new byte[16];
+ writeBE64(out, 0, h[1]);
+ writeBE64(out, 8, h[0]);
+ return UTF8String.fromString(HexFormat.of().formatHex(out));
+ }
+
+ private static void writeBE64(byte[] out, int offset, long value) {
+ for (int i = 0; i < 8; i++) {
+ out[offset + i] = (byte) (value >>> (56 - 8 * i));
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 5b39aa06f1e41..a3c4d7dcb4649 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -846,6 +846,8 @@ object FunctionRegistry {
expression[Uuid]("uuid"),
expression[Murmur3Hash]("hash"),
expression[XxHash64]("xxhash64"),
+ expression[Xxh364]("xxh3_64"),
+ expression[Xxh3128]("xxh3_128"),
expression[Sha1]("sha", true),
expression[Sha1]("sha1"),
expression[Sha2]("sha2"),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
index 085586041b07e..a3cc265dda7ed 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
@@ -248,6 +248,78 @@ case class Crc32(child: Expression)
override protected def withNewChildInternal(newChild: Expression): Crc32 = copy(child = newChild)
}
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - Returns a 64-bit hash value of the argument using the XXH3 algorithm.",
+ arguments = """
+ Arguments:
+ * expr - The expression to compute the XXH3 hash of.
+ An expression that evaluates to a binary.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_('Spark');
+ 80997306238743657
+ """,
+ since = "4.3.0",
+ group = "hash_funcs")
+case class Xxh364(child: Expression)
+ extends UnaryExpression with ImplicitCastInputTypes {
+ override def nullIntolerant: Boolean = true
+
+ override def dataType: DataType = LongType
+
+ override def inputTypes: Seq[DataType] = Seq(BinaryType)
+
+ override def contextIndependentFoldable: Boolean = child.contextIndependentFoldable
+
+ protected override def nullSafeEval(input: Any): Any =
+ XXH3.hash64(input.asInstanceOf[Array[Byte]])
+
+ override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ val cls = classOf[XXH3].getName
+ nullSafeCodeGen(ctx, ev, value => s"${ev.value} = $cls.hash64($value);")
+ }
+
+ override def prettyName: String = "xxh3_64"
+
+ override protected def withNewChildInternal(newChild: Expression): Xxh364 = copy(child = newChild)
+}
+
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - Returns a 128-bit XXH3 hash of the argument as a hex string.",
+ arguments = """
+ Arguments:
+ * expr - The expression to compute the XXH3 hash of.
+ An expression that evaluates to a binary.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_('Spark');
+ 7d57dd84c60c86ca1f4e82ab91a12b5e
+ """,
+ since = "4.3.0",
+ group = "hash_funcs")
+case class Xxh3128(child: Expression)
+ extends UnaryExpression with ImplicitCastInputTypes with DefaultStringProducingExpression {
+ override def nullIntolerant: Boolean = true
+
+ override def inputTypes: Seq[DataType] = Seq(BinaryType)
+
+ override def contextIndependentFoldable: Boolean = child.contextIndependentFoldable
+
+ protected override def nullSafeEval(input: Any): Any =
+ XXH3.hash128Hex(input.asInstanceOf[Array[Byte]])
+
+ override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ val cls = classOf[XXH3].getName
+ defineCodeGen(ctx, ev, c => s"$cls.hash128Hex($c)")
+ }
+
+ override def prettyName: String = "xxh3_128"
+
+ override protected def withNewChildInternal(newChild: Expression): Xxh3128 =
+ copy(child = newChild)
+}
/**
* A function that calculates hash value for a group of expressions. Note that the `seed` argument
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
index 02846b560ad00..d03db29257580 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
@@ -92,6 +92,17 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkConsistencyBetweenInterpretedAndCodegen(Crc32, BinaryType)
}
+ test("xxh3_64 and xxh3_128") {
+ // Concrete values match the reference implementation; XXH3Suite covers the full vector set.
+ checkEvaluation(Xxh364(Literal("Spark".getBytes(StandardCharsets.UTF_8))), 80997306238743657L)
+ checkEvaluation(Xxh3128(Literal("Spark".getBytes(StandardCharsets.UTF_8))),
+ "7d57dd84c60c86ca1f4e82ab91a12b5e")
+ checkEvaluation(Xxh364(Literal.create(null, BinaryType)), null)
+ checkEvaluation(Xxh3128(Literal.create(null, BinaryType)), null)
+ checkConsistencyBetweenInterpretedAndCodegen(Xxh364, BinaryType)
+ checkConsistencyBetweenInterpretedAndCodegen(Xxh3128, BinaryType)
+ }
+
def checkHiveHash(input: Any, dataType: DataType, expected: Long): Unit = {
// Note : All expected hashes need to be computed using Hive 1.2.1
val actual = HiveHashFunction.hash(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XXH3Suite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XXH3Suite.scala
new file mode 100644
index 0000000000000..1929e9bbe1727
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XXH3Suite.scala
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.SparkFunSuite
+
+/**
+ * Validates the [[XXH3]] port against the reference implementation's known-answer vectors
+ * (github.com/Cyan4973/xxHash). Inputs are prefixes of the same pseudo-random buffer the reference
+ * test harness uses (tests/sanity_test.c); expected values were produced with the reference
+ * `xxhash` library and cover every length branch (0, 1-3, 4-8, 9-16, 17-128, 129-240, and the
+ * long path) with both a zero and a non-zero seed.
+ */
+class XXH3Suite extends SparkFunSuite {
+
+ private val buffer: Array[Byte] = {
+ val buf = new Array[Byte](4200)
+ var byteGen = 0x9E3779B1L
+ for (i <- buf.indices) {
+ buf(i) = (byteGen >>> 56).toByte
+ byteGen *= 0x9E3779B185EBCA8DL
+ }
+ buf
+ }
+
+ private def input(len: Int): Array[Byte] = java.util.Arrays.copyOf(buffer, len)
+
+ // 64-bit: (length, seed, expected)
+ private val vectors64: Seq[(Int, Long, Long)] = Seq(
+ (0, 0L, 0x2d06800538d394c2L),
+ (1, 0L, 0xc44bdff4074eecdbL),
+ (2, 0L, 0x7a9978044cb8a8bbL),
+ (3, 0L, 0x54247382a8d6b94dL),
+ (4, 0L, 0xe5dc74bc51848a51L),
+ (5, 0L, 0xe4243f00720306bbL),
+ (7, 0L, 0x9941e0007f555e50L),
+ (8, 0L, 0x24ccc9acaa9f65e4L),
+ (9, 0L, 0x14d5001c15dd3f2bL),
+ (12, 0L, 0xa713daf0dfbb77e7L),
+ (16, 0L, 0x981b17d36c7498c9L),
+ (17, 0L, 0x796f5acd3a60f862L),
+ (32, 0L, 0x9feaddbdbf57eed3L),
+ (64, 0L, 0x9cb48487720ec49dL),
+ (100, 0L, 0x93cd95432b7d483fL),
+ (128, 0L, 0xfcff24126754d861L),
+ (129, 0L, 0x98f1b0a679a2ca29L),
+ (160, 0L, 0x9d03a319ed4cbd2bL),
+ (200, 0L, 0xbddca58935d7c038L),
+ (240, 0L, 0x81c3c2b67f568ccfL),
+ (241, 0L, 0xc5a639ecd2030e5eL),
+ (256, 0L, 0x55de574ad89d0ac5L),
+ (512, 0L, 0x617e49599013cb6bL),
+ (1024, 0L, 0xdd85c9b5c1109c5cL),
+ (2048, 0L, 0xdd59e2c3a5f038e0L),
+ (4096, 0L, 0xe91206429d1f48f9L),
+ (0, 0x9e3779b185ebca8dL, 0xa8a6b918b2f0364aL),
+ (8, 0x9e3779b185ebca8dL, 0x8f973410999b8f6bL),
+ (16, 0x9e3779b185ebca8dL, 0x663f29333b4db6b1L),
+ (64, 0x9e3779b185ebca8dL, 0x4fe8895db9b8c077L),
+ (240, 0x9e3779b185ebca8dL, 0xcc0f58c27ef3d8eeL),
+ (256, 0x9e3779b185ebca8dL, 0x4d30234b7a3aa61cL),
+ (1024, 0x9e3779b185ebca8dL, 0xef368a8a2ebabaefL)
+ )
+
+ // 128-bit: (length, seed, expected canonical hex)
+ private val vectors128: Seq[(Int, Long, String)] = Seq(
+ (0, 0L, "99aa06d3014798d86001c324468d497f"),
+ (1, 0L, "a6cd5e9392000f6ac44bdff4074eecdb"),
+ (2, 0L, "76750c3c7bf956687a9978044cb8a8bb"),
+ (3, 0L, "20efc49ff02422ea54247382a8d6b94d"),
+ (4, 0L, "970d585ac632bf8e2e7d8d6876a39fe9"),
+ (5, 0L, "62ed587687606b4e057c7ed2c01fa1d1"),
+ (7, 0L, "dd9b6039f79ec416081c22dd284a2f0a"),
+ (8, 0L, "47a7f080d82bb45664c69cab4bb21dc5"),
+ (9, 0L, "564ef6078950d457ed7ccbc501eb7501"),
+ (12, 0L, "6e3efd8fc7802b18061a192713f69ad9"),
+ (16, 0L, "c68c368ecf8a9c05562980258a998629"),
+ (17, 0L, "955fa78643ed3669abbc12d11973d7db"),
+ (32, 0L, "98fc6458710dc2e8278410a17595e3f9"),
+ (64, 0L, "6d90e81a9b0fd622efdb6a44690721a9"),
+ (100, 0L, "9b50b05817ab158e5fcbc2e3295f2476"),
+ (128, 0L, "39992220e045260aebb15e34a7fb5ab1"),
+ (129, 0L, "03815fc91f1b30b686c9e3bc8f0a3b5c"),
+ (160, 0L, "ba5d218964b622ad737126c8d7c09cee"),
+ (200, 0L, "e76ff4780fe18439eb060f1bb3126f5a"),
+ (240, 0L, "aa4202daa2769dc85c9aae94c8ebe5a0"),
+ (241, 0L, "99a80ecf0ecfc647c5a639ecd2030e5e"),
+ (256, 0L, "8b1c66091423d28855de574ad89d0ac5"),
+ (512, 0L, "18d2d110dcc9bca1617e49599013cb6b"),
+ (1024, 0L, "0d30d24071c64c57dd85c9b5c1109c5c"),
+ (2048, 0L, "f736557fd47073a5dd59e2c3a5f038e0"),
+ (4096, 0L, "b9cfaea2ca5626a4e91206429d1f48f9"),
+ (0, 0x9e3779b185ebca8dL, "00feaa732a3ce25ea986dfc5d7605bfe"),
+ (8, 0x9e3779b185ebca8dL, "f50cec145bcd5c5a7b29471dc729b5ff"),
+ (16, 0x9e3779b185ebca8dL, "6ffcb80cd33085c80346d13a7a5498c7"),
+ (64, 0x9e3779b185ebca8dL, "37b738968d40bda59405ba2affa95ceb"),
+ (240, 0x9e3779b185ebca8dL, "29d2133d6ea58c5b604e98db085c1864"),
+ (256, 0x9e3779b185ebca8dL, "aaa57235b92d5e7c4d30234b7a3aa61c"),
+ (1024, 0x9e3779b185ebca8dL, "17600efe2b493a18ef368a8a2ebabaef")
+ )
+
+ test("reference test buffer generation") {
+ val expectedFirst24 = Seq(0x00, 0x52, 0x92, 0x9B, 0xB7, 0x32, 0xA3, 0x24, 0x2D, 0x00, 0xAF,
+ 0x95, 0x0E, 0xEC, 0xB8, 0x93, 0xE3, 0xDF, 0xEF, 0x93, 0xAA, 0xD6, 0xCD, 0x2A)
+ expectedFirst24.zipWithIndex.foreach { case (b, i) =>
+ assert((buffer(i) & 0xFF) == b,
+ s"buffer($i): expected 0x${b.toHexString} got 0x${(buffer(i) & 0xFF).toHexString}")
+ }
+ }
+
+ test("XXH3 64-bit against reference vectors") {
+ vectors64.foreach { case (len, seed, expected) =>
+ val actual = XXH3.hash64(input(len), seed)
+ assert(actual == expected, s"len=$len seed=0x${seed.toHexString}: " +
+ s"expected 0x${expected.toHexString} but got 0x${actual.toHexString}")
+ }
+ }
+
+ test("XXH3 128-bit against reference vectors") {
+ vectors128.foreach { case (len, seed, expected) =>
+ val actual = XXH3.hash128Hex(input(len), seed).toString
+ assert(actual == expected, s"len=$len seed=0x${seed.toHexString}: " +
+ s"expected $expected but got $actual")
+ }
+ }
+}
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index 068460293f15c..02d6576d539a0 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -1711,6 +1711,14 @@ class PlanGenerationTestSuite extends ConnectFunSuite with Logging {
fn.crc32(fn.col("g").cast("binary"))
}
+ functionTest("xxh3_64") {
+ fn.xxh3_64(fn.col("g").cast("binary"))
+ }
+
+ functionTest("xxh3_128") {
+ fn.xxh3_128(fn.col("g").cast("binary"))
+ }
+
functionTest("hash") {
fn.hash(fn.col("b"), fn.col("id"))
}
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_128.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_128.explain
new file mode 100644
index 0000000000000..189ed0c540e9c
--- /dev/null
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_xxh3_128.explain
@@ -0,0 +1,2 @@
+Project [xxh3_128(cast(g#0 as binary)) AS xxh3_128(CAST(g AS BINARY))#0]
++- LocalRelation 10.8