diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index fd3ab715d6085..4f0bc158c7311 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -214,7 +214,7 @@ jobs: key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }} - name: Run benchmarks run: | - ./build/sbt -Pscala-${{ inputs.scala }} -Pyarn -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pspark-ganglia-lgpl Test/package + ./build/sbt -Pscala-${{ inputs.scala }} -Pyarn -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pcredential-aws -Pspark-ganglia-lgpl Test/package # Make less noisy cp conf/log4j2.properties.template conf/log4j2.properties sed -i 's/rootLogger.level = info/rootLogger.level = warn/g' conf/log4j2.properties diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6bcd93d5340e4..428a53745a45b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -150,7 +150,7 @@ jobs: docs=false java25=false fi - build=`./dev/is-changed.py -m "core,unsafe,kvstore,avro,utils,utils-java,network-common,network-shuffle,repl,launcher,examples,sketch,variant,api,catalyst,hive-thriftserver,mllib-local,mllib,graphx,streaming,sql-kafka-0-10,streaming-kafka-0-10,streaming-kinesis-asl,kubernetes,hadoop-cloud,spark-ganglia-lgpl,profiler,protobuf,yarn,connect,sql,hive,pipelines"` + build=`./dev/is-changed.py -m "core,unsafe,kvstore,avro,utils,utils-java,network-common,network-shuffle,repl,launcher,examples,sketch,variant,api,catalyst,hive-thriftserver,mllib-local,mllib,graphx,streaming,sql-kafka-0-10,streaming-kafka-0-10,streaming-kinesis-asl,credential-aws,kubernetes,hadoop-cloud,spark-ganglia-lgpl,profiler,protobuf,yarn,connect,sql,hive,pipelines"` build_core_utils=`./dev/is-changed.py -m "core,unsafe,kvstore,utils,utils-java,network-common,network-shuffle,sketch,variant,launcher"` precondition=" { @@ -276,7 +276,7 @@ jobs: mllib-local, mllib, graphx, profiler, pipelines, repl, examples - >- streaming, sql-kafka-0-10, streaming-kafka-0-10, streaming-kinesis-asl, - kubernetes, hadoop-cloud, spark-ganglia-lgpl, protobuf, connect, avro + credential-aws, kubernetes, hadoop-cloud, spark-ganglia-lgpl, protobuf, connect, avro - yarn # Here, we split Hive and SQL tests into some of slow ones and the rest of them. included-tags: [""] @@ -598,7 +598,7 @@ jobs: - name: Build Spark run: | ./build/sbt -Phadoop-3 -Pyarn -Pspark-ganglia-lgpl -Phadoop-cloud -Phive \ - -Pkubernetes -Pjvm-profiler -Pkinesis-asl -Phive-thriftserver \ + -Pkubernetes -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Phive-thriftserver \ -Pdocker-integration-tests -Pkubernetes-integration-tests -Pvolcano \ Test/package streaming-kinesis-asl-assembly/assembly connect/assembly assembly/package - name: Package compile output @@ -1125,7 +1125,7 @@ jobs: run: | export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl clean install + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws clean install # Documentation build docs: diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index f8edd0b723361..64cd4cd004c7a 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -120,7 +120,7 @@ jobs: export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ inputs.java }} - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} clean install - name: Package compile output run: | # Exclude assembly/ from the artifact: 11 of 12 matrix entries wipe it @@ -171,7 +171,7 @@ jobs: - >- repl,sql#hive-thriftserver - >- - connector#kafka-0-10,connector#kafka-0-10-sql,connector#kafka-0-10-token-provider,connector#spark-ganglia-lgpl,connector#protobuf,connector#avro,connector#kinesis-asl + connector#kafka-0-10,connector#kafka-0-10-sql,connector#kafka-0-10-token-provider,connector#spark-ganglia-lgpl,connector#protobuf,connector#avro,connector#kinesis-asl,connector#credential-aws - >- sql#api,sql#catalyst,resource-managers#yarn,resource-managers#kubernetes#core - >- @@ -325,10 +325,10 @@ jobs: # it here. if [ "$MODULES_TO_TEST" = "connect" ]; then echo "Building assembly module for connect tests." - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -pl assembly install + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} -pl assembly install fi else - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} clean install # SPARK-51628: wipe the assembly module so tests exercise the # SPARK-51600 prepend fallback path. Connect tests strongly depend # on a built assembly module, so they are excluded. @@ -339,27 +339,27 @@ jobs: fi if [[ "$INCLUDED_TAGS" != "" ]]; then - ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -Dtest.include.tags="$INCLUDED_TAGS" test -fae + ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} -Dtest.include.tags="$INCLUDED_TAGS" test -fae elif [[ "$MODULES_TO_TEST" == "connect" && "$INPUT_BRANCH" == "branch-4.0" ]]; then # SPARK-53914: Remove sql/connect/client/jdbc from `-pl` for branch-4.0, this branch can be deleted after the EOL of branch-4.0. ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} -pl sql/connect/client/jvm,sql/connect/common,sql/connect/server test -fae elif [[ "$MODULES_TO_TEST" == "connect" ]]; then ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} -pl sql/connect/client/jdbc,sql/connect/client/jvm,sql/connect/common,sql/connect/server test -fae elif [[ "$EXCLUDED_TAGS" != "" ]]; then - ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -Dtest.exclude.tags="$EXCLUDED_TAGS" test -fae + ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} -Dtest.exclude.tags="$EXCLUDED_TAGS" test -fae elif [[ "$MODULES_TO_TEST" == *"sql#hive-thriftserver"* ]]; then # To avoid a compilation loop, for the `sql/hive-thriftserver` module, run `clean install` instead - ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install -fae + ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} clean install -fae elif [[ "$MODULES_TO_TEST" == *"sql#pipelines"* && "$INPUT_BRANCH" == "branch-4.0" ]]; then # SPARK-52441: Remove sql/pipelines from TEST_MODULES for branch-4.0, this branch can be deleted after the EOL of branch-4.0. TEST_MODULES=${TEST_MODULES/,sql\/pipelines/} - ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} test -fae + ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} test -fae elif [[ "$MODULES_TO_TEST" == *"common#utils-java"* && "$INPUT_BRANCH" == "branch-4.0" ]]; then # SPARK-53138: Remove common/utils-java from TEST_MODULES for branch-4.0, this branch can be deleted after the EOL of branch-4.0. TEST_MODULES=${TEST_MODULES/,common\/utils-java/} - ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} test -fae + ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} test -fae else - ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} test -fae + ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-cloud -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Djava.version=${JAVA_VERSION/-ea} test -fae fi - name: Clean up local Maven repository run: | diff --git a/.github/workflows/python_hosted_runner_test.yml b/.github/workflows/python_hosted_runner_test.yml index 7488a0b7b9d15..772adddb55695 100644 --- a/.github/workflows/python_hosted_runner_test.yml +++ b/.github/workflows/python_hosted_runner_test.yml @@ -119,7 +119,7 @@ jobs: - name: Build Spark run: | ./build/sbt -Phadoop-3 -Pyarn -Pspark-ganglia-lgpl -Phadoop-cloud -Phive \ - -Pkubernetes -Pjvm-profiler -Pkinesis-asl -Phive-thriftserver \ + -Pkubernetes -Pjvm-profiler -Pkinesis-asl -Pcredential-aws -Phive-thriftserver \ -Pdocker-integration-tests -Pvolcano \ Test/package streaming-kinesis-asl-assembly/assembly connect/assembly assembly/package - name: Package compile output diff --git a/connector/credential-aws/pom.xml b/connector/credential-aws/pom.xml new file mode 100644 index 0000000000000..b1cdf5c58f370 --- /dev/null +++ b/connector/credential-aws/pom.xml @@ -0,0 +1,74 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.13 + 5.0.0-SNAPSHOT + ../../pom.xml + + + spark-credential-aws_2.13 + + credential-aws + + jar + Spark AWS Credential Provider + + OIDC-to-AWS credential provider that exchanges identity tokens for + temporary AWS credentials via STS AssumeRoleWithWebIdentity. + + https://spark.apache.org/ + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + provided + + + org.apache.spark + spark-tags_${scala.binary.version} + + + + + + org.apache.spark + spark-tags_${scala.binary.version} + test-jar + test + + + org.junit.jupiter + junit-jupiter + test + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/connector/credential-aws/src/main/java/org/apache/spark/security/aws/AwsStsCredentialProvider.java b/connector/credential-aws/src/main/java/org/apache/spark/security/aws/AwsStsCredentialProvider.java new file mode 100644 index 0000000000000..07d8821b65699 --- /dev/null +++ b/connector/credential-aws/src/main/java/org/apache/spark/security/aws/AwsStsCredentialProvider.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.security.aws; + +import java.net.URI; +import java.util.Map; +import java.util.Set; + +import org.apache.spark.annotation.DeveloperApi; +import org.apache.spark.security.CredentialProvider; +import org.apache.spark.security.CredentialResolutionException; +import org.apache.spark.security.ServiceCredential; +import org.apache.spark.security.UserContext; + +/** + * :: DeveloperApi :: + * AWS STS credential provider that exchanges an OIDC identity token for temporary + * AWS credentials via {@code AssumeRoleWithWebIdentity}. + * + * @since 4.3.0 + */ +@DeveloperApi +public class AwsStsCredentialProvider implements CredentialProvider { + + @Override + public void init(Map conf) { + throw new UnsupportedOperationException( + "AwsStsCredentialProvider is a stub. Full implementation in SPARK-57898."); + } + + @Override + public Set supportedSchemes() { + return Set.of("s3a"); + } + + @Override + public ServiceCredential resolve(UserContext user, URI target) + throws CredentialResolutionException { + throw new UnsupportedOperationException( + "AwsStsCredentialProvider is a stub. Full implementation in SPARK-57898."); + } +} diff --git a/connector/credential-aws/src/main/resources/META-INF/services/org.apache.spark.security.CredentialProvider b/connector/credential-aws/src/main/resources/META-INF/services/org.apache.spark.security.CredentialProvider new file mode 100644 index 0000000000000..84b31f53ec263 --- /dev/null +++ b/connector/credential-aws/src/main/resources/META-INF/services/org.apache.spark.security.CredentialProvider @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.spark.security.aws.AwsStsCredentialProvider diff --git a/connector/credential-aws/src/test/java/org/apache/spark/security/aws/AwsStsCredentialProviderSuite.java b/connector/credential-aws/src/test/java/org/apache/spark/security/aws/AwsStsCredentialProviderSuite.java new file mode 100644 index 0000000000000..cab91fcae6368 --- /dev/null +++ b/connector/credential-aws/src/test/java/org/apache/spark/security/aws/AwsStsCredentialProviderSuite.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.security.aws; + +import java.net.URI; +import java.time.Duration; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.Set; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +import org.apache.spark.security.CredentialProvider; + +/** + * Tests that {@link AwsStsCredentialProvider} is discoverable via {@link ServiceLoader}. + */ +public class AwsStsCredentialProviderSuite { + + @Test + void testServiceLoaderDiscovery() { + ServiceLoader loader = ServiceLoader.load(CredentialProvider.class); + boolean found = false; + for (CredentialProvider provider : loader) { + if (provider instanceof AwsStsCredentialProvider) { + found = true; + break; + } + } + assertTrue(found, "AwsStsCredentialProvider should be discoverable via ServiceLoader"); + } + + @Test + void testSupportedSchemes() { + AwsStsCredentialProvider provider = new AwsStsCredentialProvider(); + Set schemes = provider.supportedSchemes(); + assertEquals(Set.of("s3a"), schemes); + } + + @Test + void testInitThrowsUnsupported() { + AwsStsCredentialProvider provider = new AwsStsCredentialProvider(); + assertThrows(UnsupportedOperationException.class, () -> provider.init(Map.of())); + } + + @Test + void testResolveThrowsUnsupported() { + AwsStsCredentialProvider provider = new AwsStsCredentialProvider(); + assertThrows(UnsupportedOperationException.class, + () -> provider.resolve(null, URI.create("s3a://bucket/key"))); + } + + @Test + void testSuggestedTtl() { + AwsStsCredentialProvider provider = new AwsStsCredentialProvider(); + assertEquals(Duration.ofMinutes(15), provider.suggestedTtl()); + } +} diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 161dca83738b3..63524f3ba7c14 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -683,7 +683,7 @@ SCALA_2_12_PROFILES="-Pscala-2.12" HIVE_PROFILES="-Phive -Phive-thriftserver" # Profiles for publishing snapshots and release to Maven Central # We use Apache Hive 2.3 for publishing -PUBLISH_PROFILES="$BASE_PROFILES $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud -Pjvm-profiler" +PUBLISH_PROFILES="$BASE_PROFILES $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Phadoop-cloud -Pjvm-profiler" # Profiles for building binary releases BASE_RELEASE_PROFILES="$BASE_PROFILES -Psparkr" diff --git a/dev/lint-java b/dev/lint-java index ff431301773f3..8095de68fb2a3 100755 --- a/dev/lint-java +++ b/dev/lint-java @@ -20,7 +20,7 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)" -ERRORS=$($SCRIPT_DIR/../build/mvn -Pkinesis-asl -Pspark-ganglia-lgpl -Pkubernetes -Pyarn -Phive -Phive-thriftserver checkstyle:check | grep ERROR) +ERRORS=$($SCRIPT_DIR/../build/mvn -Pkinesis-asl -Pcredential-aws -Pspark-ganglia-lgpl -Pkubernetes -Pyarn -Phive -Phive-thriftserver checkstyle:check | grep ERROR) if test ! -z "$ERRORS"; then echo -e "Checkstyle checks failed at following occurrences:\n$ERRORS" diff --git a/dev/mima b/dev/mima index 39be02a1dd557..eecf1ad1081db 100755 --- a/dev/mima +++ b/dev/mima @@ -24,7 +24,7 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" -SPARK_PROFILES=${1:-"-Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} +SPARK_PROFILES=${1:-"-Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Phive-thriftserver -Phive"} # Capture sbt output to show errors if the command fails OLD_DEPS_OUTPUT="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" 2>&1)" || { diff --git a/dev/sbt-checkstyle b/dev/sbt-checkstyle index f2d5a0fa304ac..a148af6039181 100755 --- a/dev/sbt-checkstyle +++ b/dev/sbt-checkstyle @@ -17,7 +17,7 @@ # limitations under the License. # -SPARK_PROFILES=${1:-"-Pkinesis-asl -Pspark-ganglia-lgpl -Pkubernetes -Pyarn -Phive -Phive-thriftserver -Pjvm-profiler"} +SPARK_PROFILES=${1:-"-Pkinesis-asl -Pcredential-aws -Pspark-ganglia-lgpl -Pkubernetes -Pyarn -Phive -Phive-thriftserver -Pjvm-profiler"} # NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file # with failure (either resolution or compilation); the "q" makes SBT quit. diff --git a/dev/scalastyle b/dev/scalastyle index 09e6c2372614d..f77fb3e1ee224 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -17,7 +17,7 @@ # limitations under the License. # -SPARK_PROFILES=${1:-"-Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive -Pvolcano -Pjvm-profiler -Phadoop-cloud -Pdocker-integration-tests -Pkubernetes-integration-tests"} +SPARK_PROFILES=${1:-"-Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Phive-thriftserver -Phive -Pvolcano -Pjvm-profiler -Phadoop-cloud -Pdocker-integration-tests -Pkubernetes-integration-tests"} # NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file # with failure (either resolution or compilation); the "q" makes SBT quit. diff --git a/dev/spark-test-image-util/docs/build-docs b/dev/spark-test-image-util/docs/build-docs index ca59769f24231..c2ee826832ef6 100755 --- a/dev/spark-test-image-util/docs/build-docs +++ b/dev/spark-test-image-util/docs/build-docs @@ -35,7 +35,7 @@ FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)" SPARK_HOME="$(cd "`dirname "${BASH_SOURCE[0]}"`"/../../..; pwd)" # 1.Compile spark outside the container to prepare for generating documents inside the container. -build/sbt -Phive -Pkinesis-asl clean unidoc package +build/sbt -Phive -Pkinesis-asl -Pcredential-aws clean unidoc package # 2.Build container image. docker buildx build \ diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 77aa43eb8d947..e14cb8dd88a3a 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -410,6 +410,21 @@ def __hash__(self): ) +credential_aws = Module( + name="credential-aws", + dependencies=[tags, core], + source_file_regexes=[ + "connector/credential-aws/", + ], + build_profile_flags=[ + "-Pcredential-aws", + ], + sbt_test_goals=[ + "credential-aws/test", + ], +) + + streaming_kafka_0_10 = Module( name="streaming-kafka-0-10", dependencies=[streaming, core], diff --git a/dev/sparktestsupport/utils.py b/dev/sparktestsupport/utils.py index 029b1627bd0bd..1657aa56da67f 100755 --- a/dev/sparktestsupport/utils.py +++ b/dev/sparktestsupport/utils.py @@ -121,7 +121,8 @@ def determine_modules_to_test(changed_modules, deduplicated=True): >>> sorted([x.name for x in determine_modules_to_test( ... [modules.sql, modules.core], deduplicated=False)]) ... # doctest: +NORMALIZE_WHITESPACE - ['avro', 'catalyst', 'connect', 'core', 'docker-integration-tests', 'examples', 'graphx', + ['avro', 'catalyst', 'connect', 'core', 'credential-aws', 'docker-integration-tests', + 'examples', 'graphx', 'hive', 'hive-thriftserver', 'mllib', 'mllib-local', 'pipelines', 'protobuf', 'pyspark-connect', 'pyspark-core', 'pyspark-errors', 'pyspark-ml', 'pyspark-ml-connect', 'pyspark-mllib', 'pyspark-pandas', 'pyspark-pandas-connect', 'pyspark-pandas-slow', diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 68c61232ea2af..ad93304e73732 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -31,7 +31,7 @@ export LC_ALL=C # NOTE: These should match those in the release publishing script, and be kept in sync with # dev/create-release/release-build.sh HADOOP_MODULE_PROFILES="-Phive-thriftserver -Pkubernetes -Pyarn -Phive \ - -Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud -Pjvm-profiler" + -Pspark-ganglia-lgpl -Pkinesis-asl -Pcredential-aws -Phadoop-cloud -Pjvm-profiler" MVN="build/mvn" HADOOP_HIVE_PROFILES=( hadoop-3-hive-2.3 diff --git a/docs/_plugins/build_api_docs.rb b/docs/_plugins/build_api_docs.rb index 429cef5aa026c..4b6251b804237 100644 --- a/docs/_plugins/build_api_docs.rb +++ b/docs/_plugins/build_api_docs.rb @@ -45,7 +45,7 @@ def build_spark_if_necessary print_header "Building Spark." cd(SPARK_PROJECT_ROOT) - command = "NO_PROVIDED_SPARK_JARS=0 build/sbt -Phive -Pkinesis-asl clean package" + command = "NO_PROVIDED_SPARK_JARS=0 build/sbt -Phive -Pkinesis-asl -Pcredential-aws clean package" puts "Running '#{command}'; this may take a few minutes..." system(command) || raise("Failed to build Spark") # SPARK-53327: Use the modified ResourceImpl.class in spark-catalyst which is compatible with Java 25 @@ -129,7 +129,7 @@ def build_spark_scala_and_java_docs_if_necessary return end - command = "build/sbt -Pkinesis-asl unidoc" + command = "build/sbt -Pkinesis-asl -Pcredential-aws unidoc" puts "Running '#{command}'..." # Two filter passes on the unidoc output, plus an additive fatal-error summary: diff --git a/pom.xml b/pom.xml index e39622ef88d84..de5e2705b827b 100644 --- a/pom.xml +++ b/pom.xml @@ -3519,6 +3519,13 @@ + + credential-aws + + connector/credential-aws + + + test-java-home diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index ef027de5699f6..51f8393350fcc 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -72,10 +72,10 @@ object BuildCommons { udfWorkerProjects val optionallyEnabledProjects@Seq(kubernetes, yarn, - sparkGangliaLgpl, streamingKinesisAsl, profiler, + sparkGangliaLgpl, streamingKinesisAsl, profiler, credentialAws, dockerIntegrationTests, hadoopCloud, kubernetesIntegrationTests) = Seq("kubernetes", "yarn", - "ganglia-lgpl", "streaming-kinesis-asl", "profiler", + "ganglia-lgpl", "streaming-kinesis-asl", "profiler", "credential-aws", "docker-integration-tests", "hadoop-cloud", "kubernetes-integration-tests").map(ProjectRef(buildLocation, _)) val assemblyProjects@Seq(networkYarn, streamingKafka010Assembly, streamingKinesisAslAssembly) = @@ -416,7 +416,8 @@ object SparkBuild extends PomBuild { Seq( spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn, unsafe, tags, tokenProviderKafka010, sqlKafka010, pipelines, connectCommon, connect, - connectJdbc, connectClient, variant, connectShims, profiler, commonUtilsJava, sparkConfig, + connectJdbc, connectClient, variant, connectShims, profiler, credentialAws, + commonUtilsJava, sparkConfig, udfWorkerProto, udfWorkerCore, udfWorkerGrpc ).contains(x) }