Skip to content

Commit f5a0cd1

Browse files
author
Melony QIN
authored
Merge pull request #5 from yupeagle/master
Adding webdfs java client upload example
2 parents da0092a + 89b6e5f commit f5a0cd1

7 files changed

Lines changed: 341 additions & 0 deletions

File tree

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/README.md‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Building this example on Linux
2+
3+
## Prepare dependencies jar into work dir
4+
5+
bash ./prepare.sh
6+
7+
## Building jars
8+
9+
bash ./build.sh
10+
11+
# Running this sample
12+
13+
The App.java has hardcoded values in it mentioned in the sections below. Those should be pointed to the right values.
14+
15+
On Windows, before running this application, kinit needs to be run to get a kerberos token. `kinit` is shipped with the JRE for Windows Java distributions.
16+
17+
`<JRE_INSTALL_ROOT>\bin\kinit.exe <user>@<REALM>`
18+
19+
e.g.
20+
21+
`<JRE_INSTALL_ROOT>\bin\kinit.exe admin@ARIS.LOCAL`
22+
23+
On Linux, kinit can be used. If `kinit` is not installed, then it would need to be installed to get a kerberos token.
24+
25+
e.g.
26+
27+
kinit admin@ARIS.LOCAL
28+
29+
# Sample login.conf. There is an existing login.conf that is being checked in as well.
30+
31+
For more details about login.conf refer https://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html
32+
33+
```
34+
com.sun.security.jgss.login {
35+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true;
36+
};
37+
38+
com.sun.security.jgss.initiate {
39+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true;
40+
};
41+
42+
com.sun.security.jgss.accept {
43+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true;
44+
};
45+
46+
```
47+
48+
In App.java the following properties need to be configured or parameterized using command line.
49+
50+
```
51+
private static String GW_ENDPOINT = "https://knox.tdeupgrade.aris.local:30443/gateway/default/webhdfs/v1";
52+
53+
private static String KDC = "ARIS-WIN2016-DC.aris.local";
54+
55+
private static String DOMAIN_REALM = "ARIS.LOCAL";
56+
57+
private static String LOGIN_CONF_PATH = "login.conf";
58+
59+
60+
```
61+
# Upload file example
62+
63+
```
64+
java -cp "./running_work_dir/*" com.microsoft.mssql.App ./source /tmp/destination
65+
```

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/build.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
mvn compile
2+
mvn package
3+
cp target/webhdfsclientsample-1.0-SNAPSHOT.jar running_work_dir/
4+
cd running_work_dir

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/login.conf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
com.sun.security.jgss.login {
2+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true principal="admin";
3+
};
4+
5+
com.sun.security.jgss.initiate {
6+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=false principal="admin";
7+
};
8+
9+
com.sun.security.jgss.accept {
10+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true principal="admin";
11+
};

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/pom.xml‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
3+
<project xmlns="http://maven.apache.org/POM/4.0.0"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<groupId>com.microsoft.mssql</groupId>
8+
<artifactId>webhdfsclientsample</artifactId>
9+
<version>1.0-SNAPSHOT</version>
10+
11+
<properties>
12+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
13+
<maven.compiler.source>1.7</maven.compiler.source>
14+
<maven.compiler.target>1.7</maven.compiler.target>
15+
</properties>
16+
17+
<dependencies>
18+
<dependency>
19+
<groupId>junit</groupId>
20+
<artifactId>junit</artifactId>
21+
<version>4.11</version>
22+
<scope>test</scope>
23+
</dependency>
24+
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
25+
<dependency>
26+
<groupId>org.apache.httpcomponents</groupId>
27+
<artifactId>httpclient</artifactId>
28+
<version>4.5.13</version>
29+
</dependency>
30+
<dependency>
31+
<groupId>org.apache.httpcomponents</groupId>
32+
<artifactId>httpmime</artifactId>
33+
<version>4.5.13</version>
34+
</dependency>
35+
</dependencies>
36+
37+
<build>
38+
<pluginManagement> <!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
39+
<plugins>
40+
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
41+
<plugin>
42+
<artifactId>maven-clean-plugin</artifactId>
43+
<version>3.1.0</version>
44+
</plugin>
45+
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
46+
<plugin>
47+
<artifactId>maven-resources-plugin</artifactId>
48+
<version>3.0.2</version>
49+
<executions>
50+
<execution>
51+
<id>copy-resources</id>
52+
<phase>validate</phase>
53+
<goals>
54+
<goal>copy-resources</goal>
55+
</goals>
56+
<configuration>
57+
<outputDirectory>${basedir}/target/Crunchify</outputDirectory>
58+
<resources>
59+
<resource>
60+
<directory>resources</directory>
61+
<filtering>true</filtering>
62+
</resource>
63+
</resources>
64+
</configuration>
65+
</execution>
66+
</executions>
67+
</plugin>
68+
<plugin>
69+
<groupId>org.apache.maven.plugins</groupId>
70+
<artifactId>maven-dependency-plugin</artifactId>
71+
<executions>
72+
<execution>
73+
<id>copy-dependencies</id>
74+
<phase>prepare-package</phase>
75+
<goals>
76+
<goal>copy-dependencies</goal>
77+
</goals>
78+
<configuration>
79+
<outputDirectory>${project.build.directory}/Crunchify/lib</outputDirectory>
80+
<overWriteReleases>false</overWriteReleases>
81+
<overWriteSnapshots>false</overWriteSnapshots>
82+
<overWriteIfNewer>true</overWriteIfNewer>
83+
</configuration>
84+
</execution>
85+
</executions>
86+
</plugin>
87+
<plugin>
88+
<artifactId>maven-compiler-plugin</artifactId>
89+
<version>3.8.0</version>
90+
</plugin>
91+
<plugin>
92+
<artifactId>maven-surefire-plugin</artifactId>
93+
<version>2.22.1</version>
94+
</plugin>
95+
<plugin>
96+
<artifactId>maven-jar-plugin</artifactId>
97+
<version>3.0.2</version>
98+
</plugin>
99+
<plugin>
100+
<artifactId>maven-install-plugin</artifactId>
101+
<version>2.5.2</version>
102+
</plugin>
103+
<plugin>
104+
<artifactId>maven-deploy-plugin</artifactId>
105+
<version>2.8.2</version>
106+
</plugin>
107+
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
108+
<plugin>
109+
<artifactId>maven-site-plugin</artifactId>
110+
<version>3.7.1</version>
111+
</plugin>
112+
<plugin>
113+
<artifactId>maven-project-info-reports-plugin</artifactId>
114+
<version>3.0.0</version>
115+
</plugin>
116+
</plugins>
117+
</pluginManagement>
118+
</build>
119+
</project>

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/prepare.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Prepare dependencies jars into a work dir
2+
mvn dependency:copy-dependencies -DoutputDirectory=running_work_dir -Dhttps.protocols=TLSv1.2

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/src/main/java/com/microsoft/mssql/App.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/**
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
*/
4+
package com.microsoft.mssql;
5+
6+
import java.security.Principal;
7+
import java.io.File;
8+
9+
import org.apache.http.HttpEntity;
10+
import org.apache.http.auth.AuthScope;
11+
import org.apache.http.auth.Credentials;
12+
import org.apache.http.client.CredentialsProvider;
13+
import org.apache.http.client.methods.CloseableHttpResponse;
14+
import org.apache.http.client.methods.HttpGet;
15+
import org.apache.http.client.methods.HttpPut;
16+
import org.apache.http.client.methods.HttpUriRequest;
17+
import org.apache.http.client.methods.RequestBuilder;
18+
import org.apache.http.impl.client.CloseableHttpClient;
19+
import org.apache.http.entity.mime.MultipartEntityBuilder;
20+
import org.apache.http.impl.client.HttpClients;
21+
import org.apache.http.util.EntityUtils;
22+
23+
/**
24+
* Sample app for Encryption Zone interactions with WebHdfs.
25+
*/
26+
public class App {
27+
28+
// Properties to be configured before running the application.
29+
private static String GW_ENDPOINT = "https://knox.test.azdata.local:30443/gateway/default/webhdfs/v1";
30+
31+
private static String KDC = "winad.azdata.local";
32+
33+
private static String DOMAIN_REALM = "AZDATA.LOCAL";
34+
35+
// The existing login.conf along with this project can be modified instead of providing a new path.
36+
private static String LOGIN_CONF_PATH = "login.conf";
37+
// End of properties to be configured.
38+
39+
static Credentials emptyCredentials = new Credentials() {
40+
public String getPassword() {
41+
return null;
42+
}
43+
public Principal getUserPrincipal() {
44+
return null;
45+
}
46+
};
47+
48+
static CredentialsProvider provider = new CredentialsProvider() {
49+
@Override
50+
public void setCredentials(AuthScope authscope, Credentials credentials) {
51+
// No op
52+
}
53+
54+
@Override
55+
public Credentials getCredentials(AuthScope authscope) {
56+
//TODO: this can be enhanced to pass user name and password in case the AuthScope authscheme is basic.
57+
return emptyCredentials;
58+
}
59+
60+
@Override
61+
public void clear() {
62+
// No op. Nothing to clean up.
63+
}
64+
};
65+
66+
private static void uploadResource(String inputFilePath, String outputFilePath){
67+
System.out.println("Entering create resource");
68+
// Construct Knox endpoint
69+
String createOperationEndpoint = GW_ENDPOINT + outputFilePath + "?op=CREATE&overwrite=true";
70+
try (CloseableHttpClient client2 = HttpClients.custom().setDefaultCredentialsProvider(provider).build()) {
71+
HttpUriRequest request = new HttpPut(createOperationEndpoint);
72+
// First request to get the location in data nodes
73+
try (CloseableHttpResponse response = client2.execute(request)) {
74+
System.out.println("===============");
75+
String newlocation = response.getFirstHeader("Location").getValue();
76+
HttpEntity entity = response.getEntity();
77+
System.out.println("----------------------------------------");
78+
System.out.println(response.getStatusLine());
79+
System.out.println("----------------------------------------");
80+
if (entity != null) {
81+
System.out.println(EntityUtils.toString(entity));
82+
}
83+
System.out.println("----------------------------------------");
84+
EntityUtils.consume(entity);
85+
// Second request to put the content to that location
86+
File testUploadFile = new File(inputFilePath);
87+
HttpEntity putData = MultipartEntityBuilder.create().addBinaryBody("upfile", testUploadFile).build();
88+
HttpUriRequest putRequest = RequestBuilder.put(newlocation).setEntity(putData).build();
89+
System.out.println("Executing request " + putRequest.getRequestLine());
90+
CloseableHttpResponse response2 = client2.execute(putRequest);
91+
HttpEntity entity2 = response2.getEntity();
92+
System.out.println("----------------------------------------");
93+
System.out.println(response2.getStatusLine());
94+
System.out.println("----------------------------------------");
95+
if (entity2 != null) {
96+
System.out.println(EntityUtils.toString(entity2));
97+
}
98+
System.out.println("----------------------------------------");
99+
}
100+
} catch (Exception e) {
101+
System.out.println(e);
102+
}
103+
}
104+
105+
public static void main(String[] args) throws Exception {
106+
// These properties can be specified using command line as well by using
107+
// -Djava.security.auth.login.conf=login.conf -Djava.security.krb5.realm=MYDOMAIN.LOCAL
108+
// In case command line is used to pass these arguments, then code below setting system properties should not be used..
109+
System.setProperty("java.security.auth.login.config", LOGIN_CONF_PATH); // The login conf for Java GSS to use. See readme for a sample.
110+
System.setProperty("java.security.krb5.realm", DOMAIN_REALM); // The domain name.
111+
System.setProperty("java.security.krb5.kdc", KDC); // Provide the KDC name here
112+
System.setProperty("sun.security.krb5.debug", "true"); // This can be set to true to print debug information about Kerberos login.
113+
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
114+
115+
for(int i = 0; i < args.length; i++) {
116+
System.out.println(args[i]);
117+
}
118+
uploadResource(args[0], args[1]);
119+
}
120+
}

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/src/test/java/com/microsoft/mssql/AppTest.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.microsoft.mssql;
2+
3+
import static org.junit.Assert.assertTrue;
4+
5+
import org.junit.Test;
6+
7+
/**
8+
* Unit test for simple App.
9+
*/
10+
public class AppTest
11+
{
12+
/**
13+
* Rigorous Test :-)
14+
*/
15+
@Test
16+
public void shouldAnswerWithTrue()
17+
{
18+
assertTrue( true );
19+
}
20+
}

0 commit comments

Comments
 (0)