Skip to content

Commit 7f0c253

Browse files
yupeagleyupeagle
committed
Addd example
1 parent 4a9156a commit 7f0c253

7 files changed

Lines changed: 342 additions & 0 deletions

File tree

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/README.md‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Building this example in Linux
2+
3+
## Prepare dependencies jar into work dir
4+
5+
bash ./prepare.sh
6+
## Building jars
7+
8+
bash ./build.sh
9+
10+
# Running this sample
11+
12+
The App.java has hardcoded values in it mentioned in the sections below. Those should be pointed to the right values.
13+
14+
Before running this application, the kinit which somes with the JRE needs to be run to get a kerberos token.
15+
16+
`<JRE_INSTALL_ROOT>\bin\kinit.exe <user>@<REALM>`
17+
18+
e.g.
19+
20+
`<JRE_INSTALL_ROOT>\bin\kinit.exe admin@ARIS.LOCAL`
21+
22+
In Linux, you can directly using kinit in system.
23+
24+
e.g.
25+
26+
kinit admin@ARIS.LOCAL
27+
28+
# Sample login.conf. There is an existing login.conf that is being checked in as well.
29+
30+
For more details about login.conf refer https://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html
31+
32+
```
33+
com.sun.security.jgss.login {
34+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true;
35+
};
36+
37+
com.sun.security.jgss.initiate {
38+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true;
39+
};
40+
41+
com.sun.security.jgss.accept {
42+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true;
43+
};
44+
45+
```
46+
47+
In App.java the following properties need to be configured or parameterized using command line.
48+
49+
```
50+
private static String GW_ENDPOINT = "https://knox.tdeupgrade.aris.local:30443/gateway/default/webhdfs/v1";
51+
52+
private static String KDC = "ARIS-WIN2016-DC.aris.local";
53+
54+
private static String DOMAIN_REALM = "ARIS.LOCAL";
55+
56+
private static String LOGIN_CONF_PATH = "login.conf";
57+
58+
59+
```
60+
# Upload file example
61+
62+
```
63+
java -cp "./running_work_dir/*" com.microsoft.mssql.App ./source /tmp/destination
64+
```

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/build.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
mvn compile
2+
mvn package
3+
cp target/webhdfsclientsample-1.0-SNAPSHOT.jar running_work_dir/
4+
cd runnint_work_dir

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/login.conf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
com.sun.security.jgss.login {
2+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true principal="admin";
3+
};
4+
5+
com.sun.security.jgss.initiate {
6+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true principal="admin";
7+
};
8+
9+
com.sun.security.jgss.accept {
10+
com.sun.security.auth.module.Krb5LoginModule required client=TRUE useTicketCache=true debug=true principal="admin";
11+
};

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/pom.xml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
3+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<groupId>com.microsoft.mssql</groupId>
8+
<artifactId>webhdfsclientsample</artifactId>
9+
<version>1.0-SNAPSHOT</version>
10+
11+
<name>webhdfsclientsample</name>
12+
<!-- FIXME change it to the project's website -->
13+
<url>http://www.example.com</url>
14+
15+
<properties>
16+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
17+
<maven.compiler.source>1.7</maven.compiler.source>
18+
<maven.compiler.target>1.7</maven.compiler.target>
19+
</properties>
20+
21+
<dependencies>
22+
<dependency>
23+
<groupId>junit</groupId>
24+
<artifactId>junit</artifactId>
25+
<version>4.11</version>
26+
<scope>test</scope>
27+
</dependency>
28+
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
29+
<dependency>
30+
<groupId>org.apache.httpcomponents</groupId>
31+
<artifactId>httpclient</artifactId>
32+
<version>4.5.13</version>
33+
</dependency>
34+
<dependency>
35+
<groupId>org.apache.httpcomponents</groupId>
36+
<artifactId>httpmime</artifactId>
37+
<version>4.5.13</version>
38+
</dependency>
39+
</dependencies>
40+
41+
<build>
42+
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
43+
<plugins>
44+
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
45+
<plugin>
46+
<artifactId>maven-clean-plugin</artifactId>
47+
<version>3.1.0</version>
48+
</plugin>
49+
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
50+
<plugin>
51+
<artifactId>maven-resources-plugin</artifactId>
52+
<version>3.0.2</version>
53+
<executions>
54+
<execution>
55+
<id>copy-resources</id>
56+
<phase>validate</phase>
57+
<goals>
58+
<goal>copy-resources</goal>
59+
</goals>
60+
<configuration>
61+
<outputDirectory>${basedir}/target/Crunchify</outputDirectory>
62+
<resources>
63+
<resource>
64+
<directory>resources</directory>
65+
<filtering>true</filtering>
66+
</resource>
67+
</resources>
68+
</configuration>
69+
</execution>
70+
</executions>
71+
</plugin>
72+
<plugin>
73+
<groupId>org.apache.maven.plugins</groupId>
74+
<artifactId>maven-dependency-plugin</artifactId>
75+
<executions>
76+
<execution>
77+
<id>copy-dependencies</id>
78+
<phase>prepare-package</phase>
79+
<goals>
80+
<goal>copy-dependencies</goal>
81+
</goals>
82+
<configuration>
83+
<outputDirectory>${project.build.directory}/Crunchify/lib</outputDirectory>
84+
<overWriteReleases>false</overWriteReleases>
85+
<overWriteSnapshots>false</overWriteSnapshots>
86+
<overWriteIfNewer>true</overWriteIfNewer>
87+
</configuration>
88+
</execution>
89+
</executions>
90+
</plugin>
91+
<plugin>
92+
<artifactId>maven-compiler-plugin</artifactId>
93+
<version>3.8.0</version>
94+
</plugin>
95+
<plugin>
96+
<artifactId>maven-surefire-plugin</artifactId>
97+
<version>2.22.1</version>
98+
</plugin>
99+
<plugin>
100+
<artifactId>maven-jar-plugin</artifactId>
101+
<version>3.0.2</version>
102+
</plugin>
103+
<plugin>
104+
<artifactId>maven-install-plugin</artifactId>
105+
<version>2.5.2</version>
106+
</plugin>
107+
<plugin>
108+
<artifactId>maven-deploy-plugin</artifactId>
109+
<version>2.8.2</version>
110+
</plugin>
111+
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
112+
<plugin>
113+
<artifactId>maven-site-plugin</artifactId>
114+
<version>3.7.1</version>
115+
</plugin>
116+
<plugin>
117+
<artifactId>maven-project-info-reports-plugin</artifactId>
118+
<version>3.0.0</version>
119+
</plugin>
120+
</plugins>
121+
</pluginManagement>
122+
</build>
123+
</project>

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/prepare.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Prepare dependencies jars into a work dir
2+
mvn dependency:copy-dependencies -DoutputDirectory=running_work_dir -Dhttps.protocols=TLSv1.2

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/src/main/java/com/microsoft/mssql/App.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package com.microsoft.mssql;
2+
3+
import java.security.Principal;
4+
import java.io.File;
5+
6+
import org.apache.http.HttpEntity;
7+
import org.apache.http.auth.AuthScope;
8+
import org.apache.http.auth.Credentials;
9+
import org.apache.http.client.CredentialsProvider;
10+
import org.apache.http.client.methods.CloseableHttpResponse;
11+
import org.apache.http.client.methods.HttpGet;
12+
import org.apache.http.client.methods.HttpPut;
13+
import org.apache.http.client.methods.HttpUriRequest;
14+
import org.apache.http.client.methods.RequestBuilder;
15+
import org.apache.http.impl.client.CloseableHttpClient;
16+
import org.apache.http.entity.mime.MultipartEntityBuilder;
17+
import org.apache.http.impl.client.HttpClients;
18+
import org.apache.http.util.EntityUtils;
19+
20+
/**
21+
* Sample app for Encryption Zone interactions with WebHdfs.
22+
*/
23+
public class App {
24+
25+
// Properties to be configured before running the application.
26+
private static String GW_ENDPOINT = "https://knox.test.azdata.local:30443/gateway/default/webhdfs/v1";
27+
28+
private static String KDC = "winad.azdata.local";
29+
30+
private static String DOMAIN_REALM = "AZDATA.LOCAL";
31+
32+
// The existing login.conf along with this project can be modified instead of providing a new path.
33+
private static String LOGIN_CONF_PATH = "login.conf";
34+
// End of properties to be configured.
35+
36+
static Credentials emptyCredentials = new Credentials() {
37+
public String getPassword() {
38+
return null;
39+
}
40+
public Principal getUserPrincipal() {
41+
return null;
42+
}
43+
};
44+
45+
static CredentialsProvider provider = new CredentialsProvider() {
46+
@Override
47+
public void setCredentials(AuthScope authscope, Credentials credentials) {
48+
// No op
49+
}
50+
51+
@Override
52+
public Credentials getCredentials(AuthScope authscope) {
53+
//TODO: this can be enhanced to pass controller use name and password in case the AuthScope authscheme is basic.
54+
return emptyCredentials;
55+
}
56+
57+
@Override
58+
public void clear() {
59+
// No op. Nothing to clean up.
60+
}
61+
};
62+
private static void uploadResource(String inputFilePath, String outputFilePath){
63+
System.out.println("Entering create resource");
64+
// Construct Knox endpoint
65+
String createOperationEndpoint = GW_ENDPOINT + outputFilePath + "?op=CREATE&overwrite=true";
66+
try (CloseableHttpClient client2 = HttpClients.custom().setDefaultCredentialsProvider(provider).build()) {
67+
HttpUriRequest request = new HttpPut(createOperationEndpoint);
68+
// First request to get the location in data nodes
69+
try (CloseableHttpResponse response = client2.execute(request)) {
70+
System.out.println("===============");
71+
String newlocation = response.getFirstHeader("Location").getValue();
72+
HttpEntity entity = response.getEntity();
73+
System.out.println("----------------------------------------");
74+
System.out.println(response.getStatusLine());
75+
System.out.println("----------------------------------------");
76+
if (entity != null) {
77+
System.out.println(EntityUtils.toString(entity));
78+
}
79+
System.out.println("----------------------------------------");
80+
EntityUtils.consume(entity);
81+
// Second request to put the content to that location
82+
File testUploadFile = new File(inputFilePath);
83+
HttpEntity putData = MultipartEntityBuilder.create().addBinaryBody("upfile", testUploadFile).build();
84+
HttpUriRequest putRequest = RequestBuilder.put(newlocation).setEntity(putData).build();//setEntity(postData).build();
85+
System.out.println("Executing request " + putRequest.getRequestLine());
86+
CloseableHttpResponse response2 = client2.execute(putRequest);
87+
HttpEntity entity2 = response2.getEntity();
88+
System.out.println("----------------------------------------");
89+
System.out.println(response2.getStatusLine());
90+
System.out.println("----------------------------------------");
91+
if (entity2 != null) {
92+
System.out.println(EntityUtils.toString(entity2));
93+
}
94+
System.out.println("----------------------------------------");
95+
96+
}
97+
} catch (Exception e) {
98+
System.out.println(e);
99+
}
100+
101+
}
102+
103+
public static void main(String[] args) throws Exception {
104+
// These properties can be specified using command line as well by using
105+
// -Djava.security.auth.login.conf=login.conf -Djava.security.krb5.realm=MYDOMAIN.LOCAL
106+
// In case command line is used to pass these arguments, then code below setting system properties should not be used..
107+
System.setProperty("java.security.auth.login.config", LOGIN_CONF_PATH); // The login conf for Java GSS to use. See readme for a sample.
108+
System.setProperty("java.security.krb5.realm", DOMAIN_REALM); // The domain name.
109+
System.setProperty("java.security.krb5.kdc", KDC); // Provide the KDC name here
110+
System.setProperty("sun.security.krb5.debug", "true"); // This can be set to true to print debug information about Kerberos login.
111+
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
112+
113+
for(int i = 0; i < args.length; i++) {
114+
System.out.println(args[i]);
115+
}
116+
uploadResource(args[0], args[1]);
117+
}
118+
}

‎samples/features/sql-big-data-cluster/connectivity/webhdfs-java-client/src/test/java/com/microsoft/mssql/AppTest.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.microsoft.mssql;
2+
3+
import static org.junit.Assert.assertTrue;
4+
5+
import org.junit.Test;
6+
7+
/**
8+
* Unit test for simple App.
9+
*/
10+
public class AppTest
11+
{
12+
/**
13+
* Rigorous Test :-)
14+
*/
15+
@Test
16+
public void shouldAnswerWithTrue()
17+
{
18+
assertTrue( true );
19+
}
20+
}

0 commit comments

Comments
 (0)