Skip to content

Commit 9ca2ec0

Browse files
author
psainics
committed
Bump bigtable, patch gcs tink
1 parent 3759978 commit 9ca2ec0

2 files changed

Lines changed: 163 additions & 88 deletions

File tree

‎pom.xml‎

Lines changed: 80 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,86 @@
375375
</exclusion>
376376
</exclusions>
377377
</dependency>
378+
<dependency>
379+
<groupId>org.apache.hadoop</groupId>
380+
<artifactId>hadoop-common</artifactId>
381+
<version>${hadoop.version}</version>
382+
<scope>provided</scope>
383+
<exclusions>
384+
<exclusion>
385+
<groupId>com.google.protobuf</groupId>
386+
<artifactId>protobuf-java</artifactId>
387+
</exclusion>
388+
<exclusion>
389+
<groupId>commons-logging</groupId>
390+
<artifactId>commons-logging</artifactId>
391+
</exclusion>
392+
<exclusion>
393+
<groupId>log4j</groupId>
394+
<artifactId>log4j</artifactId>
395+
</exclusion>
396+
<exclusion>
397+
<groupId>org.slf4j</groupId>
398+
<artifactId>slf4j-log4j12</artifactId>
399+
</exclusion>
400+
<exclusion>
401+
<groupId>org.apache.avro</groupId>
402+
<artifactId>avro</artifactId>
403+
</exclusion>
404+
<exclusion>
405+
<groupId>org.apache.zookeeper</groupId>
406+
<artifactId>zookeeper</artifactId>
407+
</exclusion>
408+
<exclusion>
409+
<artifactId>guava</artifactId>
410+
<groupId>com.google.guava</groupId>
411+
</exclusion>
412+
<exclusion>
413+
<artifactId>jersey-core</artifactId>
414+
<groupId>com.sun.jersey</groupId>
415+
</exclusion>
416+
<exclusion>
417+
<artifactId>jersey-json</artifactId>
418+
<groupId>com.sun.jersey</groupId>
419+
</exclusion>
420+
<exclusion>
421+
<artifactId>jersey-server</artifactId>
422+
<groupId>com.sun.jersey</groupId>
423+
</exclusion>
424+
<exclusion>
425+
<artifactId>servlet-api</artifactId>
426+
<groupId>javax.servlet</groupId>
427+
</exclusion>
428+
<exclusion>
429+
<groupId>org.mortbay.jetty</groupId>
430+
<artifactId>jetty</artifactId>
431+
</exclusion>
432+
<exclusion>
433+
<groupId>org.mortbay.jetty</groupId>
434+
<artifactId>jetty-util</artifactId>
435+
</exclusion>
436+
<exclusion>
437+
<artifactId>jasper-compiler</artifactId>
438+
<groupId>tomcat</groupId>
439+
</exclusion>
440+
<exclusion>
441+
<artifactId>jasper-runtime</artifactId>
442+
<groupId>tomcat</groupId>
443+
</exclusion>
444+
<exclusion>
445+
<artifactId>jsp-api</artifactId>
446+
<groupId>javax.servlet.jsp</groupId>
447+
</exclusion>
448+
<exclusion>
449+
<artifactId>slf4j-api</artifactId>
450+
<groupId>org.slf4j</groupId>
451+
</exclusion>
452+
<exclusion>
453+
<groupId>org.slf4j</groupId>
454+
<artifactId>slf4j-reload4j</artifactId>
455+
</exclusion>
456+
</exclusions>
457+
</dependency>
378458
<dependency>
379459
<!--
380460
Required by bigtable-hbase-1.x-mapreduce instead of excluded non-shaded version.
@@ -393,10 +473,6 @@
393473
<groupId>log4j</groupId>
394474
<artifactId>log4j</artifactId>
395475
</exclusion>
396-
<exclusion>
397-
<groupId>org.apache.htrace</groupId>
398-
<artifactId>htrace-core</artifactId>
399-
</exclusion>
400476
</exclusions>
401477
</dependency>
402478
<dependency>
@@ -417,10 +493,6 @@
417493
<groupId>log4j</groupId>
418494
<artifactId>log4j</artifactId>
419495
</exclusion>
420-
<exclusion>
421-
<groupId>org.apache.htrace</groupId>
422-
<artifactId>htrace-core</artifactId>
423-
</exclusion>
424496
</exclusions>
425497
</dependency>
426498
<dependency>
@@ -448,86 +520,6 @@
448520
<artifactId>google-cloud-spanner</artifactId>
449521
<version>${google.cloud.spanner.version}</version>
450522
</dependency>
451-
<dependency>
452-
<groupId>org.apache.hadoop</groupId>
453-
<artifactId>hadoop-common</artifactId>
454-
<version>${hadoop.version}</version>
455-
<scope>provided</scope>
456-
<exclusions>
457-
<exclusion>
458-
<groupId>com.google.protobuf</groupId>
459-
<artifactId>protobuf-java</artifactId>
460-
</exclusion>
461-
<exclusion>
462-
<groupId>commons-logging</groupId>
463-
<artifactId>commons-logging</artifactId>
464-
</exclusion>
465-
<exclusion>
466-
<groupId>log4j</groupId>
467-
<artifactId>log4j</artifactId>
468-
</exclusion>
469-
<exclusion>
470-
<groupId>org.slf4j</groupId>
471-
<artifactId>slf4j-log4j12</artifactId>
472-
</exclusion>
473-
<exclusion>
474-
<groupId>org.apache.avro</groupId>
475-
<artifactId>avro</artifactId>
476-
</exclusion>
477-
<exclusion>
478-
<groupId>org.apache.zookeeper</groupId>
479-
<artifactId>zookeeper</artifactId>
480-
</exclusion>
481-
<exclusion>
482-
<artifactId>guava</artifactId>
483-
<groupId>com.google.guava</groupId>
484-
</exclusion>
485-
<exclusion>
486-
<artifactId>jersey-core</artifactId>
487-
<groupId>com.sun.jersey</groupId>
488-
</exclusion>
489-
<exclusion>
490-
<artifactId>jersey-json</artifactId>
491-
<groupId>com.sun.jersey</groupId>
492-
</exclusion>
493-
<exclusion>
494-
<artifactId>jersey-server</artifactId>
495-
<groupId>com.sun.jersey</groupId>
496-
</exclusion>
497-
<exclusion>
498-
<artifactId>servlet-api</artifactId>
499-
<groupId>javax.servlet</groupId>
500-
</exclusion>
501-
<exclusion>
502-
<groupId>org.mortbay.jetty</groupId>
503-
<artifactId>jetty</artifactId>
504-
</exclusion>
505-
<exclusion>
506-
<groupId>org.mortbay.jetty</groupId>
507-
<artifactId>jetty-util</artifactId>
508-
</exclusion>
509-
<exclusion>
510-
<artifactId>jasper-compiler</artifactId>
511-
<groupId>tomcat</groupId>
512-
</exclusion>
513-
<exclusion>
514-
<artifactId>jasper-runtime</artifactId>
515-
<groupId>tomcat</groupId>
516-
</exclusion>
517-
<exclusion>
518-
<artifactId>jsp-api</artifactId>
519-
<groupId>javax.servlet.jsp</groupId>
520-
</exclusion>
521-
<exclusion>
522-
<artifactId>slf4j-api</artifactId>
523-
<groupId>org.slf4j</groupId>
524-
</exclusion>
525-
<exclusion>
526-
<groupId>org.slf4j</groupId>
527-
<artifactId>slf4j-reload4j</artifactId>
528-
</exclusion>
529-
</exclusions>
530-
</dependency>
531523
<dependency>
532524
<groupId>com.google.cloud</groupId>
533525
<artifactId>google-cloud-datastore</artifactId>

‎src/main/java/io/cdap/plugin/gcp/crypto/EncryptedFileSystem.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@
2222
import org.apache.hadoop.fs.FSInputStream;
2323
import org.apache.hadoop.fs.FileSystem;
2424
import org.apache.hadoop.fs.FilterFileSystem;
25+
import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
2526
import org.apache.hadoop.fs.Path;
27+
import org.apache.hadoop.fs.PathHandle;
28+
import org.apache.hadoop.fs.impl.OpenFileParameters;
29+
import org.jetbrains.annotations.NotNull;
2630
import org.slf4j.Logger;
2731
import org.slf4j.LoggerFactory;
2832

@@ -32,6 +36,8 @@
3236
import java.nio.channels.Channels;
3337
import java.nio.channels.SeekableByteChannel;
3438
import java.util.Map;
39+
import java.util.concurrent.CompletableFuture;
40+
import java.util.concurrent.CompletionException;
3541

3642
/**
3743
* A hadoop {@link FileSystem} that support files decryption (encryption is currently not supported).
@@ -42,6 +48,7 @@ public class EncryptedFileSystem extends FilterFileSystem {
4248
private static final String FS_SCHEME = CONF_PREFIX + "scheme";
4349
private static final String FS_IMPL = CONF_PREFIX + "impl";
4450
private static final String DECRYPTOR_IMPL = CONF_PREFIX + "decryptor.impl";
51+
private static final int DEFAULT_BUFFER_SIZE = 4096;
4552

4653
private static final Logger LOG = LoggerFactory.getLogger(EncryptedFileSystem.class);
4754

@@ -103,6 +110,82 @@ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
103110
return new FSDataInputStream(new SeekableByteChannelFSInputStream(decryptor.open(fs, path, bufferSize)));
104111
}
105112

113+
/**
114+
* Opens a file asynchronously and returns a {@link FutureDataInputStreamBuilder}
115+
* to build a {@link FSDataInputStream} for the specified {@link Path}.
116+
*
117+
* <p>This implementation returns a builder that constructs an input stream by using a decryptor
118+
* to open the file through a {@link SeekableByteChannelFSInputStream}. The file is read
119+
* with a buffer size of 4096 bytes.</p>
120+
*
121+
* @param path the {@link Path} of the file to open
122+
* @return a {@link FutureDataInputStreamBuilder} that asynchronously builds a {@link FSDataInputStream}
123+
* @throws UnsupportedOperationException if the operation is not supported
124+
*/
125+
@Override
126+
public FutureDataInputStreamBuilder openFile(Path path) throws UnsupportedOperationException {
127+
return new FutureDataInputStreamBuilder() {
128+
@Override
129+
public CompletableFuture<FSDataInputStream> build()
130+
throws IllegalArgumentException, UnsupportedOperationException {
131+
return CompletableFuture.supplyAsync(() -> {
132+
try {
133+
return new FSDataInputStream(
134+
new SeekableByteChannelFSInputStream(decryptor.open(fs, path, DEFAULT_BUFFER_SIZE)));
135+
} catch (Exception e) {
136+
throw new CompletionException(e);
137+
}
138+
});
139+
}
140+
141+
@Override
142+
public FutureDataInputStreamBuilder opt(@NotNull String s, @NotNull String s1) {
143+
return this;
144+
}
145+
146+
@Override
147+
public FutureDataInputStreamBuilder opt(@NotNull String s, @NotNull String... strings) {
148+
return this;
149+
}
150+
151+
@Override
152+
public FutureDataInputStreamBuilder must(@NotNull String s, @NotNull String s1) {
153+
return this;
154+
}
155+
156+
@Override
157+
public FutureDataInputStreamBuilder must(@NotNull String s, @NotNull String... strings) {
158+
return this;
159+
}
160+
};
161+
}
162+
163+
/**
164+
* Opens a file asynchronously using the provided {@link Path}, and returns
165+
* a {@link CompletableFuture} that supplies a {@link FSDataInputStream}.
166+
*
167+
* <p>This method uses a decryptor to open the file and wraps it in a {@link SeekableByteChannelFSInputStream}.
168+
* It uses the buffer size specified in the {@code parameters}; if the buffer size is not greater than zero,
169+
* a default of 4096 bytes is used.</p>
170+
*
171+
* @param path the {@link Path} to the file to open
172+
* @param parameters the {@link OpenFileParameters} containing optional configuration, such as buffer size
173+
* @return a {@link CompletableFuture} that will complete with the {@link FSDataInputStream}
174+
* @throws CompletionException if an exception occurs during file opening
175+
*/
176+
@Override
177+
protected CompletableFuture<FSDataInputStream> openFileWithOptions(Path path, OpenFileParameters parameters) {
178+
return CompletableFuture.supplyAsync(() -> {
179+
try {
180+
int bufferSize = parameters.getBufferSize() > 0 ? parameters.getBufferSize() : 4096;
181+
return new FSDataInputStream(
182+
new SeekableByteChannelFSInputStream(decryptor.open(fs, path, bufferSize)));
183+
} catch (Exception e) {
184+
throw new CompletionException(e);
185+
}
186+
});
187+
}
188+
106189
/**
107190
* A {@link FSInputStream} implementation backed by a {@link SeekableByteChannel}.
108191
*/

0 commit comments

Comments
 (0)