Skip to content
This repository was archived by the owner on May 23, 2025. It is now read-only.

Commit 9bd7976

Browse files
ac74475ac74475
authored
Merge pull request #90 from gchq/develop
Version 0.1.1 bug fix release
2 parents cfe9a37 + 33e9b52 commit 9bd7976

212 files changed

Lines changed: 2302 additions & 2541 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,4 @@ bin/
2929
*.pem
3030
*.DS_Store
3131
_book
32-
node_modules
33-
data/example/exampleObj_file1.txt
32+
node_modules

client-impl/map-reduce-client/pom.xml renamed to client-impl/mapreduce-client/pom.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@
2222
<parent>
2323
<groupId>uk.gov.gchq.palisade</groupId>
2424
<artifactId>client-impl</artifactId>
25-
<version>0.1</version>
25+
<version>0.2-SNAPSHOT</version>
2626
</parent>
2727

28-
<artifactId>map-reduce-client</artifactId>
28+
<artifactId>mapreduce-client</artifactId>
2929

3030
<dependencies>
3131
<dependency>
@@ -51,4 +51,4 @@
5151
<scope>test</scope>
5252
</dependency>
5353
</dependencies>
54-
</project>
54+
</project>

client-impl/map-reduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/InputFormatUtils.java renamed to client-impl/mapreduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/InputFormatUtils.java

File renamed without changes.

client-impl/map-reduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputFormat.java renamed to client-impl/mapreduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputFormat.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,12 @@
2121
import org.apache.hadoop.mapreduce.JobContext;
2222
import org.apache.hadoop.mapreduce.RecordReader;
2323
import org.apache.hadoop.mapreduce.TaskAttemptContext;
24-
2524
import org.slf4j.Logger;
2625
import org.slf4j.LoggerFactory;
2726

2827
import uk.gov.gchq.palisade.data.serialise.Serialiser;
2928
import uk.gov.gchq.palisade.jsonserialisation.JSONSerialiser;
30-
import uk.gov.gchq.palisade.resource.Resource;
29+
import uk.gov.gchq.palisade.resource.LeafResource;
3130
import uk.gov.gchq.palisade.service.PalisadeService;
3231
import uk.gov.gchq.palisade.service.request.RegisterDataRequest;
3332

@@ -53,7 +52,7 @@
5352
*
5453
* @param <V> The value type for the map task
5554
*/
56-
public class PalisadeInputFormat<V> extends InputFormat<Resource, V> {
55+
public class PalisadeInputFormat<V> extends InputFormat<LeafResource, V> {
5756
private static final Logger LOGGER = LoggerFactory.getLogger(PalisadeInputFormat.class);
5857

5958
/**
@@ -206,7 +205,7 @@ public static synchronized PalisadeService getPalisadeService(final JobContext c
206205
* Creates a {@link uk.gov.gchq.palisade.mapreduce.PalisadeRecordReader}.
207206
*/
208207
@Override
209-
public RecordReader<Resource, V> createRecordReader(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
208+
public RecordReader<LeafResource, V> createRecordReader(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
210209
return new PalisadeRecordReader<>();
211210
}
212211

client-impl/map-reduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputSplit.java renamed to client-impl/mapreduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputSplit.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,14 @@
1717

1818
import org.apache.commons.lang3.builder.EqualsBuilder;
1919
import org.apache.commons.lang3.builder.HashCodeBuilder;
20-
2120
import org.apache.hadoop.io.Writable;
2221
import org.apache.hadoop.mapreduce.InputSplit;
2322
import org.apache.hadoop.util.StringUtils;
2423

2524
import uk.gov.gchq.palisade.RequestId;
2625
import uk.gov.gchq.palisade.ToStringBuilder;
2726
import uk.gov.gchq.palisade.jsonserialisation.JSONSerialiser;
28-
import uk.gov.gchq.palisade.resource.Resource;
27+
import uk.gov.gchq.palisade.resource.LeafResource;
2928
import uk.gov.gchq.palisade.service.request.ConnectionDetail;
3029
import uk.gov.gchq.palisade.service.request.DataRequestResponse;
3130

@@ -60,7 +59,7 @@ public PalisadeInputSplit() {
6059
* @param resources the resources to be processed by this split
6160
* @throws NullPointerException if anything is null
6261
*/
63-
public PalisadeInputSplit(final RequestId requestId, final Map<Resource, ConnectionDetail> resources) {
62+
public PalisadeInputSplit(final RequestId requestId, final Map<LeafResource, ConnectionDetail> resources) {
6463
Objects.requireNonNull(requestId, "requestId");
6564
Objects.requireNonNull(resources, "resources");
6665
requestResponse = new DataRequestResponse().requestId(requestId).resources(resources);
@@ -74,7 +73,7 @@ public PalisadeInputSplit(final RequestId requestId, final Map<Resource, Connect
7473
*/
7574
public PalisadeInputSplit(final DataRequestResponse requestResponse) {
7675
Objects.requireNonNull(requestResponse);
77-
this.requestResponse = new DataRequestResponse().resources(requestResponse);
76+
this.requestResponse = requestResponse;
7877
}
7978

8079
/**
@@ -118,6 +117,7 @@ public void write(final DataOutput dataOutput) throws IOException {
118117
Objects.requireNonNull(dataOutput, "dataOutput");
119118
//serialise this class to JSON and write out
120119
byte[] serial = JSONSerialiser.serialise(requestResponse);
120+
121121
dataOutput.writeInt(serial.length);
122122
dataOutput.write(serial);
123123
}

client-impl/map-reduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/PalisadeRecordReader.java renamed to client-impl/mapreduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/PalisadeRecordReader.java

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,14 @@
1818
import org.apache.hadoop.mapreduce.InputSplit;
1919
import org.apache.hadoop.mapreduce.RecordReader;
2020
import org.apache.hadoop.mapreduce.TaskAttemptContext;
21-
2221
import org.slf4j.Logger;
2322
import org.slf4j.LoggerFactory;
2423

2524
import uk.gov.gchq.palisade.data.serialise.Serialiser;
2625
import uk.gov.gchq.palisade.data.service.DataService;
2726
import uk.gov.gchq.palisade.data.service.request.ReadRequest;
2827
import uk.gov.gchq.palisade.data.service.request.ReadResponse;
29-
import uk.gov.gchq.palisade.resource.Resource;
28+
import uk.gov.gchq.palisade.resource.LeafResource;
3029
import uk.gov.gchq.palisade.service.request.ConnectionDetail;
3130
import uk.gov.gchq.palisade.service.request.DataRequestResponse;
3231

@@ -54,7 +53,7 @@
5453
* @implNote In order to do this, we create a DataRequestResponse for each Resource and send it to the data service
5554
* created by the corresponding ConnectionDetail object.
5655
*/
57-
public class PalisadeRecordReader<V> extends RecordReader<Resource, V> {
56+
public class PalisadeRecordReader<V> extends RecordReader<LeafResource, V> {
5857
private static final Logger LOGGER = LoggerFactory.getLogger(PalisadeRecordReader.class);
5958

6059
/**
@@ -70,7 +69,7 @@ public class PalisadeRecordReader<V> extends RecordReader<Resource, V> {
7069
/**
7170
* Iterates through the resources to be processed.
7271
*/
73-
private Iterator<Map.Entry<Resource, ConnectionDetail>> resIt;
72+
private Iterator<Map.Entry<LeafResource, ConnectionDetail>> resIt;
7473

7574
/**
7675
* Value supplier.
@@ -80,7 +79,7 @@ public class PalisadeRecordReader<V> extends RecordReader<Resource, V> {
8079
/**
8180
* The current Palisade resource being processed.
8281
*/
83-
private Resource currentKey;
82+
private LeafResource currentKey;
8483

8584
/**
8685
* The current value in a resource.
@@ -100,7 +99,7 @@ public class PalisadeRecordReader<V> extends RecordReader<Resource, V> {
10099
/**
101100
* Resource that was last attempted before an error occurred.
102101
*/
103-
private Resource errResource;
102+
private LeafResource errResource;
104103

105104
/**
106105
* No-arg constructor.
@@ -203,8 +202,8 @@ private boolean moveToNextResource() {
203202
* data stream
204203
*/
205204
private void setupItemStream() {
206-
Map.Entry<Resource, ConnectionDetail> entry = resIt.next();
207-
final Resource resource = entry.getKey();
205+
Map.Entry<LeafResource, ConnectionDetail> entry = resIt.next();
206+
final LeafResource resource = entry.getKey();
208207
final ConnectionDetail conDetails = entry.getValue();
209208
final DataService service = conDetails.createService();
210209
//lodge request with the data service
@@ -223,7 +222,7 @@ private void setupItemStream() {
223222
* {@inheritDoc}
224223
*/
225224
@Override
226-
public Resource getCurrentKey() throws IOException {
225+
public LeafResource getCurrentKey() throws IOException {
227226
return currentKey;
228227
}
229228

client-impl/map-reduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/ReaderFailureMode.java renamed to client-impl/mapreduce-client/src/main/java/uk/gov/gchq/palisade/mapreduce/ReaderFailureMode.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,12 @@
2424
*/
2525
public enum ReaderFailureMode {
2626
/**
27-
* Cause the entire task to fail when a {@link uk.gov.gchq.palisade.resource.Resource} cannot be retrieved from a
27+
* Cause the entire task to fail when a {@link uk.gov.gchq.palisade.resource.LeafResource} cannot be retrieved from a
2828
* {@link uk.gov.gchq.palisade.data.service.DataService}.
2929
*/
3030
FAIL_ON_READ_FAILURE,
3131
/**
32-
* Allow the task to continue when a {@link uk.gov.gchq.palisade.resource.Resource} cannot be retrieved.
32+
* Allow the task to continue when a {@link uk.gov.gchq.palisade.resource.LeafResource} cannot be retrieved.
3333
*/
3434
CONTINUE_ON_READ_FAILURE,
3535
}

client-impl/map-reduce-client/src/test/java/uk/gov/gchq/palisade/mapreduce/InputFormatUtilsTest.java renamed to client-impl/mapreduce-client/src/test/java/uk/gov/gchq/palisade/mapreduce/InputFormatUtilsTest.java

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
import org.junit.BeforeClass;
1919
import org.junit.Test;
2020

21-
import uk.gov.gchq.palisade.mapreduce.InputFormatUtils;
22-
import uk.gov.gchq.palisade.mapreduce.PalisadeInputSplit;
21+
import uk.gov.gchq.palisade.RequestId;
2322
import uk.gov.gchq.palisade.resource.StubResource;
2423
import uk.gov.gchq.palisade.service.request.DataRequestResponse;
2524
import uk.gov.gchq.palisade.service.request.StubConnectionDetail;
@@ -35,19 +34,21 @@
3534
import java.util.stream.IntStream;
3635

3736
import static org.junit.Assert.assertEquals;
37+
import static org.junit.Assert.fail;
3838

3939
public class InputFormatUtilsTest {
4040

4141
private static DataRequestResponse reqResponse;
4242

4343
@BeforeClass
4444
public static void setup() {
45-
reqResponse = new DataRequestResponse();
46-
reqResponse.getResources().put(new StubResource("type1", "id1", "format1"), new StubConnectionDetail("con1"));
47-
reqResponse.getResources().put(new StubResource("type2", "id2", "format2"), new StubConnectionDetail("con2"));
48-
reqResponse.getResources().put(new StubResource("type3", "id3", "format3"), new StubConnectionDetail("con3"));
49-
reqResponse.getResources().put(new StubResource("type4", "id4", "format4"), new StubConnectionDetail("con4"));
50-
reqResponse.getResources().put(new StubResource("type5", "id5", "format5"), new StubConnectionDetail("con5"));
45+
reqResponse = new DataRequestResponse()
46+
.requestId(new RequestId().id("testID"))
47+
.resource(new StubResource("type1", "id1", "format1"), new StubConnectionDetail("con1"))
48+
.resource(new StubResource("type2", "id2", "format2"), new StubConnectionDetail("con2"))
49+
.resource(new StubResource("type3", "id3", "format3"), new StubConnectionDetail("con3"))
50+
.resource(new StubResource("type4", "id4", "format4"), new StubConnectionDetail("con4"))
51+
.resource(new StubResource("type5", "id5", "format5"), new StubConnectionDetail("con5"));
5152
}
5253

5354
@Test
@@ -85,15 +86,15 @@ public void testListToMapCollectorEmpty() {
8586
);
8687
}
8788

88-
@Test
89-
public void shouldReturnEmptySplits() {
89+
@Test (expected = NullPointerException.class)
90+
public void shouldReturnErrorAsThereAreNoResourcesSet() {
9091
//Given
9192
DataRequestResponse req = new DataRequestResponse();
9293
PrimitiveIterator.OfInt index = IntStream.range(1, 9999).iterator();
9394
//When
94-
List<PalisadeInputSplit> result = InputFormatUtils.toInputSplits(req, index);
95+
InputFormatUtils.toInputSplits(req, index);
9596
//Then
96-
assertEquals(Collections.emptyList(), result);
97+
fail("Test should have thrown a NullPointerException.");
9798
}
9899

99100
@Test

client-impl/map-reduce-client/src/test/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputFormatTest.java renamed to client-impl/mapreduce-client/src/test/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputFormatTest.java

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import org.mockito.Mockito;
99

1010
import uk.gov.gchq.palisade.Context;
11+
import uk.gov.gchq.palisade.RequestId;
1112
import uk.gov.gchq.palisade.UserId;
1213
import uk.gov.gchq.palisade.data.serialise.Serialiser;
1314
import uk.gov.gchq.palisade.data.serialise.SimpleStringSerialiser;
@@ -82,19 +83,17 @@ public void shouldAddMultipleWithComma() {
8283
assertEquals(json, c.get(PalisadeInputFormat.REGISTER_REQUESTS_KEY));
8384
}
8485

85-
@Test
86-
public void shouldAddEmptyRequest() {
86+
@Test (expected = RuntimeException.class)
87+
public void shouldErrorWhenAddingEmptyRequest() {
8788
//Given
8889
Configuration c = new Configuration();
8990
JobContext mockJob = Mockito.mock(JobContext.class);
9091
when(mockJob.getConfiguration()).thenReturn(c);
9192
RegisterDataRequest rdr = new RegisterDataRequest();
92-
RegisterDataRequest[] rdrArray = {rdr};
93-
String json = new String(JSONSerialiser.serialise(rdrArray), PalisadeInputFormat.UTF8);
9493
//When
9594
PalisadeInputFormat.addDataRequest(mockJob, rdr);
9695
//Then
97-
assertEquals(json, c.get(PalisadeInputFormat.REGISTER_REQUESTS_KEY));
96+
fail();
9897
}
9998

10099
@Test
@@ -118,10 +117,9 @@ public void addAndGetRequests() {
118117
when(mockJob.getConfiguration()).thenReturn(c);
119118
RegisterDataRequest rdr = new RegisterDataRequest().resourceId("testResource").userId(new UserId().id("user")).context(new Context().justification("justification"));
120119
RegisterDataRequest rdr2 = new RegisterDataRequest().resourceId("testResource2").userId(new UserId().id("user2")).context(new Context().justification("justification2"));
121-
RegisterDataRequest rdr3 = new RegisterDataRequest();
122120
//When
123-
PalisadeInputFormat.addDataRequests(mockJob, rdr, rdr2, rdr3);
124-
List<RegisterDataRequest> expected = Stream.of(rdr, rdr2, rdr3).collect(Collectors.toList());
121+
PalisadeInputFormat.addDataRequests(mockJob, rdr, rdr2);
122+
List<RegisterDataRequest> expected = Stream.of(rdr, rdr2).collect(Collectors.toList());
125123
//Then
126124
assertEquals(expected, PalisadeInputFormat.getDataRequests(mockJob));
127125
}
@@ -149,17 +147,17 @@ public void shouldThrowOnNoRequests() throws IOException {
149147
@BeforeClass
150148
public static void setup() {
151149
request1 = new RegisterDataRequest().resourceId("res1").userId(new UserId().id("user1")).context(new Context().justification("just1"));
152-
req1Response = new DataRequestResponse();
153-
req1Response.getResources().put(new StubResource("type1", "id1", "format1"), new StubConnectionDetail("con1"));
154-
req1Response.getResources().put(new StubResource("type2", "id2", "format2"), new StubConnectionDetail("con2"));
155-
req1Response.getResources().put(new StubResource("type3", "id3", "format3"), new StubConnectionDetail("con3"));
156-
req1Response.getResources().put(new StubResource("type4", "id4", "format4"), new StubConnectionDetail("con4"));
157-
req1Response.getResources().put(new StubResource("type5", "id5", "format5"), new StubConnectionDetail("con5"));
150+
req1Response = new DataRequestResponse().requestId(new RequestId().id("request1"))
151+
.resource(new StubResource("type1", "id1", "format1"), new StubConnectionDetail("con1"))
152+
.resource(new StubResource("type2", "id2", "format2"), new StubConnectionDetail("con2"))
153+
.resource(new StubResource("type3", "id3", "format3"), new StubConnectionDetail("con3"))
154+
.resource(new StubResource("type4", "id4", "format4"), new StubConnectionDetail("con4"))
155+
.resource(new StubResource("type5", "id5", "format5"), new StubConnectionDetail("con5"));
158156

159157
request2 = new RegisterDataRequest().resourceId("res2").userId(new UserId().id("user2")).context(new Context().justification("just2"));
160-
req2Response = new DataRequestResponse();
161-
req2Response.getResources().put(new StubResource("type_a", "id6", "format6"), new StubConnectionDetail("con6"));
162-
req2Response.getResources().put(new StubResource("type_b", "id7", "format7"), new StubConnectionDetail("con7"));
158+
req2Response = new DataRequestResponse().requestId(new RequestId().id("request2"))
159+
.resource(new StubResource("type_a", "id6", "format6"), new StubConnectionDetail("con6"))
160+
.resource(new StubResource("type_b", "id7", "format7"), new StubConnectionDetail("con7"));
163161
}
164162

165163
/**

client-impl/map-reduce-client/src/test/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputSplitTest.java renamed to client-impl/mapreduce-client/src/test/java/uk/gov/gchq/palisade/mapreduce/PalisadeInputSplitTest.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717

1818
import org.junit.Assert;
1919
import org.junit.Test;
20+
2021
import uk.gov.gchq.palisade.RequestId;
21-
import uk.gov.gchq.palisade.mapreduce.PalisadeInputSplit;
2222
import uk.gov.gchq.palisade.resource.StubResource;
2323
import uk.gov.gchq.palisade.service.request.DataRequestResponse;
2424
import uk.gov.gchq.palisade.service.request.StubConnectionDetail;
@@ -52,9 +52,9 @@ public void shouldntAcceptNegativeLength() throws IOException {
5252
@Test
5353
public void shouldSerialiseToEqualObject() throws IOException {
5454
//Given
55-
DataRequestResponse drr = new DataRequestResponse();
56-
drr.setRequestId(new RequestId().id("test string"));
57-
drr.getResources().put(stubResource, stubConnectionDetail);
55+
DataRequestResponse drr = new DataRequestResponse()
56+
.requestId(new RequestId().id("test string"))
57+
.resource(stubResource, stubConnectionDetail);
5858
PalisadeInputSplit test = new PalisadeInputSplit(drr);
5959

6060
ByteArrayOutputStream buffer = new ByteArrayOutputStream();

0 commit comments

Comments
 (0)