Skip to content

Commit f2a8c12

Browse files
L-ApplinL-Applin
committed
split upload tests per client type: sync, async, crt, multipart-java. Lower large non-file content from 80mib to 60mib to attempt lower build time.
1 parent 4c57bfe commit f2a8c12

File tree

9 files changed

+718
-262
lines changed

9 files changed

+718
-262
lines changed

‎.github/workflows/codebuild-ci.yml‎

Lines changed: 60 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -167,11 +167,46 @@ jobs:
167167
uses: aws-actions/aws-codebuild-run-build@v1
168168
with:
169169
project-name: aws-sdk-java-v2-migration-test
170-
s3-regression-tests-upload:
170+
171+
s3-regression-tests-download:
172+
if: github.repository == 'aws/aws-sdk-java-v2'
173+
runs-on: ubuntu-latest
174+
env:
175+
REGRESSION_TEST: DownloadStreamingRegressionTesting
176+
steps:
177+
- name: Configure AWS Credentials
178+
uses: aws-actions/configure-aws-credentials@v4
179+
with:
180+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
181+
aws-region: us-west-2
182+
role-duration-seconds: 7200
183+
- name: Run s3 regression tests for downloads
184+
uses: aws-actions/aws-codebuild-run-build@v1
185+
with:
186+
project-name: aws-sdk-java-v2-s3-regression-tests
187+
env-vars-for-codebuild: REGRESSION_TEST
188+
s3-regression-tests-control-plane:
189+
if: github.repository == 'aws/aws-sdk-java-v2'
190+
runs-on: ubuntu-latest
191+
env:
192+
REGRESSION_TEST: ControlPlaneOperationRegressionTesting
193+
steps:
194+
- name: Configure AWS Credentials
195+
uses: aws-actions/configure-aws-credentials@v4
196+
with:
197+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
198+
aws-region: us-west-2
199+
role-duration-seconds: 7200
200+
- name: Run s3 regression tests for control plane
201+
uses: aws-actions/aws-codebuild-run-build@v1
202+
with:
203+
project-name: aws-sdk-java-v2-s3-regression-tests
204+
env-vars-for-codebuild: REGRESSION_TEST
205+
s3-regression-tests-upload-sync:
171206
if: github.repository == 'aws/aws-sdk-java-v2'
172207
runs-on: ubuntu-latest
173208
env:
174-
REGRESSION_TEST: UploadStreamingRegressionTesting
209+
REGRESSION_TEST: UploadSyncRegressionTesting
175210
steps:
176211
- name: Configure AWS Credentials
177212
uses: aws-actions/configure-aws-credentials@v4
@@ -184,36 +219,53 @@ jobs:
184219
with:
185220
project-name: aws-sdk-java-v2-s3-regression-tests
186221
env-vars-for-codebuild: REGRESSION_TEST
187-
s3-regression-tests-download:
222+
s3-regression-tests-upload-async:
188223
if: github.repository == 'aws/aws-sdk-java-v2'
189224
runs-on: ubuntu-latest
190225
env:
191-
REGRESSION_TEST: DownloadStreamingRegressionTesting
226+
REGRESSION_TEST: UploadAsyncRegressionTesting
192227
steps:
193228
- name: Configure AWS Credentials
194229
uses: aws-actions/configure-aws-credentials@v4
195230
with:
196231
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
197232
aws-region: us-west-2
198233
role-duration-seconds: 7200
199-
- name: Run s3 regression tests for downloads
234+
- name: Run s3 regression tests for uploads
200235
uses: aws-actions/aws-codebuild-run-build@v1
201236
with:
202237
project-name: aws-sdk-java-v2-s3-regression-tests
203238
env-vars-for-codebuild: REGRESSION_TEST
204-
s3-regression-tests-control-plane:
239+
s3-regression-tests-upload-crt:
205240
if: github.repository == 'aws/aws-sdk-java-v2'
206241
runs-on: ubuntu-latest
207242
env:
208-
REGRESSION_TEST: ControlPlaneOperationRegressionTesting
243+
REGRESSION_TEST: UploadCrtRegressionTesting
209244
steps:
210245
- name: Configure AWS Credentials
211246
uses: aws-actions/configure-aws-credentials@v4
212247
with:
213248
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
214249
aws-region: us-west-2
215250
role-duration-seconds: 7200
216-
- name: Run s3 regression tests for control plane
251+
- name: Run s3 regression tests for uploads
252+
uses: aws-actions/aws-codebuild-run-build@v1
253+
with:
254+
project-name: aws-sdk-java-v2-s3-regression-tests
255+
env-vars-for-codebuild: REGRESSION_TEST
256+
s3-regression-tests-upload-multi:
257+
if: github.repository == 'aws/aws-sdk-java-v2'
258+
runs-on: ubuntu-latest
259+
env:
260+
REGRESSION_TEST: UploadTransferManagerRegressionTesting
261+
steps:
262+
- name: Configure AWS Credentials
263+
uses: aws-actions/configure-aws-credentials@v4
264+
with:
265+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
266+
aws-region: us-west-2
267+
role-duration-seconds: 7200
268+
- name: Run s3 regression tests for uploads
217269
uses: aws-actions/aws-codebuild-run-build@v1
218270
with:
219271
project-name: aws-sdk-java-v2-s3-regression-tests

‎test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import software.amazon.awssdk.services.s3.model.LocationInfo;
4242
import software.amazon.awssdk.services.s3.model.LocationType;
4343
import software.amazon.awssdk.services.s3.model.S3Exception;
44+
import software.amazon.awssdk.services.s3.regression.upload.FlattenUploadConfig;
4445
import software.amazon.awssdk.services.s3control.S3ControlClient;
4546
import software.amazon.awssdk.services.s3control.model.CreateMultiRegionAccessPointRequest;
4647
import software.amazon.awssdk.services.s3control.model.GetMultiRegionAccessPointResponse;
@@ -89,7 +90,7 @@ public static void createBucket(S3Client s3, String bucketName, int retryCount,
8990
}
9091
} else if ("OperationAborted".equals(e.awsErrorDetails().errorCode())) {
9192
log.warn(() -> e.awsErrorDetails().errorMessage() + " --- Likely another operation is creating the bucket, "
92-
+ "just wait for the bucket to be available");
93+
+ "just wait for the bucket to be available");
9394
} else {
9495
throw e;
9596
}
@@ -194,6 +195,12 @@ public static void assumeNotAccessPointWithPathStyle(TestConfig config) {
194195
"Path style doesn't work with ARN type buckets");
195196
}
196197

198+
public static void assumeNotAccessPointWithPathStyle(FlattenUploadConfig config) {
199+
BucketType bucketType = config.getBucketType();
200+
Assumptions.assumeFalse(config.isForcePathStyle() && bucketType.isArnType(),
201+
"Path style doesn't work with ARN type buckets");
202+
}
203+
197204
public static String crc32(String s) {
198205
return crc32(s.getBytes(StandardCharsets.UTF_8));
199206
}
@@ -266,25 +273,22 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, Aw
266273
}
267274
}
268275

269-
public static S3Client makeSyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
270-
Region region, AwsCredentialsProvider provider) {
271-
switch (config.getFlavor()) {
272-
case STANDARD_SYNC:
273-
return S3Client.builder()
274-
.overrideConfiguration(overrideConfiguration)
275-
.forcePathStyle(config.isForcePathStyle())
276-
.requestChecksumCalculation(config.getRequestChecksumValidation())
277-
.region(region)
278-
.credentialsProvider(provider)
279-
.build();
280-
default:
281-
throw new RuntimeException("Unsupported sync flavor: " + config.getFlavor());
282-
}
276+
public static S3Client makeSyncClient(FlattenUploadConfig config, ClientOverrideConfiguration overrideConfiguration,
277+
Region region, AwsCredentialsProvider provider) {
278+
return S3Client.builder()
279+
.overrideConfiguration(overrideConfiguration)
280+
.forcePathStyle(config.isForcePathStyle())
281+
.requestChecksumCalculation(config.getRequestChecksumValidation())
282+
.region(region)
283+
.credentialsProvider(provider)
284+
.build();
283285
}
284286

285-
public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
286-
Region region, AwsCredentialsProvider provider) {
287-
switch (config.getFlavor()) {
287+
public static S3AsyncClient makeAsyncClient(FlattenUploadConfig config,
288+
S3ClientFlavor flavor,
289+
ClientOverrideConfiguration overrideConfiguration,
290+
Region region, AwsCredentialsProvider provider) {
291+
switch (flavor) {
288292
case STANDARD_ASYNC:
289293
return S3AsyncClient.builder()
290294
.overrideConfiguration(overrideConfiguration)
@@ -311,13 +315,15 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideCon
311315
.build();
312316
}
313317
default:
314-
throw new RuntimeException("Unsupported async flavor: " + config.getFlavor());
318+
throw new RuntimeException("Unsupported async flavor: " + flavor);
315319
}
316320
}
317321

318-
public static S3TransferManager makeTm(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
319-
Region region, AwsCredentialsProvider provider) {
320-
S3AsyncClient s3AsyncClient = makeAsyncClient(config, overrideConfiguration, region, provider);
322+
public static S3TransferManager makeTm(FlattenUploadConfig config,
323+
S3ClientFlavor flavor,
324+
ClientOverrideConfiguration overrideConfiguration,
325+
Region region, AwsCredentialsProvider provider) {
326+
S3AsyncClient s3AsyncClient = makeAsyncClient(config, flavor, overrideConfiguration, region, provider);
321327
return S3TransferManager.builder().s3Client(s3AsyncClient).build();
322328
}
323329

‎test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestCallable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public class TestCallable<ResponseT> {
2222
private SdkAutoCloseable client;
2323
private Callable<ResponseT> runnable;
2424

25-
TestCallable(SdkAutoCloseable client, Callable<ResponseT> runnable) {
25+
public TestCallable(SdkAutoCloseable client, Callable<ResponseT> runnable) {
2626
this.client = client;
2727
this.runnable = runnable;
2828
}

‎test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/FlattenUploadConfig.java‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.regression.upload;
17+
18+
import java.util.ArrayList;
19+
import java.util.List;
20+
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
21+
import software.amazon.awssdk.services.s3.regression.BucketType;
22+
import software.amazon.awssdk.utils.ToString;
23+
24+
public class FlattenUploadConfig {
25+
private BucketType bucketType;
26+
private boolean forcePathStyle;
27+
private RequestChecksumCalculation requestChecksumValidation;
28+
private UploadStreamingRegressionTesting.BodyType bodyType;
29+
private UploadStreamingRegressionTesting.ContentSize contentSize;
30+
private boolean payloadSigning;
31+
32+
public static List<FlattenUploadConfig> testConfigs() {
33+
List<FlattenUploadConfig> configs = new ArrayList<>();
34+
35+
boolean[] forcePathStyle = {true, false};
36+
boolean[] payloadSign = {true, false};
37+
RequestChecksumCalculation[] checksumValidations = {RequestChecksumCalculation.WHEN_REQUIRED,
38+
RequestChecksumCalculation.WHEN_SUPPORTED};
39+
for (boolean pathStyle : forcePathStyle) {
40+
for (RequestChecksumCalculation checksumValidation : checksumValidations) {
41+
for (BucketType bucketType : BucketType.values()) {
42+
for (UploadStreamingRegressionTesting.BodyType bodType : UploadStreamingRegressionTesting.BodyType.values()) {
43+
for (UploadStreamingRegressionTesting.ContentSize cs : UploadStreamingRegressionTesting.ContentSize.values()) {
44+
for (boolean ps : payloadSign) {
45+
FlattenUploadConfig testConfig = new FlattenUploadConfig();
46+
testConfig.setBucketType(bucketType);
47+
testConfig.setForcePathStyle(pathStyle);
48+
testConfig.setRequestChecksumValidation(checksumValidation);
49+
testConfig.setBodyType(bodType);
50+
testConfig.setContentSize(cs);
51+
testConfig.setPayloadSigning(ps);
52+
configs.add(testConfig);
53+
}
54+
}
55+
}
56+
}
57+
}
58+
}
59+
60+
return configs;
61+
}
62+
63+
public BucketType getBucketType() {
64+
return bucketType;
65+
}
66+
67+
public void setBucketType(BucketType bucketType) {
68+
this.bucketType = bucketType;
69+
}
70+
71+
public boolean isForcePathStyle() {
72+
return forcePathStyle;
73+
}
74+
75+
public void setForcePathStyle(boolean forcePathStyle) {
76+
this.forcePathStyle = forcePathStyle;
77+
}
78+
79+
public RequestChecksumCalculation getRequestChecksumValidation() {
80+
return requestChecksumValidation;
81+
}
82+
83+
public void setRequestChecksumValidation(RequestChecksumCalculation requestChecksumValidation) {
84+
this.requestChecksumValidation = requestChecksumValidation;
85+
}
86+
87+
public UploadStreamingRegressionTesting.BodyType getBodyType() {
88+
return bodyType;
89+
}
90+
91+
public void setBodyType(UploadStreamingRegressionTesting.BodyType bodyType) {
92+
this.bodyType = bodyType;
93+
}
94+
95+
public UploadStreamingRegressionTesting.ContentSize getContentSize() {
96+
return contentSize;
97+
}
98+
99+
public void setContentSize(UploadStreamingRegressionTesting.ContentSize contentSize) {
100+
this.contentSize = contentSize;
101+
}
102+
103+
public boolean isPayloadSigning() {
104+
return payloadSigning;
105+
}
106+
107+
public void setPayloadSigning(boolean payloadSigning) {
108+
this.payloadSigning = payloadSigning;
109+
}
110+
111+
@Override
112+
public String toString() {
113+
return ToString.builder("FlattenUploadConfig")
114+
.add("bucketType", bucketType)
115+
.add("forcePathStyle", forcePathStyle)
116+
.add("requestChecksumValidation", requestChecksumValidation)
117+
.add("bodyType", bodyType)
118+
.add("contentSize", contentSize)
119+
.add("payloadSigning", payloadSigning)
120+
.build();
121+
}
122+
}

0 commit comments

Comments
 (0)