Skip to content

Commit 11f4e4f

Browse files
victoriaxu098victoriaxu098
authored
Merge pull request #41 from EMCECS/bugfix-SDK-548-Improve-stability-of-tests
[SDK-548] Improve stability of tests for Java SDK
2 parents abcfaf5 + 179411d commit 11f4e4f

File tree

9 files changed

+105
-33
lines changed

9 files changed

+105
-33
lines changed

‎build.gradle

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,11 @@ repositories {
5959
}
6060

6161
dependencies {
62-
implementation 'com.emc.ecs:smart-client:2.3.0',
62+
implementation 'com.emc.ecs:smart-client:2.3.1',
6363
'com.emc.ecs:object-transform:1.1.0',
6464
'commons-codec:commons-codec:1.10',
6565
'org.jdom:jdom2:2.0.6',
66-
'org.slf4j:slf4j-api:1.7.5',
67-
'com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.2',
68-
'com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.2',
69-
'com.sun.jersey.contribs:jersey-apache-client4:1.19.4',
70-
'com.sun.jersey:jersey-client:1.19.4',
71-
'org.apache.httpcomponents:httpclient:4.2.6'
66+
'org.slf4j:slf4j-api:1.7.5'
7267
runtimeOnly 'org.slf4j:slf4j-log4j12:1.7.5'
7368
testImplementation 'junit:junit:4.12',
7469
'log4j:log4j:1.2.17'

‎src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,9 @@ public S3JerseyClient(S3Config config, ClientHandler clientHandler) {
146146
SmartConfig smartConfig = s3Config.toSmartConfig();
147147
loadBalancer = smartConfig.getLoadBalancer();
148148

149+
// S.C. - CHUNKED ENCODING (match ECS buffer size)
150+
smartConfig.setProperty(ClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE, s3Config.getChunkedEncodingSize());
151+
149152
// creates a standard (non-load-balancing) jersey client
150153
if (clientHandler == null) {
151154
client = SmartClientFactory.createStandardClient(smartConfig);
@@ -184,9 +187,6 @@ public S3JerseyClient(S3Config config, ClientHandler clientHandler) {
184187
// S.C. - GEO-PINNING
185188
if (s3Config.isGeoPinningEnabled()) loadBalancer.withVetoRules(new GeoPinningRule());
186189

187-
// S.C. - CHUNKED ENCODING (match ECS buffer size)
188-
smartConfig.setProperty(ClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE, s3Config.getChunkedEncodingSize());
189-
190190
// S.C. - CLIENT CREATION
191191
// create a load-balancing jersey client
192192
if (clientHandler == null) {
@@ -524,10 +524,8 @@ public void putObject(String bucketName, String key, Range range, Object content
524524

525525
@Override
526526
public PutObjectResult putObject(PutObjectRequest request) {
527-
528527
// enable checksum of the object
529528
request.property(RestUtil.PROPERTY_VERIFY_WRITE_CHECKSUM, Boolean.TRUE);
530-
531529
PutObjectResult result = new PutObjectResult();
532530
fillResponseEntity(result, executeAndClose(client, request));
533531
return result;

‎src/test/java/com/emc/object/s3/AbstractS3ClientTest.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ protected static S3Config s3ConfigFromProperties() throws Exception {
102102
String secretKey = TestConfig.getPropertyNotEmpty(props, TestProperties.S3_SECRET_KEY);
103103
URI endpoint = new URI(TestConfig.getPropertyNotEmpty(props, TestProperties.S3_ENDPOINT));
104104
boolean enableVhost = Boolean.parseBoolean(props.getProperty(TestProperties.ENABLE_VHOST));
105+
boolean disableSmartClient = Boolean.parseBoolean(props.getProperty(TestProperties.DISABLE_SMART_CLIENT));
105106
String proxyUri = props.getProperty(TestProperties.PROXY_URI);
106107

107108
S3Config s3Config;
@@ -117,6 +118,8 @@ protected static S3Config s3ConfigFromProperties() throws Exception {
117118

118119
if (proxyUri != null) s3Config.setProperty(ObjectConfig.PROPERTY_PROXY_URI, proxyUri);
119120

121+
if(disableSmartClient)
122+
s3Config.setSmartClient(false);
120123
// uncomment to hit a single node
121124
//s3Config.property(ObjectConfig.PROPERTY_DISABLE_POLLING, true);
122125

‎src/test/java/com/emc/object/s3/ExtendedConfigTest.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ private S3Config loadTestConfig() throws IOException {
2828
String secretKey = TestConfig.getPropertyNotEmpty(props, TestProperties.S3_SECRET_KEY);
2929
URI endpoint = URI.create(TestConfig.getPropertyNotEmpty(props, TestProperties.S3_ENDPOINT));
3030
boolean enableVhost = Boolean.parseBoolean(props.getProperty(TestProperties.ENABLE_VHOST));
31+
boolean disableSmartClient = Boolean.parseBoolean(props.getProperty(TestProperties.DISABLE_SMART_CLIENT));
3132
String proxyUri = props.getProperty(TestProperties.PROXY_URI);
3233

3334
S3Config s3Config;
@@ -43,6 +44,9 @@ private S3Config loadTestConfig() throws IOException {
4344

4445
if (proxyUri != null) s3Config.setProperty(ObjectConfig.PROPERTY_PROXY_URI, proxyUri);
4546

47+
if(disableSmartClient)
48+
s3Config.setSmartClient(false);
49+
4650
return s3Config;
4751
}
4852

‎src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,11 @@ public void testCreateJsonObjectWithStream() {
567567
public void testUpdateMetadata() {
568568
}
569569

570+
@Ignore
571+
@Override
572+
public void testExtendObjectRetentionPeriod() {
573+
}
574+
570575
@Ignore
571576
@Override
572577
public void testPreSignedUrlHeaderOverrides() throws Exception {
@@ -603,4 +608,11 @@ private void incCallCount() {
603608
}
604609
}
605610
}
611+
612+
@Override
613+
protected void assertForListVersionsPaging(int size, int requestCount)
614+
{
615+
Assert.assertEquals("The correct number of versions were NOT returned", 10, size);
616+
Assert.assertEquals("should be 5 pages", 5, requestCount);
617+
}
606618
}

‎src/test/java/com/emc/object/s3/S3JerseyClientTest.java

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -533,6 +533,7 @@ public void testListObjectsWithEncoding() {
533533
}
534534

535535
@Test
536+
// Todo: Blocked by SDK-555.
536537
public void testListAndReadVersions() throws Exception {
537538
// turn on versioning first
538539
client.setBucketVersioning(getTestBucket(),
@@ -572,10 +573,13 @@ public void testListAndReadVersions() throws Exception {
572573
Assert.assertEquals("b13f87dd03c70083eb3e98ca37372361", ((Version) version).getRawETag());
573574
Assert.assertEquals(content, client.readObject(getTestBucket(), key, version.getVersionId(), String.class));
574575
}
576+
// Todo: Blocked by SDK-555. Could be removed after SDK-555 is fixed.
577+
// Delete all the versions
578+
client.deleteVersion(getTestBucket(), key, version.getVersionId());
575579
}
576580
}
577581

578-
@Test // TODO: blocked by STORAGE-21799
582+
@Test // TODO: blocked by SDK-555
579583
public void testListVersionsPaging() {
580584
// turn on versioning first
581585
client.setBucketVersioning(getTestBucket(),
@@ -603,11 +607,23 @@ public void testListVersionsPaging() {
603607
requestCount++;
604608
} while (result.isTruncated());
605609

606-
Assert.assertEquals("The correct number of versions were NOT returned", 6, versions.size());
610+
assertForListVersionsPaging(versions.size(), requestCount);
611+
612+
// Todo: Blocked by SDK-555. Could be removed after SDK-555 is fixed.
613+
for (AbstractVersion version : versions) {
614+
// Delete all the versions
615+
client.deleteVersion(getTestBucket(), version.getKey(), version.getVersionId());
616+
}
617+
}
618+
619+
protected void assertForListVersionsPaging(int size, int requestCount)
620+
{
621+
Assert.assertEquals("The correct number of versions were NOT returned", 6, size);
607622
Assert.assertEquals("should be 3 pages", 3, requestCount);
608623
}
609624

610625
@Test
626+
// Todo: Blocked by SDK-555.
611627
public void testListVersionsPagingPrefixDelim() throws Exception {
612628
// turn on versioning first
613629
client.setBucketVersioning(getTestBucket(),
@@ -633,6 +649,15 @@ public void testListVersionsPagingPrefixDelim() throws Exception {
633649
Assert.assertEquals(1, result.getCommonPrefixes().size());
634650
Assert.assertEquals("prefix/prefix2/", result.getCommonPrefixes().get(0));
635651
Assert.assertFalse(result.isTruncated());
652+
653+
request = new ListVersionsRequest(getTestBucket());
654+
result = client.listVersions(request);
655+
656+
// Todo: Blocked by SDK-555. Could be removed after SDK-555 is fixed.
657+
for (AbstractVersion version : result.getVersions()) {
658+
// Delete all the versions
659+
client.deleteVersion(getTestBucket(), version.getKey(), version.getVersionId());
660+
}
636661
}
637662

638663
protected void createTestObjects(String prefix, int numObjects) {
@@ -1412,6 +1437,16 @@ public void testEmptyObjectChunked() {
14121437
Assert.assertEquals("", client.readObject(getTestBucket(), key, String.class));
14131438
}
14141439

1440+
@Test
1441+
public void testEmptyObjectFile() throws IOException {
1442+
String key = "empty-object";
1443+
File file = File.createTempFile("empty-object-file-test", null);
1444+
file.deleteOnExit();
1445+
PutObjectRequest request = new PutObjectRequest(getTestBucket(), key, file);
1446+
client.putObject(request);
1447+
Assert.assertEquals("", client.readObject(getTestBucket(), key, String.class));
1448+
}
1449+
14151450
@Test
14161451
public void testPutObjectWithSpace() {
14171452
String key = "This Has a Space.txt";
@@ -1790,7 +1825,7 @@ public void testCopyObjectWithMeta() throws Exception {
17901825
Assert.assertEquals(userMeta, objectMetadata.getUserMetadata());
17911826
}
17921827

1793-
@Test // TODO: blocked by STORAGE-12050
1828+
@Test // TODO: blocked by STORAGE-29721
17941829
public void testUpdateMetadata() {
17951830
String key = "update-metadata";
17961831
String content = "Hello update meta!";
@@ -1830,10 +1865,12 @@ public void testUpdateMetadata() {
18301865
client.copyObject(new CopyObjectRequest(getTestBucket(), key, getTestBucket(), key).withObjectMetadata(objectMetadata));
18311866
objectMetadata = client.getObjectMetadata(getTestBucket(), key);
18321867
Assert.assertEquals(ct, objectMetadata.getContentType());
1833-
Assert.assertEquals(cc, objectMetadata.getCacheControl());
1834-
Assert.assertEquals(cd, objectMetadata.getContentDisposition());
1835-
Assert.assertEquals(ce, objectMetadata.getContentEncoding());
1836-
Assert.assertEquals(expires.getTime(), objectMetadata.getHttpExpires());
1868+
// TODO: below assertions are blocked by STORAGE-29721,
1869+
// uncomment them if STORAGE-29721 is fixed
1870+
//Assert.assertEquals(cc, objectMetadata.getCacheControl());
1871+
//Assert.assertEquals(cd, objectMetadata.getContentDisposition());
1872+
//Assert.assertEquals(ce, objectMetadata.getContentEncoding());
1873+
//Assert.assertEquals(expires.getTime(), objectMetadata.getHttpExpires());
18371874
Assert.assertEquals(userMeta, objectMetadata.getUserMetadata());
18381875
}
18391876

@@ -2462,6 +2499,8 @@ public void testPing() {
24622499
@Test
24632500
public void testTimeouts() throws Exception {
24642501
S3Config s3Config = new S3Config(Protocol.HTTP, "8.8.4.4").withIdentity("foo").withSecretKey("bar");
2502+
s3Config.setSmartClient(((S3JerseyClient) client).getS3Config().isSmartClient());
2503+
24652504
s3Config.setRetryLimit(0); // no retries
24662505

24672506
// set timeouts

‎src/test/java/com/emc/object/s3/S3MetadataSearchTest.java

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.emc.object.s3.request.CreateBucketRequest;
66
import com.emc.object.s3.request.PutObjectRequest;
77
import com.emc.object.s3.request.QueryObjectsRequest;
8+
import com.emc.object.util.RestUtil;
89
import org.junit.Assert;
910
import org.junit.Test;
1011

@@ -126,9 +127,16 @@ public void testQueryObjects() throws Exception {
126127
.withAttribute("Size")
127128
.withQuery("(x-amz-meta-string1<='') or (x-amz-meta-string1>='')");
128129
QueryObjectsResult result = client.queryObjects(request);
130+
131+
String version = client.listDataNodes().getVersionInfo();
132+
boolean is34OrLater = version.compareTo("3.4") >= 0;
133+
129134
Assert.assertFalse(result.isTruncated());
130135
Assert.assertEquals(bucketName, result.getBucketName());
131-
Assert.assertEquals("NO MORE PAGES", result.getNextMarker());
136+
if (is34OrLater)
137+
Assert.assertNull(result.getNextMarker());
138+
else
139+
Assert.assertEquals("NO MORE PAGES", result.getNextMarker());
132140
Assert.assertNotNull(result.getObjects());
133141
Assert.assertEquals(1, result.getObjects().size());
134142

@@ -175,41 +183,43 @@ public void testListObjectsWithEncoding() {
175183
));
176184

177185
String badField = "bad-field";
186+
String badFieldValue = "bad\u001dfield";
178187
client.putObject(new PutObjectRequest(getTestBucket(), badField, new byte[0]).withObjectMetadata(
179-
new S3ObjectMetadata().addEncodedUserMetadata("index-field", "bad\u001dfield")
188+
new S3ObjectMetadata().addEncodedUserMetadata("index-field", badFieldValue)
180189
.addUserMetadata("field-valid", "false")
181190
.addUserMetadata("key-valid", "true")
182191
));
183192

184193
try {
185194
// list the bad key
186195
QueryObjectsRequest request = new QueryObjectsRequest(bucketName).withEncodingType(EncodingType.url)
187-
.withQuery("x-amz-meta-field-valid=='true'");
196+
.withQuery("(x-amz-meta-field-valid=='true') and (x-amz-meta-index-field>'')").withSorted("x-amz-meta-index-field");
188197
QueryObjectsResult result = client.queryObjects(request);
189198

190199
Assert.assertEquals(2, result.getObjects().size());
191-
Assert.assertEquals(badKey, result.getObjects().get(0).getObjectName());
192-
Assert.assertEquals(goodKey, result.getObjects().get(1).getObjectName());
200+
Assert.assertEquals(badKey, RestUtil.urlDecode(result.getObjects().get(0).getObjectName()));
201+
Assert.assertEquals(goodKey, RestUtil.urlDecode(result.getObjects().get(1).getObjectName()));
193202

194203
// list a good field, with bad field results
195204
request = new QueryObjectsRequest(bucketName).withEncodingType(EncodingType.url)
196205
.withQuery("x-amz-meta-field-valid=='false'");
197206
result = client.queryObjects(request);
198207

199208
Assert.assertEquals(1, result.getObjects().size());
200-
Assert.assertEquals(badField, result.getObjects().get(0).getObjectName());
209+
Assert.assertEquals(badField, RestUtil.urlDecode(result.getObjects().get(0).getObjectName()));
201210

202211
// list a bad field
203212
request = new QueryObjectsRequest(bucketName).withEncodingType(EncodingType.url)
204-
.withQuery("x-amz-meta-index-field=='bad\u001dfield'");
213+
.withQuery("x-amz-meta-index-field=='" + RestUtil.urlEncode(badFieldValue) + "'");
205214
result = client.queryObjects(request);
206215

207216
Assert.assertEquals(1, result.getObjects().size());
208-
Assert.assertEquals(badField, result.getObjects().get(0).getObjectName());
217+
Assert.assertEquals(badField, RestUtil.urlDecode(result.getObjects().get(0).getObjectName()));
209218

210219
List<QueryMetadata> queryMds = result.getObjects().get(0).getQueryMds();
211220

212-
Assert.assertEquals(1, queryMds.size());
221+
//SYSMD and USERMD
222+
Assert.assertEquals(2, queryMds.size());
213223
QueryMetadata usermd = null;
214224
for (QueryMetadata m : queryMds) {
215225
switch (m.getType()) {
@@ -219,10 +229,10 @@ public void testListObjectsWithEncoding() {
219229
}
220230
}
221231
Assert.assertNotNull(usermd);
222-
223-
Assert.assertEquals("bad\u001dfield", usermd.getMdMap().get("x-amz-meta-index-field"));
224-
Assert.assertEquals("false", usermd.getMdMap().get("x-amz-meta-field-valid"));
225-
Assert.assertEquals("true", usermd.getMdMap().get("x-amz-meta-key-valid"));
232+
//badFieldValue has to be stored in url encoded format. Limit by SDK-553, user application needs to record encoded or not.
233+
Assert.assertEquals(RestUtil.urlEncode(badFieldValue), RestUtil.urlDecode(usermd.getMdMap().get("x-amz-meta-index-field")));
234+
Assert.assertEquals("false", RestUtil.urlDecode(usermd.getMdMap().get("x-amz-meta-field-valid")));
235+
Assert.assertEquals("true", RestUtil.urlDecode(usermd.getMdMap().get("x-amz-meta-key-valid")));
226236
} finally {
227237
client.deleteObject(getTestBucket(), badKey);
228238
}
@@ -256,9 +266,16 @@ public void testCaseSensitivity() throws Exception {
256266
.withAttribute("Size")
257267
.withQuery("(x-amz-meta-STRING1<='') or (x-amz-meta-STRING1>='')");
258268
QueryObjectsResult result = client.queryObjects(request);
269+
270+
String version = client.listDataNodes().getVersionInfo();
271+
boolean is34OrLater = version.compareTo("3.4") >= 0;
272+
259273
Assert.assertFalse(result.isTruncated());
260274
Assert.assertEquals(bucketName, result.getBucketName());
261-
Assert.assertEquals("NO MORE PAGES", result.getNextMarker());
275+
if (is34OrLater)
276+
Assert.assertNull(result.getNextMarker());
277+
else
278+
Assert.assertEquals("NO MORE PAGES", result.getNextMarker());
262279
Assert.assertNotNull(result.getObjects());
263280
Assert.assertEquals(1, result.getObjects().size());
264281

‎src/test/java/com/emc/object/util/TestProperties.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828

2929
public class TestProperties {
3030
public static final String ENABLE_VHOST = "enableVhost";
31+
public static final String DISABLE_SMART_CLIENT = "disableSmartClient";
3132

3233
public static final String S3_ENDPOINT = "s3.endpoint";
3334
public static final String S3_ACCESS_KEY = "s3.access_key";

‎src/test/resources/test.properties.template

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,8 @@ s3.endpoint=http[s]://<ecshost>[:9020|:9021]
1515
## Uncomment to enable vhost-style requests (including the namespace)
1616
#enableVhost=true
1717

18+
## Uncomment to disable smart client
19+
#disableSmartClient=true
20+
1821
## Optional -- HTTP proxy. Useful for debugging.
1922
#http.proxyUri=http://localhost:8888

0 commit comments

Comments
 (0)