Skip to content

Commit 48c4a51

Browse files
xiaoxin-renxiaoxin-ren
committed
[SDK-649] Fix source VersionId issue and refine the test case.
1 parent a30f21e commit 48c4a51

File tree

3 files changed

+66
-27
lines changed

3 files changed

+66
-27
lines changed

‎src/main/java/com/emc/object/s3/LargeFileUploader.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -911,10 +911,11 @@ public MultipartPartETag call() {
911911
// we were paused or aborted, so should not start any more tasks
912912
throw new CancellationException();
913913
} else {
914-
log.debug("uploading {}/{}, uploadId: {}, partNumber {} (offset: {}, length: {})",
915-
bucket, key, uploadId, partNumber, offset, length);
914+
log.debug("copying {}/{}, uploadId: {}, partNumber {} (offset: {}, length: {}), versionId: {}",
915+
bucket, key, uploadId, partNumber, offset, length, sourceVersionId);
916916
CopyPartRequest copyPartRequest = new CopyPartRequest(srcBucket, srcKey, bucket, key, resumeContext.getUploadId(), partNumber)
917-
.withSourceRange(new Range(offset, offset + length - 1));
917+
.withSourceRange(new Range(offset, offset + length - 1))
918+
.withSourceVersionId(sourceVersionId);
918919
try {
919920
CopyPartResult result = s3Client.copyPart(copyPartRequest);
920921
return new MultipartPartETag(result.getPartNumber(), result.getRawETag());

‎src/main/java/com/emc/object/s3/request/CopyPartRequest.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,10 @@ public CopyPartRequest withSourceRange(Range sourceRange) {
7979
setSourceRange(sourceRange);
8080
return this;
8181
}
82+
83+
@Override
84+
public CopyPartRequest withSourceVersionId(String sourceVersionId) {
85+
super.setSourceVersionId(sourceVersionId);
86+
return this;
87+
}
8288
}

‎src/test/java/com/emc/object/s3/LargeFileUploaderTest.java

Lines changed: 56 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,13 @@
2626
*/
2727
package com.emc.object.s3;
2828

29-
import com.emc.object.s3.bean.GetObjectResult;
30-
import com.emc.object.s3.bean.MultipartPart;
31-
import com.emc.object.s3.bean.MultipartPartETag;
32-
import com.emc.object.s3.bean.Upload;
29+
import com.emc.object.s3.bean.*;
3330
import com.emc.object.s3.jersey.S3JerseyClient;
3431
import com.emc.object.s3.lfu.LargeFileMultipartSource;
3532
import com.emc.object.s3.lfu.LargeFileUpload;
3633
import com.emc.object.s3.lfu.LargeFileUploaderResumeContext;
3734
import com.emc.object.s3.lfu.PartMismatchException;
38-
import com.emc.object.s3.request.AbortMultipartUploadRequest;
39-
import com.emc.object.s3.request.GetObjectRequest;
40-
import com.emc.object.s3.request.ListMultipartUploadsRequest;
41-
import com.emc.object.s3.request.UploadPartRequest;
35+
import com.emc.object.s3.request.*;
4236
import com.emc.object.util.ProgressListener;
4337
import com.emc.rest.util.StreamUtil;
4438
import com.emc.util.RandomInputStream;
@@ -170,37 +164,75 @@ public void testLargeFileUploaderProgressListener() throws Exception {
170164
}
171165

172166
@Test
173-
public void testLFUCopy() {
167+
public void testLFUCopy() throws Exception {
168+
// enable versioning on the bucket
169+
client.setBucketVersioning(getTestBucket(), new VersioningConfiguration().withStatus(VersioningConfiguration.Status.Enabled));
170+
174171
String srcKey = "lfu-source.bin";
175172
String dstKey = "lfu-single-copy.bin";
176-
String dstKey2 = "lfu-multipart-copy.bin";
177-
178173
byte[] data = new byte[(int) MockMultipartSource.totalSize];
179174
new Random().nextBytes(data);
175+
String srcMD5Sum = DigestUtils.md5Hex(data);
176+
client.putObject(getTestBucket(), srcKey, data, null);
177+
String versionId0 = client.listVersions(getTestBucket(), null).getVersions().get(0).getVersionId();
178+
S3ObjectMetadata sourceMetadata0 = client.getObjectMetadata(new GetObjectMetadataRequest(getTestBucket(), srcKey).withVersionId(versionId0));
179+
Assert.assertEquals(MockMultipartSource.totalSize, sourceMetadata0.getContentLength().longValue());
180+
Assert.assertEquals(srcMD5Sum, sourceMetadata0.getETag());
181+
182+
//Upload new Version Object
183+
new Random().nextBytes(data);
184+
String srcMD5SumNew = DigestUtils.md5Hex(data);
180185
client.putObject(getTestBucket(), srcKey, data, null);
181-
S3ObjectMetadata sourceMetadata = client.getObjectMetadata(getTestBucket(), srcKey);
182-
Assert.assertEquals(MockMultipartSource.totalSize, sourceMetadata.getContentLength().longValue());
186+
String versionId1 = client.listVersions(getTestBucket(), null).getVersions().get(0).getVersionId();
187+
S3ObjectMetadata sourceMetadata1 = client.getObjectMetadata(new GetObjectMetadataRequest(getTestBucket(), srcKey).withVersionId(versionId1));
188+
Assert.assertEquals(MockMultipartSource.totalSize, sourceMetadata1.getContentLength().longValue());
189+
Assert.assertEquals(srcMD5SumNew, sourceMetadata1.getETag());
183190

184-
//Single Copy
191+
//Single Copy from old version object
185192
S3ObjectMetadata objectMetadata = new S3ObjectMetadata().addUserMetadata("key", "value");
186193
LargeFileUploader uploader = new TestLargeFileUploader(client, getTestBucket(), srcKey, getTestBucket(), dstKey)
194+
.withObjectMetadata(objectMetadata)
195+
.withSourceVersionId(versionId0);
196+
uploader.upload();
197+
Assert.assertEquals(0, uploader.getBytesTransferred());
198+
Assert.assertEquals(srcMD5Sum, uploader.getETag());
199+
Assert.assertEquals(MockMultipartSource.totalSize, client.getObjectMetadata(getTestBucket(), dstKey).getContentLength().longValue());
200+
Assert.assertEquals(objectMetadata.getUserMetadata(), client.getObjectMetadata(getTestBucket(), dstKey).getUserMetadata());
201+
202+
//Single Copy from new version object (without providing version Id)
203+
uploader = new TestLargeFileUploader(client, getTestBucket(), srcKey, getTestBucket(), dstKey)
187204
.withObjectMetadata(objectMetadata);
188205
uploader.upload();
189206
Assert.assertEquals(0, uploader.getBytesTransferred());
190-
Assert.assertEquals(sourceMetadata.getETag(), uploader.getETag());
207+
Assert.assertEquals(srcMD5SumNew, uploader.getETag());
208+
Assert.assertEquals(MockMultipartSource.totalSize, client.getObjectMetadata(getTestBucket(), dstKey).getContentLength().longValue());
209+
Assert.assertEquals(objectMetadata.getUserMetadata(), client.getObjectMetadata(getTestBucket(), dstKey).getUserMetadata());
210+
211+
//Multi Part Copy from old version object
212+
uploader = new TestLargeFileUploader(client, getTestBucket(), srcKey, getTestBucket(), dstKey)
213+
.withObjectMetadata(objectMetadata)
214+
.withPartSize(MockMultipartSource.partSize)
215+
.withMpuThreshold(MockMultipartSource.partSize)
216+
.withSourceVersionId(versionId0);
217+
uploader.upload();
218+
Assert.assertEquals(0, uploader.getBytesTransferred());
219+
Assert.assertTrue(uploader.getETag().contains("-")); // hyphen signifies multipart / updated object
191220
Assert.assertEquals(MockMultipartSource.totalSize, client.getObjectMetadata(getTestBucket(), dstKey).getContentLength().longValue());
192221
Assert.assertEquals(objectMetadata.getUserMetadata(), client.getObjectMetadata(getTestBucket(), dstKey).getUserMetadata());
222+
Assert.assertEquals(srcMD5Sum, DigestUtils.md5Hex(client.getObject(getTestBucket(), dstKey).getObject()));
193223

194-
//Multi Part Copy
195-
LargeFileUploader uploader2 = new TestLargeFileUploader(client, getTestBucket(), srcKey, getTestBucket(), dstKey2)
224+
//Multi Part Copy from new version object
225+
uploader = new TestLargeFileUploader(client, getTestBucket(), srcKey, getTestBucket(), dstKey)
196226
.withObjectMetadata(objectMetadata)
197227
.withPartSize(MockMultipartSource.partSize)
198-
.withMpuThreshold(MockMultipartSource.partSize);
199-
uploader2.upload();
200-
Assert.assertEquals(0, uploader2.getBytesTransferred());
201-
Assert.assertTrue(uploader2.getETag().contains("-")); // hyphen signifies multipart / updated object
202-
Assert.assertEquals(MockMultipartSource.totalSize, client.getObjectMetadata(getTestBucket(), dstKey2).getContentLength().longValue());
203-
Assert.assertEquals(objectMetadata.getUserMetadata(), client.getObjectMetadata(getTestBucket(), dstKey2).getUserMetadata());
228+
.withMpuThreshold(MockMultipartSource.partSize)
229+
.withSourceVersionId(versionId1);
230+
uploader.upload();
231+
Assert.assertEquals(0, uploader.getBytesTransferred());
232+
Assert.assertTrue(uploader.getETag().contains("-")); // hyphen signifies multipart / updated object
233+
Assert.assertEquals(MockMultipartSource.totalSize, client.getObjectMetadata(getTestBucket(), dstKey).getContentLength().longValue());
234+
Assert.assertEquals(objectMetadata.getUserMetadata(), client.getObjectMetadata(getTestBucket(), dstKey).getUserMetadata());
235+
Assert.assertEquals(srcMD5SumNew, DigestUtils.md5Hex(client.getObject(getTestBucket(), dstKey).getObject()));
204236
}
205237

206238
@Test
@@ -244,7 +276,7 @@ public void testLFUCopyPauseResume() throws Exception {
244276
.withResumeContext(resumeContext);
245277
lfu.doMultipartUpload();
246278

247-
// Unfortunately, MPU Etag is not preserved even if copy parts matches the original multipart upload.
279+
// Unfortunately, MPU ETag is not preserved even if copy parts matches the original multipart upload.
248280
// So the content verification is done by calculating md5sum instead of checking ETag.
249281
//Assert.assertEquals(mockMultipartSource.getMpuETag(), client.getObjectMetadata(getTestBucket(), dstKey).getETag());
250282
Assert.assertEquals(DigestUtils.md5Hex(client.getObject(getTestBucket(), srcKey).getObject()), DigestUtils.md5Hex(client.getObject(getTestBucket(), dstKey).getObject()));

0 commit comments

Comments
 (0)