Skip to content

Commit

Permalink
Add unit tests for GoogleCloudStorageClientImpl. (#1095)
Browse files Browse the repository at this point in the history 
* Add unit tests for GoogleCloudStorageClientImpl.

* Add unit tests for compose and copy object methods.

* Add support for customer supplied encryption keys for copy.
  • Loading branch information
@shilpi23pandey
shilpi23pandey committed Jan 4, 2024
1 parent 2f79ae3 commit 5396469
Show file tree
Hide file tree
Showing 5 changed files with 719 additions and 68 deletions.
86 changes: 69 additions & 17 deletions gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageExceptions.createFileNotFoundException;
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.decodeMetadata;
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.encodeMetadata;
import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.validateCopyArguments;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
Expand Down Expand Up @@ -83,6 +82,7 @@
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -219,6 +219,33 @@ public void createBucket(String bucketName, CreateBucketOptions options) throws
}
}

/**
* See {@link GoogleCloudStorage#copy(String, List, String, List)} for details about expected
* behavior.
*/
@Override
public void copy(
String srcBucketName,
List<String> srcObjectNames,
String dstBucketName,
List<String> dstObjectNames)
throws IOException {
checkArgument(srcObjectNames != null, "srcObjectNames must not be null");
checkArgument(dstObjectNames != null, "dstObjectNames must not be null");
checkArgument(
srcObjectNames.size() == dstObjectNames.size(),
"Must supply same number of elements in srcObjects and dstObjects");

Map<StorageResourceId, StorageResourceId> sourceToDestinationObjectsMap =
new HashMap<>(srcObjectNames.size());
for (int i = 0; i < srcObjectNames.size(); i++) {
sourceToDestinationObjectsMap.put(
new StorageResourceId(srcBucketName, srcObjectNames.get(i)),
new StorageResourceId(dstBucketName, dstObjectNames.get(i)));
}
copy(sourceToDestinationObjectsMap);
}

/**
* See {@link GoogleCloudStorage#copy(String, List, String, List)} for details about expected
* behavior.
Expand Down Expand Up @@ -281,6 +308,14 @@ private void copyInternal(
copyRequestBuilder.setTarget(BlobId.of(dstBucketName, dstObjectName));
}

if (storageOptions.getEncryptionKey() != null) {
copyRequestBuilder.setSourceOptions(
BlobSourceOption.decryptionKey(storageOptions.getEncryptionKey().value()));
copyRequestBuilder.setTarget(
copyRequestBuilder.build().getTarget().getBlobId(),
BlobTargetOption.encryptionKey(storageOptions.getEncryptionKey().value()));
}

if (storageOptions.getMaxRewriteChunkSize() > 0) {
copyRequestBuilder.setMegabytesCopiedPerChunk(
// Convert raw byte size into Mib.
Expand Down Expand Up @@ -387,7 +422,7 @@ private static GoogleCloudStorageItemInfo createItemInfoForBucket(
bucket.asBucketInfo().getCreateTimeOffsetDateTime().toInstant().toEpochMilli(),
bucket.asBucketInfo().getUpdateTimeOffsetDateTime().toInstant().toEpochMilli(),
bucket.getLocation(),
bucket.getStorageClass().name());
bucket.getStorageClass() == null ? null : bucket.getStorageClass().name());
}

/** See {@link GoogleCloudStorage#deleteObjects(List)} for details about the expected behavior. */
Expand Down Expand Up @@ -638,26 +673,25 @@ public List<GoogleCloudStorageItemInfo> updateItems(List<UpdatableItemInfo> item
new FutureCallback<>() {
@Override
public void onSuccess(Blob blob) {
logger.atFiner().log(
"updateItems: Successfully updated object '%s' for resourceId '%s'",
blob, resourceId);
resultItemInfos.put(resourceId, createItemInfoForBlob(resourceId, blob));
}

@Override
public void onFailure(Throwable throwable) {
if (throwable instanceof Exception
&& errorExtractor.getErrorType((Exception) throwable) == ErrorType.NOT_FOUND) {
logger.atFiner().log(
"updateItems: object not found %s: %s", resourceId, throwable);
if (blob == null) {
// Indicated that the blob was not found.
logger.atFiner().log("updateItems: object not found %s", resourceId);
resultItemInfos.put(
resourceId, GoogleCloudStorageItemInfo.createNotFound(resourceId));
} else {
innerExceptions.add(
new IOException(
String.format("Error updating '%s' object", resourceId), throwable));
logger.atFiner().log(
"updateItems: Successfully updated object '%s' for resourceId '%s'",
blob, resourceId);
resultItemInfos.put(resourceId, createItemInfoForBlob(resourceId, blob));
}
}

@Override
public void onFailure(Throwable throwable) {
innerExceptions.add(
new IOException(
String.format("Error updating '%s' object", resourceId), throwable));
}
});
}
} finally {
Expand Down Expand Up @@ -694,6 +728,24 @@ private static Map<String, String> encodeMetadata(Map<String, byte[]> metadata)
return Maps.transformValues(metadata, GoogleCloudStorageClientImpl::encodeMetadataValues);
}

@Override
public void compose(
String bucketName, List<String> sources, String destination, String contentType)
throws IOException {
logger.atFiner().log("compose(%s, %s, %s, %s)", bucketName, sources, destination, contentType);
List<StorageResourceId> sourceIds =
sources.stream()
.map(objectName -> new StorageResourceId(bucketName, objectName))
.collect(Collectors.toList());
StorageResourceId destinationId = new StorageResourceId(bucketName, destination);
CreateObjectOptions options =
CreateObjectOptions.DEFAULT_OVERWRITE.toBuilder()
.setContentType(contentType)
.setEnsureEmptyObjectsMetadataMatch(false)
.build();
composeObjects(sourceIds, destinationId, options);
}

/**
* See {@link GoogleCloudStorage#composeObjects(List, StorageResourceId, CreateObjectOptions)}}
* for details about expected behavior.
Expand Down
155 changes: 154 additions & 1 deletion gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/MockStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,20 @@
package com.google.cloud.hadoop.gcsio.testing;

import com.google.protobuf.AbstractMessage;
import com.google.protobuf.Empty;
import com.google.storage.v2.Bucket;
import com.google.storage.v2.ComposeObjectRequest;
import com.google.storage.v2.CreateBucketRequest;
import com.google.storage.v2.DeleteBucketRequest;
import com.google.storage.v2.DeleteObjectRequest;
import com.google.storage.v2.GetObjectRequest;
import com.google.storage.v2.ListBucketsRequest;
import com.google.storage.v2.ListBucketsResponse;
import com.google.storage.v2.Object;
import com.google.storage.v2.RewriteObjectRequest;
import com.google.storage.v2.RewriteResponse;
import com.google.storage.v2.StorageGrpc.StorageImplBase;
import com.google.storage.v2.UpdateObjectRequest;
import io.grpc.stub.StreamObserver;
import java.util.ArrayList;
import java.util.LinkedList;
Expand All @@ -29,7 +40,7 @@
public final class MockStorage extends StorageImplBase {

private List<AbstractMessage> requests;
private Queue<Object> responses;
private Queue<java.lang.Object> responses;

public MockStorage() {
requests = new ArrayList<>();
Expand Down Expand Up @@ -72,4 +83,146 @@ public void createBucket(CreateBucketRequest request, StreamObserver<Bucket> res
Exception.class.getName())));
}
}

@Override
public void composeObject(ComposeObjectRequest request, StreamObserver<Object> responseObserver) {
java.lang.Object response = responses.poll();
if (response instanceof Object) {
requests.add(request);
responseObserver.onNext(((Object) response));
responseObserver.onCompleted();
} else if (response instanceof Exception) {
responseObserver.onError(((Exception) response));
} else {
responseObserver.onError(
new IllegalArgumentException(
String.format(
"Unrecognized response type %s for method ComposeObject, expected %s or %s",
response == null ? "null" : response.getClass().getName(),
Object.class.getName(),
Exception.class.getName())));
}
}

@Override
public void deleteBucket(DeleteBucketRequest request, StreamObserver<Empty> responseObserver) {
java.lang.Object response = responses.poll();
if (response instanceof Empty) {
requests.add(request);
responseObserver.onNext(((Empty) response));
responseObserver.onCompleted();
} else if (response instanceof Exception) {
responseObserver.onError(((Exception) response));
} else {
responseObserver.onError(
new IllegalArgumentException(
String.format(
"Unrecognized response type %s for method DeleteBucket, expected %s or %s",
response == null ? "null" : response.getClass().getName(),
Empty.class.getName(),
Exception.class.getName())));
}
}

@Override
public void listBuckets(
ListBucketsRequest request, StreamObserver<ListBucketsResponse> responseObserver) {
java.lang.Object response = responses.poll();
if (response instanceof ListBucketsResponse) {
requests.add(request);
responseObserver.onNext(((ListBucketsResponse) response));
responseObserver.onCompleted();
} else if (response instanceof Exception) {
responseObserver.onError(((Exception) response));
} else {
responseObserver.onError(
new IllegalArgumentException(
String.format(
"Unrecognized response type %s for method ListBuckets, expected %s or %s",
response == null ? "null" : response.getClass().getName(),
ListBucketsResponse.class.getName(),
Exception.class.getName())));
}
}

@Override
public void deleteObject(DeleteObjectRequest request, StreamObserver<Empty> responseObserver) {
java.lang.Object response = responses.poll();
if (response instanceof Empty) {
requests.add(request);
responseObserver.onNext(((Empty) response));
responseObserver.onCompleted();
} else if (response instanceof Exception) {
responseObserver.onError(((Exception) response));
} else {
responseObserver.onError(
new IllegalArgumentException(
String.format(
"Unrecognized response type %s for method DeleteObject, expected %s or %s",
response == null ? "null" : response.getClass().getName(),
Empty.class.getName(),
Exception.class.getName())));
}
}

@Override
public void getObject(GetObjectRequest request, StreamObserver<Object> responseObserver) {
java.lang.Object response = responses.poll();
if (response instanceof Object) {
requests.add(request);
responseObserver.onNext(((Object) response));
responseObserver.onCompleted();
} else if (response instanceof Exception) {
responseObserver.onError(((Exception) response));
} else {
responseObserver.onError(
new IllegalArgumentException(
String.format(
"Unrecognized response type %s for method GetObject, expected %s or %s",
response == null ? "null" : response.getClass().getName(),
Object.class.getName(),
Exception.class.getName())));
}
}

@Override
public void updateObject(UpdateObjectRequest request, StreamObserver<Object> responseObserver) {
java.lang.Object response = responses.poll();
if (response instanceof Object) {
requests.add(request);
responseObserver.onNext(((Object) response));
responseObserver.onCompleted();
} else if (response instanceof Exception) {
responseObserver.onError(((Exception) response));
} else {
responseObserver.onError(
new IllegalArgumentException(
String.format(
"Unrecognized response type %s for method UpdateObject, expected %s or %s",
response == null ? "null" : response.getClass().getName(),
Object.class.getName(),
Exception.class.getName())));
}
}

@Override
public void rewriteObject(
RewriteObjectRequest request, StreamObserver<RewriteResponse> responseObserver) {
java.lang.Object response = responses.poll();
if (response instanceof RewriteResponse) {
requests.add(request);
responseObserver.onNext(((RewriteResponse) response));
responseObserver.onCompleted();
} else if (response instanceof Exception) {
responseObserver.onError(((Exception) response));
} else {
responseObserver.onError(
new IllegalArgumentException(
String.format(
"Unrecognized response type %s for method RewriteObject, expected %s or %s",
response == null ? "null" : response.getClass().getName(),
RewriteResponse.class.getName(),
Exception.class.getName())));
}
}
}
Loading

0 comments on commit 5396469

Please sign in to comment.