Skip to content

Commit

Permalink
Add support for Start Offset in FakeStorageRpc
Browse files Browse the repository at this point in the history 
Add support for End Offset in FakeStorageRpc

Closes googleapis#1273
  • Loading branch information
@shubhwip
shubhwip authored and Shubham Jain committed Oct 7, 2023
1 parent 3692fd0 commit a156713
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 23 deletions.
38 changes: 26 additions & 12 deletions ...-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing/FakeStorageRpc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,24 +34,14 @@
import com.google.cloud.storage.spi.v1.StorageRpc;
import com.google.cloud.storage.testing.StorageRpcTestBase;
import com.google.common.base.Preconditions;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.io.*;
import java.math.BigInteger;
import java.net.URLDecoder;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -108,6 +98,8 @@ class FakeStorageRpc extends StorageRpcTestBase {

private final boolean throwIfOption;

private static final Comparator<String> lexicographicalStringComparator = String::compareTo;

/** @param throwIfOption if true, we throw when given any option */
public FakeStorageRpc(boolean throwIfOption) {
this.throwIfOption = throwIfOption;
Expand Down Expand Up @@ -142,6 +134,8 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
String preprefix = "";
String pageToken = null;
long maxResults = Long.MAX_VALUE;
String startOffset = null;
String endOffset = null;
for (Map.Entry<Option, ?> e : options.entrySet()) {
switch (e.getKey()) {
case PAGE_TOKEN:
Expand All @@ -162,6 +156,18 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
case MAX_RESULTS:
maxResults = (Long) e.getValue();
break;
case START_OFF_SET:
startOffset = (String) e.getValue();
if (startOffset.startsWith("/")) {
startOffset = startOffset.substring(1);
}
break;
case END_OFF_SET:
endOffset = (String) e.getValue();
if (endOffset.startsWith("/")) {
endOffset = endOffset.substring(1);
}
break;
case USER_PROJECT:
// prevent unsupported operation
break;
Expand All @@ -177,6 +183,14 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
if (!so.getBucket().equals(bucket) || !so.getName().startsWith(prefix)) {
continue;
}
if (startOffset != null
&& lexicographicalStringComparator.compare(so.getName(), startOffset) < 0) {
continue;
}
if (endOffset != null
&& lexicographicalStringComparator.compare(so.getName(), endOffset) >= 0) {
continue;
}
if (processedAsFolder(so, delimiter, prefix, folders)) {
continue;
}
Expand Down
122 changes: 111 additions & 11 deletions ...io/src/test/java/com/google/cloud/storage/contrib/nio/testing/LocalStorageHelperTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,17 @@
package com.google.cloud.storage.contrib.nio.testing;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;

import com.google.api.client.util.Lists;
import com.google.api.gax.paging.Page;
import com.google.cloud.WriteChannel;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import com.google.cloud.storage.*;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.List;
import java.util.stream.IntStream;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
Expand Down Expand Up @@ -180,4 +176,108 @@ public void testCopyOperationOverwritesExistingFile() {
assertThat(localStorageService.readAllBytes(BlobId.of(bucket, original)))
.isEqualTo(replacementContent);
}

@Test
public void testListObjectsWithStartOffsetOnly() {
// Arrange
String bucket = "bucket";
byte[] originalContent = "original content".getBytes();
IntStream.range(1, 10)
.forEach(
i ->
localStorageService.create(
BlobInfo.newBuilder(bucket, String.valueOf(i)).build(), originalContent));
// Act
Page<Blob> pages = localStorageService.list(bucket, Storage.BlobListOption.startOffset("5"));
List<Blob> blobs = Lists.newArrayList(pages.iterateAll());
// Assert
assertEquals(6, blobs.size());
}

@Test
public void testListObjectsWithEndOffsetOnly() {
// Arrange
String bucket = "bucket";
byte[] originalContent = "original content".getBytes();
IntStream.range(1, 10)
.forEach(
i ->
localStorageService.create(
BlobInfo.newBuilder(bucket, String.valueOf(i)).build(), originalContent));
// Act
Page<Blob> pages = localStorageService.list(bucket, Storage.BlobListOption.endOffset("5"));
List<Blob> blobs = Lists.newArrayList(pages.iterateAll());
// Assert
assertEquals(4, blobs.size());
}

@Test
public void testListObjectsWithStartOffsetAndEndOffset() {
// Arrange
String bucket = "bucket";
byte[] originalContent = "original content".getBytes();
IntStream.range(1, 10)
.forEach(
i ->
localStorageService.create(
BlobInfo.newBuilder(bucket, String.valueOf(i)).build(), originalContent));
// Act
Page<Blob> pages =
localStorageService.list(
bucket, Storage.BlobListOption.startOffset("1"), Storage.BlobListOption.endOffset("5"));
List<Blob> blobs = Lists.newArrayList(pages.iterateAll());
// Assert
System.out.println(blobs);
assertEquals(4, blobs.size());
}

@Test
public void testListObjectsWithPrefixAndStartOffsetAndEndOffset() {
// Arrange
String bucket = "bucket";
byte[] originalContent = "original content".getBytes();
IntStream.range(1, 10)
.forEach(
i -> {
localStorageService.create(
BlobInfo.newBuilder(bucket, "abc/" + i).build(), originalContent);
localStorageService.create(
BlobInfo.newBuilder(bucket, "bcd/" + i).build(), originalContent);
});
// Act
Page<Blob> pages =
localStorageService.list(
bucket,
Storage.BlobListOption.startOffset("abc/1"),
Storage.BlobListOption.endOffset("abc/5"),
Storage.BlobListOption.prefix("abc/"));
List<Blob> blobs = Lists.newArrayList(pages.iterateAll());
// Assert
assertEquals(4, blobs.size());
}

@Test
public void testListObjectsWithPrefixAndStartOffsetAndEndOffsetWithFileSeparatorInBeginning() {
// Arrange
String bucket = "bucket";
byte[] originalContent = "original content".getBytes();
IntStream.range(1, 10)
.forEach(
i -> {
localStorageService.create(
BlobInfo.newBuilder(bucket, "abc/" + i).build(), originalContent);
localStorageService.create(
BlobInfo.newBuilder(bucket, "bcd/" + i).build(), originalContent);
});
// Act
Page<Blob> pages =
localStorageService.list(
bucket,
Storage.BlobListOption.startOffset("/abc/1"),
Storage.BlobListOption.endOffset("/abc/5"),
Storage.BlobListOption.prefix("/abc/"));
List<Blob> blobs = Lists.newArrayList(pages.iterateAll());
// Assert
assertEquals(4, blobs.size());
}
}

0 comments on commit a156713

Please # to comment.