Skip to content

Commit 59060da

Browse files
justinsbjustinsb
committed
Add support for paged file lists (>10k files)
1 parent 6d1dc4e commit 59060da

1 file changed

Lines changed: 99 additions & 26 deletions

File tree

‎openstack-api/src/main/java/org/openstack/client/storage/ObjectsResource.java‎

Lines changed: 99 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
import java.io.FileInputStream;
55
import java.io.IOException;
66
import java.io.InputStream;
7-
import java.util.List;
7+
import java.util.ArrayDeque;
8+
import java.util.Iterator;
9+
import java.util.Queue;
810

911
import javax.ws.rs.core.MediaType;
1012
import javax.ws.rs.core.MultivaluedMap;
@@ -19,7 +21,6 @@
1921

2022
import com.google.common.base.Splitter;
2123
import com.google.common.base.Strings;
22-
import com.google.common.collect.Lists;
2324
import com.sun.jersey.api.client.ClientResponse;
2425

2526
public class ObjectsResource extends StorageResourceBase {
@@ -31,40 +32,112 @@ public Iterable<StorageObject> list(String prefix, String delimiter) {
3132
return list(prefix, delimiter, true);
3233
}
3334

34-
public Iterable<StorageObject> list(String prefix, String delimiter, boolean fetchMetadata) {
35-
RequestBuilder requestBuilder = resource();
35+
public Iterable<StorageObject> list(final String prefix, final String delimiter, final boolean fetchMetadata) {
36+
return new Iterable<StorageObject>() {
37+
@Override
38+
public Iterator<StorageObject> iterator() {
39+
return new FileIterator(prefix, delimiter, fetchMetadata);
40+
}
41+
};
42+
}
43+
44+
class FileIterator implements Iterator<StorageObject> {
45+
final String prefix;
46+
final String delimiter;
47+
final boolean fetchMetadata;
48+
49+
Queue<StorageObject> queue = null;
3650

37-
if (prefix != null) {
38-
requestBuilder.addQueryParameter("prefix", prefix);
51+
String marker;
52+
boolean couldHaveMore = true;
53+
54+
public FileIterator(String prefix, String delimiter, boolean fetchMetadata) {
55+
super();
56+
this.prefix = prefix;
57+
this.delimiter = delimiter;
58+
this.fetchMetadata = fetchMetadata;
3959
}
40-
if (!Strings.isNullOrEmpty(delimiter)) {
41-
requestBuilder.addQueryParameter("delimiter", delimiter);
60+
61+
@Override
62+
public boolean hasNext() {
63+
if (queue == null || queue.isEmpty()) {
64+
if (couldHaveMore) {
65+
queue = nextPage(marker);
66+
}
67+
if (queue == null || queue.isEmpty()) {
68+
queue = null;
69+
}
70+
}
71+
72+
return (queue != null && !queue.isEmpty());
4273
}
4374

44-
if (fetchMetadata) {
45-
requestBuilder.clearAcceptTypes();
46-
requestBuilder.addAcceptType(MediaType.APPLICATION_JSON_TYPE);
75+
@Override
76+
public StorageObject next() {
77+
if (queue == null) {
78+
throw new IllegalStateException();
79+
}
4780

48-
StorageObjectList storageObjectList = requestBuilder.get(StorageObjectList.class);
81+
StorageObject so = queue.remove();
82+
marker = so.getName();
83+
return so;
84+
}
4985

50-
return storageObjectList.getObjects();
51-
} else {
52-
requestBuilder.clearAcceptTypes();
53-
requestBuilder.addAcceptType(MediaType.TEXT_PLAIN_TYPE);
54-
55-
String listing = requestBuilder.get(String.class);
56-
List<StorageObject> list = Lists.newArrayList();
57-
for (String line : Splitter.on("\n").split(listing)) {
58-
if (line.isEmpty()) {
59-
continue;
86+
@Override
87+
public void remove() {
88+
throw new UnsupportedOperationException();
89+
}
90+
91+
Queue<StorageObject> nextPage(String marker) {
92+
RequestBuilder requestBuilder = resource();
93+
94+
if (prefix != null) {
95+
requestBuilder.addQueryParameter("prefix", prefix);
96+
}
97+
if (!Strings.isNullOrEmpty(delimiter)) {
98+
requestBuilder.addQueryParameter("delimiter", delimiter);
99+
}
100+
101+
if (!Strings.isNullOrEmpty(marker)) {
102+
requestBuilder.addQueryParameter("marker", marker);
103+
}
104+
105+
int limit = 10000;
106+
107+
if (fetchMetadata) {
108+
requestBuilder.clearAcceptTypes();
109+
requestBuilder.addAcceptType(MediaType.APPLICATION_JSON_TYPE);
110+
111+
StorageObjectList storageObjectList = requestBuilder.get(StorageObjectList.class);
112+
113+
ArrayDeque<StorageObject> list = new ArrayDeque<StorageObject>();
114+
list.addAll(storageObjectList.getObjects());
115+
if (list.size() != limit) {
116+
couldHaveMore = false;
117+
}
118+
return list;
119+
} else {
120+
requestBuilder.clearAcceptTypes();
121+
requestBuilder.addAcceptType(MediaType.TEXT_PLAIN_TYPE);
122+
123+
String listing = requestBuilder.get(String.class);
124+
ArrayDeque<StorageObject> list = new ArrayDeque<StorageObject>();
125+
for (String line : Splitter.on("\n").split(listing)) {
126+
if (line.isEmpty()) {
127+
continue;
128+
}
129+
130+
StorageObject storageObject = new StorageObject();
131+
storageObject.setName(line);
132+
list.add(storageObject);
60133
}
61134

62-
StorageObject storageObject = new StorageObject();
63-
storageObject.setName(line);
64-
list.add(storageObject);
135+
if (list.size() != limit) {
136+
couldHaveMore = false;
137+
}
138+
return list;
65139
}
66140

67-
return list;
68141
}
69142
}
70143

0 commit comments

Comments
 (0)