Skip to content

Commit bcf8244

Browse files
author
Thomas Darimont
committed
DATAMONGO-1165 - Add support for Streaming large result lists.
WIP
1 parent a7533ea commit bcf8244

File tree

12 files changed

+176
-207
lines changed

12 files changed

+176
-207
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
<properties>
3030
<project.type>multi</project.type>
3131
<dist.id>spring-data-mongodb</dist.id>
32-
<springdata.commons>1.10.0.BUILD-SNAPSHOT</springdata.commons>
32+
<springdata.commons>1.10.0.DATACMNS-650-SNAPSHOT</springdata.commons>
3333
<mongo>2.13.0</mongo>
3434
<mongo.osgi>2.13.0</mongo.osgi>
3535
</properties>

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
2525
import org.springframework.data.mongodb.core.aggregation.TypedAggregation;
2626
import org.springframework.data.mongodb.core.convert.MongoConverter;
27+
import org.springframework.data.mongodb.core.convert.QueryMapper;
2728
import org.springframework.data.mongodb.core.mapreduce.GroupBy;
2829
import org.springframework.data.mongodb.core.mapreduce.GroupByResults;
2930
import org.springframework.data.mongodb.core.mapreduce.MapReduceOptions;
@@ -33,7 +34,6 @@
3334
import org.springframework.data.mongodb.core.query.NearQuery;
3435
import org.springframework.data.mongodb.core.query.Query;
3536
import org.springframework.data.mongodb.core.query.Update;
36-
import org.springframework.data.mongodb.util.CloseableIterator;
3737

3838
import com.mongodb.CommandResult;
3939
import com.mongodb.DBCollection;
@@ -961,5 +961,10 @@ <T> T findAndModify(Query query, Update update, FindAndModifyOptions options, Cl
961961
*/
962962
MongoConverter getConverter();
963963

964-
CloseableIterator<Object> getStreamingMappingCursor(Query query, Class<?> entityType);
964+
/**
965+
* Returns the underlying {@link QueryMapper}.
966+
*
967+
* @return
968+
*/
969+
QueryMapper getQueryMapper();
965970
}

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java

Lines changed: 8 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,6 @@
9595
import org.springframework.data.mongodb.core.query.NearQuery;
9696
import org.springframework.data.mongodb.core.query.Query;
9797
import org.springframework.data.mongodb.core.query.Update;
98-
import org.springframework.data.mongodb.util.CloseableIterator;
9998
import org.springframework.jca.cci.core.ConnectionCallback;
10099
import org.springframework.util.Assert;
101100
import org.springframework.util.CollectionUtils;
@@ -104,7 +103,6 @@
104103

105104
import com.mongodb.BasicDBObject;
106105
import com.mongodb.CommandResult;
107-
import com.mongodb.Cursor;
108106
import com.mongodb.DB;
109107
import com.mongodb.DBCollection;
110108
import com.mongodb.DBCursor;
@@ -314,6 +312,14 @@ public MongoConverter getConverter() {
314312
return this.mongoConverter;
315313
}
316314

315+
/* (non-Javadoc)
316+
* @see org.springframework.data.mongodb.core.MongoOperations#getQueryMapper()
317+
*/
318+
@Override
319+
public QueryMapper getQueryMapper() {
320+
return queryMapper;
321+
}
322+
317323
public String getCollectionName(Class<?> entityClass) {
318324
return this.determineCollectionName(entityClass);
319325
}
@@ -2263,82 +2269,4 @@ public GeoResult<T> doWith(DBObject object) {
22632269
return new GeoResult<T>(doWith, new Distance(distance, metric));
22642270
}
22652271
}
2266-
2267-
@Override
2268-
public CloseableIterator<Object> getStreamingMappingCursor(Query query, Class<?> entityClass) {
2269-
2270-
MongoPersistentEntity<?> entity = mappingContext.getPersistentEntity(entityClass);
2271-
2272-
DBCollection collection = getCollection(getCollectionName(entityClass));
2273-
2274-
DBObject mappedFields = queryMapper.getMappedFields(query.getFieldsObject(), entity);
2275-
DBObject mappedQuery = queryMapper.getMappedObject(query.getQueryObject(), entity);
2276-
2277-
return new CloseableIterableCusorAdapter(collection.find(mappedQuery, mappedFields), mongoConverter, entityClass);
2278-
}
2279-
2280-
static class CloseableIterableCusorAdapter implements CloseableIterator<Object> {
2281-
2282-
private volatile Cursor cursor;
2283-
private MongoConverter converter;
2284-
private Class<?> entityClass;
2285-
2286-
public CloseableIterableCusorAdapter(Cursor cursor, MongoConverter converter, Class<?> entityClass) {
2287-
2288-
this.cursor = cursor;
2289-
this.converter = converter;
2290-
this.entityClass = entityClass;
2291-
}
2292-
2293-
@Override
2294-
public boolean hasNext() {
2295-
2296-
if (cursor == null) {
2297-
return false;
2298-
}
2299-
2300-
try {
2301-
return cursor.hasNext();
2302-
} catch (Exception ex) {
2303-
throw new RuntimeException("error on hasNext");
2304-
}
2305-
}
2306-
2307-
@Override
2308-
public Object next() {
2309-
2310-
if (cursor == null) {
2311-
return null;
2312-
}
2313-
2314-
try {
2315-
2316-
DBObject item = cursor.next();
2317-
Object converted = converter.read(entityClass, item);
2318-
2319-
return converted;
2320-
} catch (Exception ex) {
2321-
throw new RuntimeException("error on next");
2322-
}
2323-
}
2324-
2325-
@Override
2326-
public void close() throws IOException {
2327-
2328-
Cursor c = cursor;
2329-
try {
2330-
c.close();
2331-
} finally {
2332-
2333-
cursor = null;
2334-
converter = null;
2335-
entityClass = null;
2336-
}
2337-
}
2338-
2339-
@Override
2340-
public Iterator<Object> iterator() {
2341-
return this;
2342-
}
2343-
}
23442272
}

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/convert/QueryMapper.java

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2011-2014 the original author or authors.
2+
* Copyright 2011-2015 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -81,6 +81,23 @@ public QueryMapper(MongoConverter converter) {
8181
this.mappingContext = converter.getMappingContext();
8282
}
8383

84+
/**
85+
* Replaces the property keys used in the given {@link DBObject} with the appropriate keys by using the
86+
* {@link PersistentEntity} metadata linked with the given {@code entityClass}.
87+
*
88+
* @see #getMappedObject(DBObject, MongoPersistentEntity)
89+
* @param query must not be {@literal null}.
90+
* @param entityClass must not be {@literal null}.
91+
* @return
92+
* @since 1.7
93+
*/
94+
public DBObject getMappedObject(DBObject query, Class<?> entityClass) {
95+
96+
Assert.notNull(entityClass, "Entity class must not be null!");
97+
98+
return getMappedObject(query, mappingContext.getPersistentEntity(entityClass));
99+
}
100+
84101
/**
85102
* Replaces the property keys used in the given {@link DBObject} with the appropriate keys by using the
86103
* {@link PersistentEntity} metadata.
@@ -144,9 +161,26 @@ public DBObject getMappedSort(DBObject sortObject, MongoPersistentEntity<?> enti
144161
return mappedSort;
145162
}
146163

164+
/**
165+
* Maps fields to retrieve to the {@link MongoPersistentEntity}s properties linked to the given {@code entityClass}.
166+
* <p>
167+
*
168+
* @see QueryMapper#getMappedFields(DBObject, MongoPersistentEntity)
169+
* @param fieldsObject
170+
* @param entityClass must not be {@literal null}
171+
* @return
172+
* @since 1.7
173+
*/
174+
public DBObject getMappedFields(DBObject fieldsObject, Class<?> entityClass) {
175+
176+
Assert.notNull(entityClass, "Entity class must not be null!");
177+
178+
return getMappedFields(fieldsObject, mappingContext.getPersistentEntity(entityClass));
179+
}
180+
147181
/**
148182
* Maps fields to retrieve to the {@link MongoPersistentEntity}s properties. <br />
149-
* Also onverts and potentially adds missing property {@code $meta} representation.
183+
* Also converts and potentially adds missing property {@code $meta} representation.
150184
*
151185
* @param fieldsObject
152186
* @param entity
@@ -283,7 +317,7 @@ protected Object getMappedValue(Field documentField, Object value) {
283317
} else if (valueDbo.containsField("$ne")) {
284318
resultDbo.put("$ne", convertId(valueDbo.get("$ne")));
285319
} else {
286-
return getMappedObject(resultDbo, null);
320+
return getMappedObject(resultDbo, (MongoPersistentEntity<?>) null);
287321
}
288322

289323
return resultDbo;

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/gridfs/GridFsTemplate.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.springframework.data.mongodb.MongoDbFactory;
2727
import org.springframework.data.mongodb.core.convert.MongoConverter;
2828
import org.springframework.data.mongodb.core.convert.QueryMapper;
29+
import org.springframework.data.mongodb.core.mapping.MongoPersistentEntity;
2930
import org.springframework.data.mongodb.core.query.Query;
3031
import org.springframework.util.Assert;
3132
import org.springframework.util.StringUtils;
@@ -257,7 +258,7 @@ private DBObject getMappedQuery(Query query) {
257258
}
258259

259260
private DBObject getMappedQuery(DBObject query) {
260-
return query == null ? null : queryMapper.getMappedObject(query, null);
261+
return query == null ? null : queryMapper.getMappedObject(query, (MongoPersistentEntity<?>) null);
261262
}
262263

263264
private GridFS getGridFs() {

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import java.util.Collections;
1919
import java.util.List;
2020

21+
import org.springframework.dao.DataAccessException;
2122
import org.springframework.data.domain.PageImpl;
2223
import org.springframework.data.domain.Pageable;
2324
import org.springframework.data.domain.Slice;
@@ -27,16 +28,21 @@
2728
import org.springframework.data.geo.GeoResult;
2829
import org.springframework.data.geo.GeoResults;
2930
import org.springframework.data.geo.Point;
31+
import org.springframework.data.mongodb.core.CollectionCallback;
3032
import org.springframework.data.mongodb.core.MongoOperations;
33+
import org.springframework.data.mongodb.core.convert.QueryMapper;
3134
import org.springframework.data.mongodb.core.query.NearQuery;
3235
import org.springframework.data.mongodb.core.query.Query;
33-
import org.springframework.data.mongodb.util.CloseableIterator;
36+
import org.springframework.data.mongodb.util.CloseableIterableCusorAdapter;
3437
import org.springframework.data.repository.query.ParameterAccessor;
3538
import org.springframework.data.repository.query.RepositoryQuery;
39+
import org.springframework.data.util.CloseableIterator;
3640
import org.springframework.data.util.TypeInformation;
3741
import org.springframework.util.Assert;
38-
import org.springframework.util.ClassUtils;
3942

43+
import com.mongodb.DBCollection;
44+
import com.mongodb.DBObject;
45+
import com.mongodb.MongoException;
4046
import com.mongodb.WriteResult;
4147

4248
/**
@@ -420,19 +426,28 @@ private Object deleteAndConvertResult(Query query, MongoEntityMetadata<?> metada
420426

421427
final class StreamExecution extends Execution {
422428

423-
private final boolean IS_JAVA_7 = ClassUtils.isPresent("java.lang.AutoCloseable",
424-
StreamExecution.class.getClassLoader());
425-
426429
@Override
427-
Object execute(Query query) {
430+
Object execute(final Query query) {
428431

429-
Class<?> entityType = getQueryMethod().getEntityInformation().getJavaType();
430-
CloseableIterator<Object> result = operations.getStreamingMappingCursor(query, entityType);
431-
return IS_JAVA_7 ? wrapInAutoCloseableIterator(result) : result;
432-
}
432+
final Class<?> entityType = getQueryMethod().getEntityInformation().getJavaType();
433+
final QueryMapper queryMapper = operations.getQueryMapper();
434+
435+
@SuppressWarnings("unchecked")
436+
CloseableIterator<Object> result = (CloseableIterator<Object>) operations.execute(entityType,
437+
new CollectionCallback<Object>() {
438+
439+
@Override
440+
public Object doInCollection(DBCollection collection) throws MongoException, DataAccessException {
441+
442+
DBObject mappedFields = queryMapper.getMappedFields(query.getFieldsObject(), entityType);
443+
DBObject mappedQuery = queryMapper.getMappedObject(query.getQueryObject(), entityType);
444+
445+
return new CloseableIterableCusorAdapter<Object>(collection.find(mappedQuery, mappedFields), operations
446+
.getConverter(), entityType);
447+
}
448+
});
433449

434-
private CloseableIterator<Object> wrapInAutoCloseableIterator(CloseableIterator<Object> iter) {
435-
return new ForwardingAutoCloseableIterator<Object>(iter);
450+
return result;
436451
}
437452
}
438453
}

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ForwardingAutoCloseableIterator.java

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,10 @@
2929
import org.springframework.data.mongodb.core.mapping.MongoPersistentProperty;
3030
import org.springframework.data.mongodb.repository.Meta;
3131
import org.springframework.data.mongodb.repository.Query;
32-
import org.springframework.data.mongodb.util.CloseableIterator;
3332
import org.springframework.data.repository.core.RepositoryMetadata;
3433
import org.springframework.data.repository.query.QueryMethod;
3534
import org.springframework.data.util.ClassTypeInformation;
35+
import org.springframework.data.util.CloseableIterator;
3636
import org.springframework.data.util.TypeInformation;
3737
import org.springframework.util.Assert;
3838
import org.springframework.util.StringUtils;

‎spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbSerializer.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ protected String getKeyForPath(Path<?> expr, PathMetadata<?> metadata) {
100100
protected DBObject asDBObject(String key, Object value) {
101101

102102
if (ID_KEY.equals(key)) {
103-
return mapper.getMappedObject(super.asDBObject(key, value), null);
103+
return mapper.getMappedObject(super.asDBObject(key, value), (MongoPersistentEntity<?>) null);
104104
}
105105

106106
return super.asDBObject(key, value instanceof Pattern ? value : converter.convertToMongoType(value));

0 commit comments

Comments
 (0)