From e9eabf9028971b50f6c5f7355fd828d9a0143296 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Tue, 28 Feb 2023 09:05:47 +0100 Subject: [PATCH 1/7] Prepare issue branch. --- pom.xml | 4 ++-- spring-data-mongodb-benchmarks/pom.xml | 2 +- spring-data-mongodb-distribution/pom.xml | 2 +- spring-data-mongodb/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index d864b2e4e6..e899b2ae73 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.data spring-data-mongodb-parent - 4.1.0-SNAPSHOT + 4.1.0-GH-4308-SNAPSHOT pom Spring Data MongoDB @@ -26,7 +26,7 @@ multi spring-data-mongodb - 3.1.0-SNAPSHOT + 3.1.0-GH-2151-SNAPSHOT 4.9.0 ${mongo} 1.19 diff --git a/spring-data-mongodb-benchmarks/pom.xml b/spring-data-mongodb-benchmarks/pom.xml index 1b2a1390e6..322d0b60ac 100644 --- a/spring-data-mongodb-benchmarks/pom.xml +++ b/spring-data-mongodb-benchmarks/pom.xml @@ -7,7 +7,7 @@ org.springframework.data spring-data-mongodb-parent - 4.1.0-SNAPSHOT + 4.1.0-GH-4308-SNAPSHOT ../pom.xml diff --git a/spring-data-mongodb-distribution/pom.xml b/spring-data-mongodb-distribution/pom.xml index 8db8d798fb..4fbddd7958 100644 --- a/spring-data-mongodb-distribution/pom.xml +++ b/spring-data-mongodb-distribution/pom.xml @@ -15,7 +15,7 @@ org.springframework.data spring-data-mongodb-parent - 4.1.0-SNAPSHOT + 4.1.0-GH-4308-SNAPSHOT ../pom.xml diff --git a/spring-data-mongodb/pom.xml b/spring-data-mongodb/pom.xml index 597ca94f38..698e179d07 100644 --- a/spring-data-mongodb/pom.xml +++ b/spring-data-mongodb/pom.xml @@ -13,7 +13,7 @@ org.springframework.data spring-data-mongodb-parent - 4.1.0-SNAPSHOT + 4.1.0-GH-4308-SNAPSHOT ../pom.xml From dbf2a16e833852c734236148d603c81429d40eec Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Tue, 28 Feb 2023 10:58:47 +0100 Subject: [PATCH 2/7] Add support for scrolling using offset- and keyset-based strategies. We now support scrolling through large query results using ScrollPosition and Window's of data. --- .../data/mongodb/core/EntityOperations.java | 36 ++++- .../mongodb/core/ExecutableFindOperation.java | 26 +++- .../core/ExecutableFindOperationSupport.java | 14 +- .../data/mongodb/core/MongoOperations.java | 49 +++++- .../data/mongodb/core/MongoTemplate.java | 92 +++++++++-- .../data/mongodb/core/QueryOperations.java | 3 +- .../mongodb/core/ReactiveFindOperation.java | 22 ++- .../core/ReactiveFindOperationSupport.java | 7 + .../mongodb/core/ReactiveMongoOperations.java | 48 +++++- .../mongodb/core/ReactiveMongoTemplate.java | 90 +++++++++-- .../data/mongodb/core/ScrollUtils.java | 145 +++++++++++++++++ .../data/mongodb/core/query/Query.java | 78 ++++++++++ .../repository/query/AbstractMongoQuery.java | 3 + .../query/AbstractReactiveMongoQuery.java | 3 + .../query/ConvertingParameterAccessor.java | 8 +- .../repository/query/MongoQueryMethod.java | 7 +- .../query/ReactiveMongoQueryMethod.java | 18 ++- .../support/FetchableFluentQuerySupport.java | 34 ++-- .../QuerydslMongoPredicateExecutor.java | 19 ++- .../support/ReactiveFluentQuerySupport.java | 24 ++- ...eactiveQuerydslMongoPredicateExecutor.java | 20 ++- .../ReactiveSpringDataMongodbQuery.java | 11 +- .../support/SimpleMongoRepository.java | 19 ++- .../SimpleReactiveMongoRepository.java | 22 ++- .../support/SpringDataMongodbQuery.java | 17 +- .../data/mongodb/util/BsonUtils.java | 29 ++-- .../core/MongoTemplateScrollTests.java | 147 ++++++++++++++++++ .../data/mongodb/core/MongoTemplateTests.java | 1 - .../ReactiveMongoTemplateScrollTests.java | 144 +++++++++++++++++ ...tractPersonRepositoryIntegrationTests.java | 75 ++++++++- .../mongodb/repository/PersonRepository.java | 26 +++- .../mongodb/repository/PersonSummaryDto.java | 9 ++ .../ReactiveMongoRepositoryTests.java | 63 +++++++- .../query/StubParameterAccessor.java | 6 + src/main/asciidoc/index.adoc | 2 + src/main/asciidoc/reference/mongodb.adoc | 1 + 36 files changed, 1196 insertions(+), 122 deletions(-) create mode 100644 spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java create mode 100644 spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java create mode 100644 spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java index db07166b5a..16d084a677 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -44,6 +45,7 @@ import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.timeseries.Granularity; import org.springframework.data.mongodb.core.validation.Validator; +import org.springframework.data.mongodb.util.BsonUtils; import org.springframework.data.projection.EntityProjection; import org.springframework.data.projection.EntityProjectionIntrospector; import org.springframework.data.projection.ProjectionFactory; @@ -454,6 +456,9 @@ default boolean isVersionedEntity() { * @since 2.1.2 */ boolean isNew(); + + Map extractKeys(Document sortObject); + } /** @@ -475,7 +480,7 @@ interface AdaptibleEntity extends Entity { T populateIdIfNecessary(@Nullable Object id); /** - * Initializes the version property of the of the current entity if available. + * Initializes the version property of the current entity if available. * * @return the entity with the version property updated if available. */ @@ -567,6 +572,19 @@ public T getBean() { public boolean isNew() { return map.get(ID_FIELD) != null; } + + @Override + public Map extractKeys(Document sortObject) { + + Map keyset = new LinkedHashMap<>(); + keyset.put(ID_FIELD, getId()); + + for (String key : sortObject.keySet()) { + keyset.put(key, BsonUtils.resolveValue(map, key)); + } + + return keyset; + } } private static class SimpleMappedEntity> extends UnmappedEntity { @@ -701,6 +719,22 @@ public T getBean() { public boolean isNew() { return entity.isNew(propertyAccessor.getBean()); } + + @Override + public Map extractKeys(Document sortObject) { + + Map keyset = new LinkedHashMap<>(); + keyset.put(entity.getRequiredIdProperty().getName(), getId()); + + for (String key : sortObject.keySet()) { + + // TODO: make this work for nested properties + MongoPersistentProperty persistentProperty = entity.getRequiredPersistentProperty(key); + keyset.put(key, propertyAccessor.getProperty(persistentProperty)); + } + + return keyset; + } } private static class AdaptibleMappedEntity extends MappedEntity implements AdaptibleEntity { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java index 0b7616fa30..e8b38c8e81 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java @@ -20,6 +20,8 @@ import java.util.stream.Stream; import org.springframework.dao.DataAccessException; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.geo.GeoResults; import org.springframework.data.mongodb.core.query.CriteriaDefinition; import org.springframework.data.mongodb.core.query.NearQuery; @@ -124,12 +126,24 @@ default Optional first() { Stream stream(); /** - * Get the number of matching elements. - *
- * This method uses an {@link com.mongodb.client.MongoCollection#countDocuments(org.bson.conversions.Bson, com.mongodb.client.model.CountOptions) aggregation - * execution} even for empty {@link Query queries} which may have an impact on performance, but guarantees shard, - * session and transaction compliance. In case an inaccurate count satisfies the applications needs use - * {@link MongoOperations#estimatedCount(String)} for empty queries instead. + * Return a scroll of elements either starting or resuming at + * {@link org.springframework.data.domain.ScrollPosition}. + * + * @param scrollPosition the scroll position. + * @return a scroll of the resulting elements. + * @since 4.1 + * @see org.springframework.data.domain.OffsetScrollPosition + * @see org.springframework.data.domain.KeysetScrollPosition + */ + Scroll scroll(ScrollPosition scrollPosition); + + /** + * Get the number of matching elements.
+ * This method uses an + * {@link com.mongodb.client.MongoCollection#countDocuments(org.bson.conversions.Bson, com.mongodb.client.model.CountOptions) + * aggregation execution} even for empty {@link Query queries} which may have an impact on performance, but + * guarantees shard, session and transaction compliance. In case an inaccurate count satisfies the applications + * needs use {@link MongoOperations#estimatedCount(String)} for empty queries instead. * * @return total number of matching elements. */ diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java index 1038ee2327..81d7557e78 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java @@ -21,6 +21,8 @@ import org.bson.Document; import org.springframework.dao.IncorrectResultSizeDataAccessException; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.query.NearQuery; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.SerializationUtils; @@ -71,8 +73,8 @@ static class ExecutableFindSupport private final @Nullable String collection; private final Query query; - ExecutableFindSupport(MongoTemplate template, Class domainType, Class returnType, - @Nullable String collection, Query query) { + ExecutableFindSupport(MongoTemplate template, Class domainType, Class returnType, @Nullable String collection, + Query query) { this.template = template; this.domainType = domainType; this.returnType = returnType; @@ -138,6 +140,11 @@ public Stream stream() { return doStream(); } + @Override + public Scroll scroll(ScrollPosition scrollPosition) { + return template.doScroll(query.with(scrollPosition), domainType, returnType, getCollectionName()); + } + @Override public TerminatingFindNear near(NearQuery nearQuery) { return () -> template.geoNear(nearQuery, domainType, getCollectionName(), returnType); @@ -168,8 +175,7 @@ private List doFind(@Nullable CursorPreparer preparer) { Document fieldsObject = query.getFieldsObject(); return template.doFind(template.createDelegate(query), getCollectionName(), queryObject, fieldsObject, domainType, - returnType, - getCursorPreparer(query, preparer)); + returnType, getCursorPreparer(query, preparer)); } private List doFindDistinct(String field) { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java index 4727c0b8db..cd019aae6d 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java @@ -23,6 +23,8 @@ import java.util.stream.Stream; import org.bson.Document; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.GeoResults; import org.springframework.data.mongodb.core.BulkOperations.BulkMode; import org.springframework.data.mongodb.core.aggregation.Aggregation; @@ -319,7 +321,8 @@ default MongoCollection createView(String name, Class source, Aggre * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - MongoCollection createView(String name, Class source, AggregationPipeline pipeline, @Nullable ViewOptions options); + MongoCollection createView(String name, Class source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * Create a view with the provided name. The view content is defined by the {@link AggregationPipeline pipeline} on @@ -331,7 +334,8 @@ default MongoCollection createView(String name, Class source, Aggre * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - MongoCollection createView(String name, String source, AggregationPipeline pipeline, @Nullable ViewOptions options); + MongoCollection createView(String name, String source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * A set of collection names. @@ -802,6 +806,45 @@ MapReduceResults mapReduce(Query query, String inputCollectionName, Strin */ List find(Query query, Class entityClass, String collectionName); + /** + * Query for a scroll window of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @return the converted scroll. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Scroll scroll(Query query, Class entityType); + + /** + * Query for a scroll of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @param collectionName name of the collection to retrieve the objects from. + * @return the converted scroll. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Scroll scroll(Query query, Class entityType, String collectionName); + /** * Returns a document with the given id mapped onto the given class. The collection the query is ran against will be * derived from the given target class as well. @@ -1175,7 +1218,7 @@ T findAndReplace(Query query, S replacement, FindAndReplaceOptions option * @param entityClass class that determines the collection to use. Must not be {@literal null}. * @return the count of matching documents. * @throws org.springframework.data.mapping.MappingException if the collection name cannot be - * {@link #getCollectionName(Class) derived} from the given type. + * {@link #getCollectionName(Class) derived} from the given type. * @see #exactCount(Query, Class) * @see #estimatedCount(Class) */ diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java index c23548ea3a..f3f951da97 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java @@ -44,6 +44,8 @@ import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.dao.support.PersistenceExceptionTranslator; import org.springframework.data.convert.EntityReader; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.Distance; import org.springframework.data.geo.GeoResult; import org.springframework.data.geo.GeoResults; @@ -64,6 +66,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; +import org.springframework.data.mongodb.core.ScrollUtils.KeySetCursorQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -847,6 +850,48 @@ public List find(Query query, Class entityClass, String collectionName new QueryCursorPreparer(query, entityClass)); } + @Override + public Scroll scroll(Query query, Class entityType) { + + Assert.notNull(entityType, "Entity type must not be null"); + + return scroll(query, entityType, getCollectionName(entityType)); + } + + @Override + public Scroll scroll(Query query, Class entityType, String collectionName) { + return doScroll(query, entityType, entityType, collectionName); + } + + Scroll doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { + + Assert.notNull(query, "Query must not be null"); + Assert.notNull(collectionName, "CollectionName must not be null"); + Assert.notNull(sourceClass, "Entity type must not be null"); + Assert.notNull(targetClass, "Target type must not be null"); + + ReadDocumentCallback callback = new ReadDocumentCallback<>(mongoConverter, targetClass, collectionName); + int limit = query.isLimited() ? query.getLimit() + 1 : Integer.MAX_VALUE; + + if (query.hasKeyset()) { + + KeySetCursorQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + operations.getIdPropertyName(sourceClass)); + + List result = doFind(collectionName, createDelegate(query), keysetPaginationQuery.query(), + keysetPaginationQuery.fields(), sourceClass, + new QueryCursorPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback); + + return ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), result, operations); + } + + List result = doFind(collectionName, createDelegate(query), query.getQueryObject(), query.getFieldsObject(), + sourceClass, new QueryCursorPreparer(query, query.getSortObject(), limit, query.getSkip(), sourceClass), + callback); + + return ScrollUtils.createWindow(result, query.getLimit(), OffsetScrollPosition.positionFunction(query.getSkip())); + } + @Nullable @Override public T findById(Object id, Class entityClass) { @@ -953,7 +998,7 @@ public GeoResults geoNear(NearQuery near, Class domainType, String col optionsBuilder.readPreference(near.getReadPreference()); } - if(near.hasReadConcern()) { + if (near.hasReadConcern()) { optionsBuilder.readConcern(near.getReadConcern()); } @@ -2837,13 +2882,24 @@ private static MongoConverter getDefaultMongoConverter(MongoDatabaseFactory fact return converter; } - private Document getMappedSortObject(Query query, Class type) { + @Nullable + private Document getMappedSortObject(@Nullable Query query, Class type) { - if (query == null || ObjectUtils.isEmpty(query.getSortObject())) { + if (query == null) { return null; } - return queryMapper.getMappedSort(query.getSortObject(), mappingContext.getPersistentEntity(type)); + return getMappedSortObject(query.getSortObject(), type); + } + + @Nullable + private Document getMappedSortObject(Document sortObject, Class type) { + + if (ObjectUtils.isEmpty(sortObject)) { + return null; + } + + return queryMapper.getMappedSort(sortObject, mappingContext.getPersistentEntity(type)); } /** @@ -3206,11 +3262,23 @@ public T doWith(Document document) { class QueryCursorPreparer implements CursorPreparer { private final Query query; + + private final Document sortObject; + + private final int limit; + + private final long skip; private final @Nullable Class type; QueryCursorPreparer(Query query, @Nullable Class type) { + this(query, query.getSortObject(), query.getLimit(), query.getSkip(), type); + } + QueryCursorPreparer(Query query, Document sortObject, int limit, long skip, @Nullable Class type) { this.query = query; + this.sortObject = sortObject; + this.limit = limit; + this.skip = skip; this.type = type; } @@ -3225,20 +3293,20 @@ public FindIterable prepare(FindIterable iterable) { Meta meta = query.getMeta(); HintFunction hintFunction = HintFunction.from(query.getHint()); - if (query.getSkip() <= 0 && query.getLimit() <= 0 && ObjectUtils.isEmpty(query.getSortObject()) - && hintFunction.isEmpty() && !meta.hasValues() && query.getCollation().isEmpty()) { + if (skip <= 0 && limit <= 0 && ObjectUtils.isEmpty(sortObject) && hintFunction.isEmpty() && !meta.hasValues() + && query.getCollation().isEmpty()) { return cursorToUse; } try { - if (query.getSkip() > 0) { - cursorToUse = cursorToUse.skip((int) query.getSkip()); + if (skip > 0) { + cursorToUse = cursorToUse.skip((int) skip); } - if (query.getLimit() > 0) { - cursorToUse = cursorToUse.limit(query.getLimit()); + if (limit > 0) { + cursorToUse = cursorToUse.limit(limit); } - if (!ObjectUtils.isEmpty(query.getSortObject())) { - Document sort = type != null ? getMappedSortObject(query, type) : query.getSortObject(); + if (!ObjectUtils.isEmpty(sortObject)) { + Document sort = type != null ? getMappedSortObject(sortObject, type) : sortObject; cursorToUse = cursorToUse.sort(sort); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java index 05aeda069b..4e8c5f63de 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/QueryOperations.java @@ -28,6 +28,7 @@ import org.bson.BsonValue; import org.bson.Document; import org.bson.codecs.Codec; +import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.springframework.data.mapping.PropertyPath; import org.springframework.data.mapping.PropertyReferenceException; @@ -776,7 +777,7 @@ Document applyShardKey(MongoPersistentEntity domainType, Document filter, Document filterWithShardKey = new Document(filter); getMappedShardKeyFields(domainType) - .forEach(key -> filterWithShardKey.putIfAbsent(key, BsonUtils.resolveValue(shardKeySource, key))); + .forEach(key -> filterWithShardKey.putIfAbsent(key, BsonUtils.resolveValue((Bson) shardKeySource, key))); return filterWithShardKey; } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java index 31de934ec7..1a81f92a9a 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java @@ -18,6 +18,8 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.geo.GeoResult; import org.springframework.data.mongodb.core.query.CriteriaDefinition; import org.springframework.data.mongodb.core.query.NearQuery; @@ -87,14 +89,23 @@ interface TerminatingFind { */ Flux all(); + /** + * Return a scroll of elements either starting or resuming at {@link ScrollPosition}. + * + * @param scrollPosition the scroll position. + * @return a scroll of the resulting elements. + * @since 4.1 + * @see org.springframework.data.domain.OffsetScrollPosition + * @see org.springframework.data.domain.KeysetScrollPosition + */ + Mono> scroll(ScrollPosition scrollPosition); + /** * Get all matching elements using a {@link com.mongodb.CursorType#TailableAwait tailable cursor}. The stream will * not be completed unless the {@link org.reactivestreams.Subscription} is - * {@link org.reactivestreams.Subscription#cancel() canceled}. - *
+ * {@link org.reactivestreams.Subscription#cancel() canceled}.
* However, the stream may become dead, or invalid, if either the query returns no match or the cursor returns the - * document at the "end" of the collection and then the application deletes that document. - *
+ * document at the "end" of the collection and then the application deletes that document.
* A stream that is no longer in use must be {@link reactor.core.Disposable#dispose()} disposed} otherwise the * streams will linger and exhaust resources.
* NOTE: Requires a capped collection. @@ -105,8 +116,7 @@ interface TerminatingFind { Flux tail(); /** - * Get the number of matching elements. - *
+ * Get the number of matching elements.
* This method uses an * {@link com.mongodb.reactivestreams.client.MongoCollection#countDocuments(org.bson.conversions.Bson, com.mongodb.client.model.CountOptions) * aggregation execution} even for empty {@link Query queries} which may have an impact on performance, but diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java index 4dcf62aacd..13894c896f 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java @@ -20,6 +20,8 @@ import org.bson.Document; import org.springframework.dao.IncorrectResultSizeDataAccessException; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.CollectionPreparerSupport.ReactiveCollectionPreparerDelegate; import org.springframework.data.mongodb.core.query.NearQuery; import org.springframework.data.mongodb.core.query.Query; @@ -137,6 +139,11 @@ public Flux all() { return doFind(null); } + @Override + public Mono> scroll(ScrollPosition scrollPosition) { + return template.doScroll(query.with(scrollPosition), domainType, returnType, getCollectionName()); + } + @Override public Flux tail() { return doFind(template.new TailingQueryFindPublisherPreparer(query, domainType)); diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java index 323ca9dd95..d252cb4a40 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java @@ -25,7 +25,8 @@ import org.bson.Document; import org.reactivestreams.Publisher; import org.reactivestreams.Subscription; - +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.GeoResult; import org.springframework.data.mongodb.ReactiveMongoDatabaseFactory; import org.springframework.data.mongodb.core.aggregation.Aggregation; @@ -279,7 +280,8 @@ default Mono> createView(String name, Class source, * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - Mono> createView(String name, Class source, AggregationPipeline pipeline, @Nullable ViewOptions options); + Mono> createView(String name, Class source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * Create a view with the provided name. The view content is defined by the {@link AggregationPipeline pipeline} on @@ -291,7 +293,8 @@ default Mono> createView(String name, Class source, * @param options additional settings to apply when creating the view. Can be {@literal null}. * @since 4.0 */ - Mono> createView(String name, String source, AggregationPipeline pipeline, @Nullable ViewOptions options); + Mono> createView(String name, String source, AggregationPipeline pipeline, + @Nullable ViewOptions options); /** * A set of collection names. @@ -462,6 +465,45 @@ default Mono> createView(String name, Class source, */ Flux find(Query query, Class entityClass, String collectionName); + /** + * Query for a scroll of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @return {@link Mono} emitting the converted scroll. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Mono> scroll(Query query, Class entityType); + + /** + * Query for a scroll of objects of type T from the specified collection.
+ * Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with + * {@link Query#limit(int)} to limit large query results for efficient scrolling.
+ * Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. + * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
+ * If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way + * to map objects since the test for class type is done in the client and not on the server. + * + * @param query the query class that specifies the criteria used to find a record and also an optional fields + * specification. Must not be {@literal null}. + * @param entityType the parametrized type of the returned list. + * @param collectionName name of the collection to retrieve the objects from. + * @return {@link Mono} emitting the converted scroll window. + * @since 4.1 + * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) + * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) + */ + Mono> scroll(Query query, Class entityType, String collectionName); + /** * Returns a document with the given id mapped onto the given class. The collection the query is ran against will be * derived from the given target class as well. diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java index 3983ee1e0c..55f924e6e9 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java @@ -58,6 +58,8 @@ import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.dao.support.PersistenceExceptionTranslator; import org.springframework.data.convert.EntityReader; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; import org.springframework.data.geo.Distance; import org.springframework.data.geo.GeoResult; import org.springframework.data.geo.Metric; @@ -78,6 +80,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; +import org.springframework.data.mongodb.core.ScrollUtils.KeySetCursorQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -826,6 +829,49 @@ public Flux find(@Nullable Query query, Class entityClass, String coll query.getFieldsObject(), entityClass, new QueryFindPublisherPreparer(query, entityClass)); } + @Override + public Mono> scroll(Query query, Class entityType) { + + Assert.notNull(entityType, "Entity type must not be null"); + + return scroll(query, entityType, getCollectionName(entityType)); + } + + @Override + public Mono> scroll(Query query, Class entityType, String collectionName) { + return doScroll(query, entityType, entityType, collectionName); + } + + Mono> doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { + + Assert.notNull(query, "Query must not be null"); + Assert.notNull(collectionName, "CollectionName must not be null"); + Assert.notNull(sourceClass, "Entity type must not be null"); + Assert.notNull(targetClass, "Target type must not be null"); + + int limit = query.isLimited() ? query.getLimit() + 1 : Integer.MAX_VALUE; + + if (query.hasKeyset()) { + + KeySetCursorQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + operations.getIdPropertyName(sourceClass)); + + Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), + keysetPaginationQuery.query(), keysetPaginationQuery.fields(), targetClass, + new QueryFindPublisherPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass)).collectList(); + + return result.map(it -> ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), it, operations)); + } + + Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), query.getQueryObject(), + query.getFieldsObject(), targetClass, + new QueryFindPublisherPreparer(query, query.getSortObject(), limit, query.getSkip(), sourceClass)) + .collectList(); + + return result.map( + it -> ScrollUtils.createWindow(it, query.getLimit(), OffsetScrollPosition.positionFunction(query.getSkip()))); + } + @Override public Mono findById(Object id, Class entityClass) { return findById(id, entityClass, getCollectionName(entityClass)); @@ -1004,7 +1050,7 @@ protected Flux> geoNear(NearQuery near, Class entityClass, S optionsBuilder.readPreference(near.getReadPreference()); } - if(near.hasReadConcern()) { + if (near.hasReadConcern()) { optionsBuilder.readConcern(near.getReadConcern()); } @@ -2652,13 +2698,24 @@ private MappingMongoConverter getDefaultMongoConverter() { return converter; } + @Nullable private Document getMappedSortObject(Query query, Class type) { if (query == null) { return null; } - return queryMapper.getMappedSort(query.getSortObject(), mappingContext.getPersistentEntity(type)); + return getMappedSortObject(query.getSortObject(), type); + } + + @Nullable + private Document getMappedSortObject(Document sortObject, Class type) { + + if (ObjectUtils.isEmpty(sortObject)) { + return null; + } + + return queryMapper.getMappedSort(sortObject, mappingContext.getPersistentEntity(type)); } // Callback implementations @@ -3088,11 +3145,24 @@ public Mono> doWith(Document object) { class QueryFindPublisherPreparer implements FindPublisherPreparer { private final Query query; + + private final Document sortObject; + + private final int limit; + + private final long skip; private final @Nullable Class type; QueryFindPublisherPreparer(Query query, @Nullable Class type) { + this(query, query.getSortObject(), query.getLimit(), query.getSkip(), type); + } + + QueryFindPublisherPreparer(Query query, Document sortObject, int limit, long skip, @Nullable Class type) { this.query = query; + this.sortObject = sortObject; + this.limit = limit; + this.skip = skip; this.type = type; } @@ -3107,23 +3177,23 @@ public FindPublisher prepare(FindPublisher findPublisher) { HintFunction hintFunction = HintFunction.from(query.getHint()); Meta meta = query.getMeta(); - if (query.getSkip() <= 0 && query.getLimit() <= 0 && ObjectUtils.isEmpty(query.getSortObject()) - && hintFunction.isEmpty() && !meta.hasValues()) { + if (skip <= 0 && limit <= 0 && ObjectUtils.isEmpty(sortObject) && hintFunction.isEmpty() + && !meta.hasValues()) { return findPublisherToUse; } try { - if (query.getSkip() > 0) { - findPublisherToUse = findPublisherToUse.skip((int) query.getSkip()); + if (skip > 0) { + findPublisherToUse = findPublisherToUse.skip((int) skip); } - if (query.getLimit() > 0) { - findPublisherToUse = findPublisherToUse.limit(query.getLimit()); + if (limit > 0) { + findPublisherToUse = findPublisherToUse.limit(limit); } - if (!ObjectUtils.isEmpty(query.getSortObject())) { - Document sort = type != null ? getMappedSortObject(query, type) : query.getSortObject(); + if (!ObjectUtils.isEmpty(sortObject)) { + Document sort = type != null ? getMappedSortObject(sortObject, type) : sortObject; findPublisherToUse = findPublisherToUse.sort(sort); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java new file mode 100644 index 0000000000..6458fa9d2a --- /dev/null +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java @@ -0,0 +1,145 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.mongodb.core; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.IntFunction; + +import org.bson.Document; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.mongodb.core.EntityOperations.Entity; +import org.springframework.data.mongodb.core.query.Query; + +/** + * Utilities to run scroll queries and create {@link Scroll} results. + * + * @author Mark Paluch + * @since 4.1 + */ +class ScrollUtils { + + /** + * Create the actual query to run keyset-based pagination. Affects projection, sorting, and the criteria. + * + * @param query + * @param idPropertyName + * @return + */ + static KeySetCursorQuery createKeysetPaginationQuery(Query query, String idPropertyName) { + + Document sortObject = query.isSorted() ? query.getSortObject() : new Document(); + sortObject.put(idPropertyName, 1); + + // make sure we can extract the keyset + Document fieldsObject = query.getFieldsObject(); + if (!fieldsObject.isEmpty()) { + for (String field : sortObject.keySet()) { + fieldsObject.put(field, 1); + } + } + + Document queryObject = query.getQueryObject(); + + List or = (List) queryObject.getOrDefault("$or", new ArrayList<>()); + + // TODO: reverse scrolling + Map keysetValues = query.getKeyset().getKeys(); + Document keysetSort = new Document(); + List sortKeys = new ArrayList<>(sortObject.keySet()); + + if (!keysetValues.isEmpty() && !keysetValues.keySet().containsAll(sortKeys)) { + throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values"); + } + + // first query doesn't come with a keyset + if (!keysetValues.isEmpty()) { + + // build matrix query for keyset paging that contains sort^2 queries + // reflecting a query that follows sort order semantics starting from the last returned keyset + for (int i = 0; i < sortKeys.size(); i++) { + + Document sortConstraint = new Document(); + + for (int j = 0; j < sortKeys.size(); j++) { + + String sortSegment = sortKeys.get(j); + int sortOrder = sortObject.getInteger(sortSegment); + Object o = keysetValues.get(sortSegment); + + if (j >= i) { // tail segment + sortConstraint.put(sortSegment, new Document(sortOrder == 1 ? "$gt" : "$lt", o)); + break; + } + + sortConstraint.put(sortSegment, o); + } + + if (!sortConstraint.isEmpty()) { + or.add(sortConstraint); + } + } + } + + if (!keysetSort.isEmpty()) { + or.add(keysetSort); + } + if (!or.isEmpty()) { + queryObject.put("$or", or); + } + + return new KeySetCursorQuery(queryObject, fieldsObject, sortObject); + } + + static Scroll createWindow(Document sortObject, int limit, List result, EntityOperations operations) { + + IntFunction positionFunction = value -> { + + T last = result.get(value); + Entity entity = operations.forEntity(last); + + Map keys = entity.extractKeys(sortObject); + return KeysetScrollPosition.of(keys); + }; + + return createWindow(result, limit, positionFunction); + } + + static Scroll createWindow(List result, int limit, IntFunction positionFunction) { + return Scroll.from(getSubList(result, limit), positionFunction, hasMoreElements(result, limit)); + } + + static boolean hasMoreElements(List result, int limit) { + return !result.isEmpty() && result.size() > limit; + } + + static List getSubList(List result, int limit) { + + if (limit > 0 && result.size() > limit) { + return result.subList(0, limit); + } + + return result; + } + + record KeySetCursorQuery(Document query, Document fields, Document sort) { + + } + +} diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java index 912e7d5cea..e631852a42 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/query/Query.java @@ -30,7 +30,10 @@ import java.util.Set; import org.bson.Document; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.OffsetScrollPosition; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Order; import org.springframework.data.mongodb.InvalidMongoDbApiUsageException; @@ -64,6 +67,8 @@ public class Query implements ReadConcernAware, ReadPreferenceAware { private Sort sort = Sort.unsorted(); private long skip; private int limit; + + private KeysetScrollPosition keysetScrollPosition; private @Nullable ReadConcern readConcern; private @Nullable ReadPreference readPreference; @@ -255,6 +260,67 @@ public Query with(Pageable pageable) { return with(pageable.getSort()); } + /** + * Sets the given cursor position on the {@link Query} instance. Will transparently set {@code skip}. + * + * @param position must not be {@literal null}. + * @return this. + */ + public Query with(ScrollPosition position) { + + Assert.notNull(position, "ScrollPosition must not be null"); + + if (position instanceof OffsetScrollPosition offset) { + return with(offset); + } + + if (position instanceof KeysetScrollPosition keyset) { + return with(keyset); + } + + throw new IllegalArgumentException(String.format("ScrollPosition %s not supported", position)); + } + + /** + * Sets the given cursor position on the {@link Query} instance. Will transparently set {@code skip}. + * + * @param position must not be {@literal null}. + * @return this. + */ + public Query with(OffsetScrollPosition position) { + + Assert.notNull(position, "ScrollPosition must not be null"); + + this.skip = position.getOffset(); + this.keysetScrollPosition = null; + return this; + } + + /** + * Sets the given cursor position on the {@link Query} instance. Will transparently reset {@code skip}. + * + * @param position must not be {@literal null}. + * @return this. + */ + public Query with(KeysetScrollPosition position) { + + Assert.notNull(position, "ScrollPosition must not be null"); + + this.skip = 0; + this.keysetScrollPosition = position; + + return this; + } + + public boolean hasKeyset() { + return keysetScrollPosition != null; + } + + @Nullable + public KeysetScrollPosition getKeyset() { + return keysetScrollPosition; + } + /** * Adds a {@link Sort} to the {@link Query} instance. * @@ -384,11 +450,22 @@ public long getSkip() { return this.skip; } + /** + * Returns whether the query is {@link #limit(int) limited}. + * + * @return {@code true} if the query is limited; {@code false} otherwise. + * @since 4.1 + */ + public boolean isLimited() { + return this.limit > 0; + } + /** * Get the maximum number of documents to be return. {@literal Zero} or a {@literal negative} value indicates no * limit. * * @return number of documents to return. + * @see #isLimited() */ public int getLimit() { return this.limit; @@ -688,4 +765,5 @@ public int hashCode() { public static boolean isRestrictedTypeKey(String key) { return RESTRICTED_TYPES_KEY.equals(key); } + } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java index 823b64d324..930a733315 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractMongoQuery.java @@ -167,6 +167,9 @@ private MongoQueryExecution getExecution(ConvertingParameterAccessor accessor, F return q -> operation.matching(q).stream(); } else if (method.isCollectionQuery()) { return q -> operation.matching(q.with(accessor.getPageable()).with(accessor.getSort())).all(); + } else if (method.isScrollQuery()) { + return q -> operation.matching(q.with(accessor.getPageable()).with(accessor.getSort())) + .scroll(accessor.getScrollPosition()); } else if (method.isPageQuery()) { return new PagedExecution(operation, accessor.getPageable()); } else if (isCountQuery()) { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java index ed1fee68c6..fbb078b43e 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/AbstractReactiveMongoQuery.java @@ -203,6 +203,9 @@ private ReactiveMongoQueryExecution getExecutionToWrap(MongoParameterAccessor ac return (q, t, c) -> operation.matching(q.with(accessor.getPageable())).tail(); } else if (method.isCollectionQuery()) { return (q, t, c) -> operation.matching(q.with(accessor.getPageable())).all(); + } else if (method.isScrollQuery()) { + return (q, t, c) -> operation.matching(q.with(accessor.getPageable()).with(accessor.getSort())) + .scroll(accessor.getScrollPosition()); } else if (isCountQuery()) { return (q, t, c) -> operation.matching(q).count(); } else if (isExistsQuery()) { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java index 21513efbfb..b3ecef9856 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ConvertingParameterAccessor.java @@ -23,6 +23,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Range; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.geo.Distance; import org.springframework.data.geo.Point; @@ -71,6 +72,11 @@ public PotentiallyConvertingIterator iterator() { return new ConvertingIterator(delegate.iterator()); } + @Override + public ScrollPosition getScrollPosition() { + return delegate.getScrollPosition(); + } + public Pageable getPageable() { return delegate.getPageable(); } @@ -197,7 +203,7 @@ private static Collection asCollection(@Nullable Object source) { if (source instanceof Iterable) { - if(source instanceof Collection) { + if (source instanceof Collection) { return new ArrayList<>((Collection) source); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java index 451e743389..1b8f6b6a58 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/MongoQueryMethod.java @@ -335,8 +335,8 @@ public boolean hasAnnotatedCollation() { public String getAnnotatedCollation() { return doFindAnnotation(Collation.class).map(Collation::value) // - .orElseThrow(() -> new IllegalStateException( - "Expected to find @Collation annotation but did not; Make sure to check hasAnnotatedCollation() before.")); + .orElseThrow(() -> new IllegalStateException( + "Expected to find @Collation annotation but did not; Make sure to check hasAnnotatedCollation() before.")); } /** @@ -420,7 +420,8 @@ public void verify() { if (isModifyingQuery()) { - if (isCollectionQuery() || isSliceQuery() || isPageQuery() || isGeoNearQuery() || !isNumericOrVoidReturnValue()) { // + if (isCollectionQuery() || isScrollQuery() || isSliceQuery() || isPageQuery() || isGeoNearQuery() + || !isNumericOrVoidReturnValue()) { // throw new IllegalStateException( String.format("Update method may be void or return a numeric value (the number of updated documents)." + "Offending method: %s", method)); diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java index a64822a263..904c184a8f 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/query/ReactiveMongoQueryMethod.java @@ -66,7 +66,7 @@ public ReactiveMongoQueryMethod(Method method, RepositoryMetadata metadata, Proj super(method, metadata, projectionFactory, mappingContext); this.method = method; - this.isCollectionQuery = Lazy.of(() -> (!(isPageQuery() || isSliceQuery()) + this.isCollectionQuery = Lazy.of(() -> (!(isPageQuery() || isSliceQuery() || isScrollQuery()) && ReactiveWrappers.isMultiValueType(metadata.getReturnType(method).getType()) || super.isCollectionQuery())); } @@ -136,7 +136,16 @@ public void verify() { boolean multiWrapper = ReactiveWrappers.isMultiValueType(returnType.getType()); boolean singleWrapperWithWrappedPageableResult = ReactiveWrappers.isSingleValueType(returnType.getType()) && (PAGE_TYPE.isAssignableFrom(returnType.getRequiredComponentType()) - || SLICE_TYPE.isAssignableFrom(returnType.getRequiredComponentType())); + || SLICE_TYPE.isAssignableFrom(returnType.getRequiredComponentType())); + + if (hasParameterOfType(method, Sort.class)) { + throw new IllegalStateException(String.format("Method must not have Pageable *and* Sort parameter;" + + " Use sorting capabilities on Pageable instead; Offending method: %s", method)); + } + + if (isScrollQuery()) { + return; + } if (singleWrapperWithWrappedPageableResult) { throw new InvalidDataAccessApiUsageException( @@ -149,11 +158,6 @@ public void verify() { "Method has to use a either multi-item reactive wrapper return type or a wrapped Page/Slice type; Offending method: %s", method.toString())); } - - if (hasParameterOfType(method, Sort.class)) { - throw new IllegalStateException(String.format("Method must not have Pageable *and* Sort parameter;" - + " Use sorting capabilities on Pageable instead; Offending method: %s", method)); - } } super.verify(); diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java index 74abf97ab2..79d26a524f 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/FetchableFluentQuerySupport.java @@ -33,17 +33,21 @@ abstract class FetchableFluentQuerySupport implements FluentQuery.Fetchabl private final P predicate; private final Sort sort; + + private final int limit; + private final Class resultType; private final List fieldsToInclude; - FetchableFluentQuerySupport(P predicate, Sort sort, Class resultType, List fieldsToInclude) { + FetchableFluentQuerySupport(P predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { this.predicate = predicate; this.sort = sort; + this.limit = limit; this.resultType = resultType; this.fieldsToInclude = fieldsToInclude; } - /* + /* * (non-Javadoc) * @see org.springframework.data.repository.query.FluentQuery.FetchableFluentQuery#sortBy(org.springframework.data.domain.Sort) */ @@ -52,10 +56,18 @@ public FluentQuery.FetchableFluentQuery sortBy(Sort sort) { Assert.notNull(sort, "Sort must not be null"); - return create(predicate, sort, resultType, fieldsToInclude); + return create(predicate, sort, limit, resultType, fieldsToInclude); } - /* + @Override + public FluentQuery.FetchableFluentQuery limit(int limit) { + + Assert.isTrue(limit > 0, "Limit must be greater zero"); + + return create(predicate, sort, limit, resultType, fieldsToInclude); + } + + /* * (non-Javadoc) * @see org.springframework.data.repository.query.FluentQuery.FetchableFluentQuery#as(java.lang.Class) */ @@ -64,10 +76,10 @@ public FluentQuery.FetchableFluentQuery as(Class projection) { Assert.notNull(projection, "Projection target type must not be null"); - return create(predicate, sort, projection, fieldsToInclude); + return create(predicate, sort, limit, projection, fieldsToInclude); } - /* + /* * (non-Javadoc) * @see org.springframework.data.repository.query.FluentQuery.FetchableFluentQuery#project(java.util.Collection) */ @@ -76,11 +88,11 @@ public FluentQuery.FetchableFluentQuery project(Collection properties Assert.notNull(properties, "Projection properties must not be null"); - return create(predicate, sort, resultType, new ArrayList<>(properties)); + return create(predicate, sort, limit, resultType, new ArrayList<>(properties)); } - protected abstract FetchableFluentQuerySupport create(P predicate, Sort sort, Class resultType, - List fieldsToInclude); + protected abstract FetchableFluentQuerySupport create(P predicate, Sort sort, int limit, + Class resultType, List fieldsToInclude); P getPredicate() { return predicate; @@ -90,6 +102,10 @@ Sort getSort() { return sort; } + int getLimit() { + return limit; + } + Class getResultType() { return resultType; } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java index 0399dfd5c8..4cf3ab3fe2 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java @@ -25,6 +25,8 @@ import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.query.BasicQuery; @@ -228,17 +230,17 @@ private SpringDataMongodbQuery applySorting(SpringDataMongodbQuery query, class FluentQuerydsl extends FetchableFluentQuerySupport { FluentQuerydsl(Predicate predicate, Class resultType) { - this(predicate, Sort.unsorted(), resultType, Collections.emptyList()); + this(predicate, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - FluentQuerydsl(Predicate predicate, Sort sort, Class resultType, List fieldsToInclude) { - super(predicate, sort, resultType, fieldsToInclude); + FluentQuerydsl(Predicate predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { + super(predicate, sort, limit, resultType, fieldsToInclude); } @Override - protected FluentQuerydsl create(Predicate predicate, Sort sort, Class resultType, + protected FluentQuerydsl create(Predicate predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { - return new FluentQuerydsl<>(predicate, sort, resultType, fieldsToInclude); + return new FluentQuerydsl<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -256,6 +258,11 @@ public List all() { return createQuery().fetch(); } + @Override + public Scroll scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Page page(Pageable pageable) { @@ -296,6 +303,8 @@ private void customize(BasicQuery query) { if (getSort().isSorted()) { query.with(getSort()); } + + query.limit(getLimit()); } } } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java index 505a7c0c44..9147243648 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveFluentQuerySupport.java @@ -33,12 +33,14 @@ abstract class ReactiveFluentQuerySupport implements FluentQuery.ReactiveF private final P predicate; private final Sort sort; + private final int limit; private final Class resultType; private final List fieldsToInclude; - ReactiveFluentQuerySupport(P predicate, Sort sort, Class resultType, List fieldsToInclude) { + ReactiveFluentQuerySupport(P predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { this.predicate = predicate; this.sort = sort; + this.limit = limit; this.resultType = resultType; this.fieldsToInclude = fieldsToInclude; } @@ -52,7 +54,15 @@ public ReactiveFluentQuery sortBy(Sort sort) { Assert.notNull(sort, "Sort must not be null"); - return create(predicate, sort, resultType, fieldsToInclude); + return create(predicate, sort, limit, resultType, fieldsToInclude); + } + + @Override + public ReactiveFluentQuery limit(int limit) { + + Assert.isTrue(limit > 0, "Limit must be greater zero"); + + return create(predicate, sort, limit, resultType, fieldsToInclude); } /* @@ -64,7 +74,7 @@ public ReactiveFluentQuery as(Class projection) { Assert.notNull(projection, "Projection target type must not be null"); - return create(predicate, sort, projection, fieldsToInclude); + return create(predicate, sort, limit, projection, fieldsToInclude); } /* @@ -76,10 +86,10 @@ public ReactiveFluentQuery project(Collection properties) { Assert.notNull(properties, "Projection properties must not be null"); - return create(predicate, sort, resultType, new ArrayList<>(properties)); + return create(predicate, sort, limit, resultType, new ArrayList<>(properties)); } - protected abstract ReactiveFluentQuerySupport create(P predicate, Sort sort, Class resultType, + protected abstract ReactiveFluentQuerySupport create(P predicate, Sort sort, int limit, Class resultType, List fieldsToInclude); P getPredicate() { @@ -90,6 +100,10 @@ Sort getSort() { return sort; } + int getLimit() { + return limit; + } + Class getResultType() { return resultType; } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java index a21bbb6c17..ff2b442931 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java @@ -26,6 +26,8 @@ import org.reactivestreams.Publisher; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ReactiveMongoOperations; import org.springframework.data.mongodb.core.query.BasicQuery; @@ -195,17 +197,18 @@ private ReactiveSpringDataMongodbQuery applySorting(ReactiveSpringDataMongodb class ReactiveFluentQuerydsl extends ReactiveFluentQuerySupport { ReactiveFluentQuerydsl(Predicate predicate, Class resultType) { - this(predicate, Sort.unsorted(), resultType, Collections.emptyList()); + this(predicate, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - ReactiveFluentQuerydsl(Predicate predicate, Sort sort, Class resultType, List fieldsToInclude) { - super(predicate, sort, resultType, fieldsToInclude); + ReactiveFluentQuerydsl(Predicate predicate, Sort sort, int limit, Class resultType, + List fieldsToInclude) { + super(predicate, sort, limit, resultType, fieldsToInclude); } @Override - protected ReactiveFluentQuerydsl create(Predicate predicate, Sort sort, Class resultType, + protected ReactiveFluentQuerydsl create(Predicate predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { - return new ReactiveFluentQuerydsl<>(predicate, sort, resultType, fieldsToInclude); + return new ReactiveFluentQuerydsl<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -223,6 +226,11 @@ public Flux all() { return createQuery().fetch(); } + @Override + public Mono> scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Mono> page(Pageable pageable) { @@ -260,6 +268,8 @@ private void customize(BasicQuery query) { if (getSort().isSorted()) { query.with(getSort()); } + + query.limit(getLimit()); } } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java index 7cc102e4d6..cc27f81d68 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java @@ -24,9 +24,10 @@ import java.util.function.Consumer; import org.bson.Document; - import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.ReactiveFindOperation; import org.springframework.data.mongodb.core.ReactiveMongoOperations; @@ -90,6 +91,10 @@ Flux fetch() { return createQuery().flatMapMany(it -> find.matching(it).all()); } + Mono> scroll(ScrollPosition scrollPosition) { + return createQuery().flatMap(it -> find.matching(it).scroll(scrollPosition)); + } + /** * Fetch all matching query results as page. * @@ -97,8 +102,8 @@ Flux fetch() { */ Mono> fetchPage(Pageable pageable) { - Mono> content = createQuery().map(it -> it.with(pageable)) - .flatMapMany(it -> find.matching(it).all()).collectList(); + Mono> content = createQuery().map(it -> it.with(pageable)).flatMapMany(it -> find.matching(it).all()) + .collectList(); return content.flatMap(it -> ReactivePageableExecutionUtils.getPage(it, pageable, fetchCount())); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java index 23a459123a..750be26aeb 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java @@ -32,6 +32,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ExecutableFindOperation; import org.springframework.data.mongodb.core.MongoOperations; @@ -361,17 +363,17 @@ private List findAll(@Nullable Query query) { class FluentQueryByExample extends FetchableFluentQuerySupport, T> { FluentQueryByExample(Example example, Class resultType) { - this(example, Sort.unsorted(), resultType, Collections.emptyList()); + this(example, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - FluentQueryByExample(Example example, Sort sort, Class resultType, List fieldsToInclude) { - super(example, sort, resultType, fieldsToInclude); + FluentQueryByExample(Example example, Sort sort, int limit, Class resultType, List fieldsToInclude) { + super(example, sort, limit, resultType, fieldsToInclude); } @Override - protected FluentQueryByExample create(Example predicate, Sort sort, Class resultType, + protected FluentQueryByExample create(Example predicate, Sort sort, int limit, Class resultType, List fieldsToInclude) { - return new FluentQueryByExample<>(predicate, sort, resultType, fieldsToInclude); + return new FluentQueryByExample<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -389,6 +391,11 @@ public List all() { return createQuery().all(); } + @Override + public Scroll scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Page page(Pageable pageable) { @@ -427,6 +434,8 @@ private ExecutableFindOperation.TerminatingFind createQuery(UnaryOperator findAll(Query query) { class ReactiveFluentQueryByExample extends ReactiveFluentQuerySupport, T> { ReactiveFluentQueryByExample(Example example, Class resultType) { - this(example, Sort.unsorted(), resultType, Collections.emptyList()); + this(example, Sort.unsorted(), 0, resultType, Collections.emptyList()); } - ReactiveFluentQueryByExample(Example example, Sort sort, Class resultType, List fieldsToInclude) { - super(example, sort, resultType, fieldsToInclude); + ReactiveFluentQueryByExample(Example example, Sort sort, int limit, Class resultType, + List fieldsToInclude) { + super(example, sort, limit, resultType, fieldsToInclude); } @Override - protected ReactiveFluentQueryByExample create(Example predicate, Sort sort, Class resultType, - List fieldsToInclude) { - return new ReactiveFluentQueryByExample<>(predicate, sort, resultType, fieldsToInclude); + protected ReactiveFluentQueryByExample create(Example predicate, Sort sort, int limit, + Class resultType, List fieldsToInclude) { + return new ReactiveFluentQueryByExample<>(predicate, sort, limit, resultType, fieldsToInclude); } @Override @@ -432,6 +435,11 @@ public Flux all() { return createQuery().all(); } + @Override + public Mono> scroll(ScrollPosition scrollPosition) { + return createQuery().scroll(scrollPosition); + } + @Override public Mono> page(Pageable pageable) { @@ -465,6 +473,8 @@ private ReactiveFindOperation.TerminatingFind createQuery(UnaryOperator typ * @param type must not be {@literal null}. * @param collectionName must not be {@literal null} or empty. */ - public SpringDataMongodbQuery(MongoOperations operations, Class type, - String collectionName) { + public SpringDataMongodbQuery(MongoOperations operations, Class type, String collectionName) { this(operations, type, type, collectionName, it -> {}); } @@ -133,6 +133,17 @@ public void close() { } } + public Scroll scroll(ScrollPosition scrollPosition) { + + try { + return find.matching(createQuery()).scroll(scrollPosition); + } catch (RuntimeException e) { + return handleException(e, Scroll.from(Collections.emptyList(), value -> { + throw new UnsupportedOperationException(); + })); + } + } + @Override public Stream stream() { diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java index b8d4093f75..248b03ab27 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/util/BsonUtils.java @@ -24,16 +24,7 @@ import java.util.function.Function; import java.util.stream.StreamSupport; -import org.bson.BSONObject; -import org.bson.BsonBinary; -import org.bson.BsonBoolean; -import org.bson.BsonDouble; -import org.bson.BsonInt32; -import org.bson.BsonInt64; -import org.bson.BsonObjectId; -import org.bson.BsonString; -import org.bson.BsonValue; -import org.bson.Document; +import org.bson.*; import org.bson.codecs.DocumentCodec; import org.bson.codecs.configuration.CodecRegistry; import org.bson.conversions.Bson; @@ -489,7 +480,7 @@ public static Document parse(String json, @Nullable CodecRegistryProvider codecR } /** - * Resolve a the value for a given key. If the given {@link Bson} value contains the key the value is immediately + * Resolve the value for a given key. If the given {@link Bson} value contains the key the value is immediately * returned. If not and the key contains a path using the dot ({@code .}) notation it will try to resolve the path by * inspecting the individual parts. If one of the intermediate ones is {@literal null} or cannot be inspected further * (wrong) type, {@literal null} is returned. @@ -501,8 +492,22 @@ public static Document parse(String json, @Nullable CodecRegistryProvider codecR */ @Nullable public static Object resolveValue(Bson bson, String key) { + return resolveValue(asMap(bson), key); + } - Map source = asMap(bson); + /** + * Resolve the value for a given key. If the given {@link Map} value contains the key the value is immediately + * returned. If not and the key contains a path using the dot ({@code .}) notation it will try to resolve the path by + * inspecting the individual parts. If one of the intermediate ones is {@literal null} or cannot be inspected further + * (wrong) type, {@literal null} is returned. + * + * @param source the source to inspect. Must not be {@literal null}. + * @param key the key to lookup. Must not be {@literal null}. + * @return can be {@literal null}. + * @since 4.1 + */ + @Nullable + public static Object resolveValue(Map source, String key) { if (source.containsKey(key) || !key.contains(".")) { return source.get(key); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java new file mode 100644 index 0000000000..865f54f202 --- /dev/null +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java @@ -0,0 +1,147 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.mongodb.core; + +import static org.springframework.data.mongodb.core.query.Criteria.*; +import static org.springframework.data.mongodb.test.util.Assertions.*; + +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.Stream; + +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.data.auditing.IsNewAwareAuditingHandler; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Sort; +import org.springframework.data.mapping.context.PersistentEntities; +import org.springframework.data.mongodb.core.MongoTemplateTests.PersonWithIdPropertyOfTypeUUIDListener; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.test.util.Client; +import org.springframework.data.mongodb.test.util.MongoClientExtension; +import org.springframework.data.mongodb.test.util.MongoTestTemplate; + +import com.mongodb.client.MongoClient; + +/** + * Integration tests for {@link Scroll} queries. + * + * @author Mark Paluch + */ +@ExtendWith(MongoClientExtension.class) +class MongoTemplateScrollTests { + + static @Client MongoClient client; + + public static final String DB_NAME = "mongo-template-scroll-tests"; + + ConfigurableApplicationContext context = new GenericApplicationContext(); + + MongoTestTemplate template = new MongoTestTemplate(cfg -> { + + cfg.configureDatabaseFactory(it -> { + + it.client(client); + it.defaultDb(DB_NAME); + }); + + cfg.configureMappingContext(it -> { + it.autocreateIndex(false); + it.initialEntitySet(AuditablePerson.class); + }); + + cfg.configureApplicationContext(it -> { + it.applicationContext(context); + it.addEventListener(new PersonWithIdPropertyOfTypeUUIDListener()); + }); + + cfg.configureAuditing(it -> { + it.auditingHandler(ctx -> { + return new IsNewAwareAuditingHandler(PersistentEntities.of(ctx)); + }); + }); + }); + + @BeforeEach + void setUp() { + template.remove(Person.class).all(); + } + + @ParameterizedTest // GH-4308 + @MethodSource("positions") + public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + + Person john20 = new Person("John", 20); + Person john40_1 = new Person("John", 40); + Person john40_2 = new Person("John", 40); + Person jane_20 = new Person("Jane", 20); + Person jane_40 = new Person("Jane", 40); + Person jane_42 = new Person("Jane", 42); + + template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42)); + Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); + q.with(scrollPosition); + + Scroll scroll = template.scroll(q, resultType, "person"); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(2); + assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); + + scroll = template.scroll(q.with(scroll.lastPosition()).limit(3), resultType, "person"); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(3); + assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + + scroll = template.scroll(q.with(scroll.lastPosition()).limit(1), resultType, "person"); + + assertThat(scroll.hasNext()).isFalse(); + assertThat(scroll.isLast()).isTrue(); + assertThat(scroll).hasSize(1); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + } + + static Stream positions() { + + return Stream.of(args(KeysetScrollPosition.initial(), Person.class, Function.identity()), // + args(KeysetScrollPosition.initial(), Document.class, MongoTemplateScrollTests::toDocument), // + args(OffsetScrollPosition.initial(), Person.class, Function.identity())); + } + + private static Arguments args(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + return Arguments.of(scrollPosition, resultType, assertionConverter); + } + + static Document toDocument(Person person) { + return new Document("_class", person.getClass().getName()).append("_id", person.getId()).append("active", true) + .append("firstName", person.getFirstName()).append("age", person.getAge()); + } +} diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java index 6af8f74680..9bfbe989b6 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateTests.java @@ -46,7 +46,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; - import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.support.GenericApplicationContext; import org.springframework.core.convert.converter.Converter; diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java new file mode 100644 index 0000000000..ede69c396f --- /dev/null +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java @@ -0,0 +1,144 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.mongodb.core; + +import static org.springframework.data.mongodb.core.query.Criteria.*; +import static org.springframework.data.mongodb.test.util.Assertions.*; + +import reactor.test.StepVerifier; + +import java.time.Duration; +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.Stream; + +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.OffsetScrollPosition; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Sort; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.test.util.Client; +import org.springframework.data.mongodb.test.util.MongoClientExtension; +import org.springframework.data.mongodb.test.util.ReactiveMongoTestTemplate; + +import com.mongodb.reactivestreams.client.MongoClient; + +/** + * Integration tests for {@link Scroll} queries. + * + * @author Mark Paluch + */ +@ExtendWith(MongoClientExtension.class) +class ReactiveMongoTemplateScrollTests { + + static @Client MongoClient client; + + public static final String DB_NAME = "mongo-template-scroll-tests"; + + ConfigurableApplicationContext context = new GenericApplicationContext(); + + private ReactiveMongoTestTemplate template = new ReactiveMongoTestTemplate(cfg -> { + + cfg.configureDatabaseFactory(it -> { + + it.client(client); + it.defaultDb(DB_NAME); + }); + + cfg.configureApplicationContext(it -> { + it.applicationContext(context); + }); + }); + + @BeforeEach + void setUp() { + template.remove(Person.class).all() // + .as(StepVerifier::create) // + .expectNextCount(1) // + .verifyComplete(); + } + + @ParameterizedTest // GH-4308 + @MethodSource("positions") + public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + + Person john20 = new Person("John", 20); + Person john40_1 = new Person("John", 40); + Person john40_2 = new Person("John", 40); + Person jane_20 = new Person("Jane", 20); + Person jane_40 = new Person("Jane", 40); + Person jane_42 = new Person("Jane", 42); + + template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42)) // + .as(StepVerifier::create) // + .expectNextCount(6) // + .verifyComplete(); + + Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); + q.with(scrollPosition); + + Scroll scroll = template.scroll(q, resultType, "person").block(Duration.ofSeconds(10)); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(2); + assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); + + scroll = template.scroll(q.limit(3).with(scroll.lastPosition()), resultType, "person") + .block(Duration.ofSeconds(10)); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(3); + assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + + scroll = template.scroll(q.limit(1).with(scroll.lastPosition()), resultType, "person") + .block(Duration.ofSeconds(10)); + + assertThat(scroll.hasNext()).isFalse(); + assertThat(scroll.isLast()).isTrue(); + assertThat(scroll).hasSize(1); + assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + } + + static Stream positions() { + + return Stream.of(args(KeysetScrollPosition.initial(), Person.class, Function.identity()), // + args(KeysetScrollPosition.initial(), Document.class, MongoTemplateScrollTests::toDocument), // + args(OffsetScrollPosition.initial(), Person.class, Function.identity())); + } + + private static Arguments args(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter) { + return Arguments.of(scrollPosition, resultType, assertionConverter); + } + + static Document toDocument(Person person) { + return new Document("_class", person.getClass().getName()).append("_id", person.getId()).append("active", true) + .append("firstName", person.getFirstName()).append("age", person.getAge()); + } +} diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java index 97dec780d2..697bf33b95 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java @@ -38,17 +38,11 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.IncorrectResultSizeDataAccessException; -import org.springframework.data.domain.Example; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Range; -import org.springframework.data.domain.Slice; -import org.springframework.data.domain.Sort; +import org.springframework.data.domain.*; +import org.springframework.data.domain.ExampleMatcher.GenericPropertyMatcher; import org.springframework.data.domain.Sort.Direction; import org.springframework.data.geo.Box; import org.springframework.data.geo.Circle; @@ -208,6 +202,29 @@ void findsPagedPersons() { assertThat(result).contains(dave, stefan); } + @Test // GH-4308 + void appliesScrollPositionCorrectly() { + + Scroll page = repository.findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc("*a*", + KeysetScrollPosition.initial()); + + assertThat(page.isLast()).isFalse(); + assertThat(page.size()).isEqualTo(2); + assertThat(page).contains(carter); + } + + @Test // GH-4308 + void appliesScrollPositionWithProjectionCorrectly() { + + Scroll page = repository.findCursorProjectionByLastnameLike("*a*", + PageRequest.of(0, 2, Sort.by(Direction.ASC, "lastname", "firstname"))); + + assertThat(page.isLast()).isFalse(); + assertThat(page.size()).isEqualTo(2); + + assertThat(page).element(0).isEqualTo(new PersonSummaryDto(carter.getFirstname(), carter.getLastname())); + } + @Test void executesPagedFinderCorrectly() { @@ -936,6 +953,21 @@ void shouldFindPersonsWhenUsingQueryDslPerdicatedOnIdProperty() { assertThat(repository.findAll(person.id.in(Arrays.asList(dave.id, carter.id)))).contains(dave, carter); } + @Test // DATAMONGO-969 + void shouldScrollPersonsWhenUsingQueryDslPerdicatedOnIdProperty() { + + Scroll scroll = repository.findBy(person.id.in(asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())); + + assertThat(scroll).containsExactly(boyd, carter); + + ScrollPosition resumeFrom = scroll.lastPosition(); + scroll = repository.findBy(person.id.in(asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(resumeFrom)); + + assertThat(scroll).containsOnly(dave); + } + @Test // DATAMONGO-1030 void executesSingleEntityQueryWithProjectionCorrectly() { @@ -1142,6 +1174,33 @@ void findAllByExampleShouldResolveStuffCorrectly() { assertThat(result).hasSize(2); } + @Test // GH-4308 + void scrollByExampleShouldReturnCorrectResult() { + + Person sample = new Person(); + sample.setLastname("M"); + + // needed to tweak stuff a bit since some field are automatically set - so we need to undo this + ReflectionTestUtils.setField(sample, "id", null); + ReflectionTestUtils.setField(sample, "createdAt", null); + ReflectionTestUtils.setField(sample, "email", null); + + Scroll result = repository.findBy( + Example.of(sample, ExampleMatcher.matching().withMatcher("lastname", GenericPropertyMatcher::startsWith)), + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())); + + assertThat(result).containsOnly(dave, leroi); + assertThat(result.hasNext()).isTrue(); + + ScrollPosition position = result.lastPosition(); + result = repository.findBy( + Example.of(sample, ExampleMatcher.matching().withMatcher("lastname", GenericPropertyMatcher::startsWith)), + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(position)); + + assertThat(result).containsOnly(oliver); + assertThat(result.hasNext()).isFalse(); + } + @Test // DATAMONGO-1425 void findsPersonsByFirstnameNotContains() { diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java index d668bde4c7..b34b8cecde 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java @@ -23,9 +23,11 @@ import java.util.regex.Pattern; import java.util.stream.Stream; +import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Range; +import org.springframework.data.domain.Scroll; import org.springframework.data.domain.Slice; import org.springframework.data.domain.Sort; import org.springframework.data.geo.Box; @@ -115,7 +117,27 @@ public interface PersonRepository extends MongoRepository, Query List findByAgeLessThan(int age, Sort sort); /** - * Returns a page of {@link Person}s with a lastname mathing the given one (*-wildcards supported). + * Returns a scroll of {@link Person}s with a lastname matching the given one (*-wildcards supported). + * + * @param lastname + * @param scrollPosition + * @return + */ + Scroll findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc(String lastname, + KeysetScrollPosition scrollPosition); + + /** + * Returns a scroll of {@link Person}s applying projections with a lastname matching the given one (*-wildcards + * supported). + * + * @param lastname + * @param pageable + * @return + */ + Scroll findCursorProjectionByLastnameLike(String lastname, Pageable pageable); + + /** + * Returns a page of {@link Person}s with a lastname matching the given one (*-wildcards supported). * * @param lastname * @param pageable @@ -429,7 +451,7 @@ Person findPersonByManyArguments(String firstname, String lastname, String email @Update("{ '$inc' : { 'visits' : ?1 } }") int updateAllByLastname(String lastname, int increment); - @Update( pipeline = {"{ '$set' : { 'visits' : { '$add' : [ '$visits', ?1 ] } } }"}) + @Update(pipeline = { "{ '$set' : { 'visits' : { '$add' : [ '$visits', ?1 ] } } }" }) void findAndIncrementVisitsViaPipelineByLastname(String lastname, int increment); @Update("{ '$inc' : { 'visits' : ?#{[1]} } }") diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java index 3b5d7b861d..a17ba71a29 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonSummaryDto.java @@ -15,9 +15,18 @@ */ package org.springframework.data.mongodb.repository; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; + /** * @author Oliver Gierke */ +@EqualsAndHashCode +@AllArgsConstructor +@NoArgsConstructor +@ToString public class PersonSummaryDto { String firstname; diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java index 2a8ea15423..a4b83c33bf 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java @@ -29,24 +29,29 @@ import reactor.core.publisher.Mono; import reactor.test.StepVerifier; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.reactivestreams.Publisher; - import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.dao.IncorrectResultSizeDataAccessException; +import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Direction; import org.springframework.data.geo.Circle; @@ -285,6 +290,36 @@ void shouldUseTailableCursorWithDtoProjection() { cappedRepository.findDtoProjectionByKey("value").as(StepVerifier::create).expectNextCount(1).thenCancel().verify(); } + @Test // GH-4308 + void appliesScrollingCorrectly() { + + Scroll scroll = repository + .findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc("*", KeysetScrollPosition.initial()).block(); + + assertThat(scroll).hasSize(2); + assertThat(scroll).containsSequence(alicia, boyd); + assertThat(scroll.isLast()).isFalse(); + + Scroll nextScroll = repository + .findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc("*", scroll.lastPosition()).block(); + + assertThat(nextScroll).hasSize(2); + assertThat(nextScroll).containsSequence(carter, dave); + assertThat(nextScroll.isLast()).isFalse(); + } + + @Test // GH-4308 + void appliesScrollingWithProjectionCorrectly() { + + repository + .findCursorProjectionByLastnameLike("*", PageRequest.of(0, 2, Sort.by(Direction.ASC, "firstname", "lastname"))) // + .flatMapIterable(Function.identity()) // + .as(StepVerifier::create) // + .expectNext(new PersonSummaryDto(alicia.getFirstname(), alicia.getLastname())) // + .expectNextCount(1) // + .verifyComplete(); + } + @Test // DATAMONGO-1444 @DirtiesState void findsPeopleByLocationWithinCircle() { @@ -436,6 +471,27 @@ void shouldFindPersonsWhenUsingQueryDslPerdicatedOnIdProperty() { }).verifyComplete(); } + @Test // GH-4308 + void shouldScrollWithId() { + + List> capture = new ArrayList<>(); + repository.findBy(person.id.in(Arrays.asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())) // + .as(StepVerifier::create) // + .recordWith(() -> capture).assertNext(actual -> { + assertThat(actual).hasSize(2).containsExactly(boyd, carter); + }).verifyComplete(); + + Scroll scroll = capture.get(0); + + repository.findBy(person.id.in(Arrays.asList(dave.id, carter.id, boyd.id)), // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(scroll.lastPosition())) // + .as(StepVerifier::create) // + .recordWith(() -> capture).assertNext(actual -> { + assertThat(actual).containsOnly(dave); + }).verifyComplete(); + } + @Test // DATAMONGO-2153 void findListOfSingleValue() { @@ -712,6 +768,11 @@ interface ReactivePersonRepository @Query("{ lastname: { $in: ?0 }, age: { $gt : ?1 } }") Flux findStringQuery(Flux lastname, Mono age); + Mono> findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc(String lastname, + ScrollPosition scrollPosition); + + Mono> findCursorProjectionByLastnameLike(String lastname, Pageable pageable); + Flux findByLocationWithin(Circle circle); Flux findByLocationWithin(Circle circle, Pageable pageable); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java index 400351b1bd..134d3ed5b9 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/query/StubParameterAccessor.java @@ -21,6 +21,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Range; import org.springframework.data.domain.Range.Bound; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.geo.Distance; import org.springframework.data.geo.Point; @@ -72,6 +73,11 @@ public StubParameterAccessor(Object... values) { } } + @Override + public ScrollPosition getScrollPosition() { + return null; + } + public Pageable getPageable() { return null; } diff --git a/src/main/asciidoc/index.adoc b/src/main/asciidoc/index.adoc index 93b7896d88..b99bb0bb4f 100644 --- a/src/main/asciidoc/index.adoc +++ b/src/main/asciidoc/index.adoc @@ -6,6 +6,8 @@ ifdef::backend-epub3[:front-cover-image: image:epub-cover.png[Front Cover,1050,1 :spring-data-commons-docs: ../../../../spring-data-commons/src/main/asciidoc :store: Mongo +:feature-scroll: true + (C) 2008-2022 The original authors. NOTE: Copies of this document may be made for your own use and for distribution to others, provided that you do not charge any fee for such copies and further provided that each copy contains this Copyright Notice, whether distributed in print or electronically. diff --git a/src/main/asciidoc/reference/mongodb.adoc b/src/main/asciidoc/reference/mongodb.adoc index 45f54b6cab..101cf2758b 100644 --- a/src/main/asciidoc/reference/mongodb.adoc +++ b/src/main/asciidoc/reference/mongodb.adoc @@ -1289,6 +1289,7 @@ The `Query` class has some additional methods that provide options for the query * `Query` *limit* `(int limit)` used to limit the size of the returned results to the provided limit (used for paging) * `Query` *skip* `(int skip)` used to skip the provided number of documents in the results (used for paging) * `Query` *with* `(Sort sort)` used to provide sort definition for the results +* `Query` *with* `(ScrollPosition position)` used to provide a scroll position (Offset- or Keyset-based pagination) to start or resume a `Scroll` [[mongo-template.querying.field-selection]] ==== Selecting fields From 68c444be4fbe8082e4010e6cb664205346fc7d35 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Fri, 10 Mar 2023 09:50:14 +0100 Subject: [PATCH 3/7] Add support for keyset extraction of nested property paths. Closes #4326 --- .../data/mongodb/core/EntityOperations.java | 79 ++++++++++++++++--- .../core/EntityOperationsUnitTests.java | 72 ++++++++++++++++- .../core/MongoTemplateScrollTests.java | 67 +++++++++++++++- 3 files changed, 203 insertions(+), 15 deletions(-) diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java index 16d084a677..a8e0d8e170 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java @@ -124,7 +124,7 @@ Entity forEntity(T entity) { return new SimpleMappedEntity((Map) entity); } - return MappedEntity.of(entity, context); + return MappedEntity.of(entity, context, this); } /** @@ -148,7 +148,7 @@ AdaptibleEntity forEntity(T entity, ConversionService conversionService) return new SimpleMappedEntity((Map) entity); } - return AdaptibleMappedEntity.of(entity, context, conversionService); + return AdaptibleMappedEntity.of(entity, context, conversionService, this); } /** @@ -386,6 +386,16 @@ interface Entity { */ Object getId(); + /** + * Returns the property value for {@code key}. + * + * @param key + * @return + * @since 4.1 + */ + @Nullable + Object getPropertyValue(String key); + /** * Returns the {@link Query} to find the entity by its identifier. * @@ -457,6 +467,11 @@ default boolean isVersionedEntity() { */ boolean isNew(); + /** + * @param sortObject + * @return + * @since 3.1 + */ Map extractKeys(Document sortObject); } @@ -518,7 +533,12 @@ public String getIdFieldName() { @Override public Object getId() { - return map.get(ID_FIELD); + return getPropertyValue(ID_FIELD); + } + + @Override + public Object getPropertyValue(String key) { + return map.get(key); } @Override @@ -613,23 +633,26 @@ private static class MappedEntity implements Entity { private final MongoPersistentEntity entity; private final IdentifierAccessor idAccessor; private final PersistentPropertyAccessor propertyAccessor; + private final EntityOperations entityOperations; protected MappedEntity(MongoPersistentEntity entity, IdentifierAccessor idAccessor, - PersistentPropertyAccessor propertyAccessor) { + PersistentPropertyAccessor propertyAccessor, EntityOperations entityOperations) { this.entity = entity; this.idAccessor = idAccessor; this.propertyAccessor = propertyAccessor; + this.entityOperations = entityOperations; } private static MappedEntity of(T bean, - MappingContext, MongoPersistentProperty> context) { + MappingContext, MongoPersistentProperty> context, + EntityOperations entityOperations) { MongoPersistentEntity entity = context.getRequiredPersistentEntity(bean.getClass()); IdentifierAccessor identifierAccessor = entity.getIdentifierAccessor(bean); PersistentPropertyAccessor propertyAccessor = entity.getPropertyAccessor(bean); - return new MappedEntity<>(entity, identifierAccessor, propertyAccessor); + return new MappedEntity<>(entity, identifierAccessor, propertyAccessor, entityOperations); } @Override @@ -642,6 +665,11 @@ public Object getId() { return idAccessor.getRequiredIdentifier(); } + @Override + public Object getPropertyValue(String key) { + return propertyAccessor.getProperty(entity.getRequiredPersistentProperty(key)); + } + @Override public Query getByIdQuery() { @@ -728,13 +756,38 @@ public Map extractKeys(Document sortObject) { for (String key : sortObject.keySet()) { - // TODO: make this work for nested properties - MongoPersistentProperty persistentProperty = entity.getRequiredPersistentProperty(key); - keyset.put(key, propertyAccessor.getProperty(persistentProperty)); + if (key.indexOf('.') != -1) { + + // follow the path across nested levels. + // TODO: We should have a MongoDB-specific property path abstraction to allow diving into Document. + keyset.put(key, getNestedPropertyValue(key)); + } else { + keyset.put(key, getPropertyValue(key)); + } } return keyset; } + + @Nullable + private Object getNestedPropertyValue(String key) { + + String[] segments = key.split("\\."); + Entity currentEntity = this; + Object currentValue = null; + + for (int i = 0; i < segments.length; i++) { + + String segment = segments[i]; + currentValue = currentEntity.getPropertyValue(segment); + + if (i < segments.length - 1) { + currentEntity = entityOperations.forEntity(currentValue); + } + } + + return currentValue; + } } private static class AdaptibleMappedEntity extends MappedEntity implements AdaptibleEntity { @@ -744,9 +797,9 @@ private static class AdaptibleMappedEntity extends MappedEntity implements private final IdentifierAccessor identifierAccessor; private AdaptibleMappedEntity(MongoPersistentEntity entity, IdentifierAccessor identifierAccessor, - ConvertingPropertyAccessor propertyAccessor) { + ConvertingPropertyAccessor propertyAccessor, EntityOperations entityOperations) { - super(entity, identifierAccessor, propertyAccessor); + super(entity, identifierAccessor, propertyAccessor, entityOperations); this.entity = entity; this.propertyAccessor = propertyAccessor; @@ -755,14 +808,14 @@ private AdaptibleMappedEntity(MongoPersistentEntity entity, IdentifierAccesso private static AdaptibleEntity of(T bean, MappingContext, MongoPersistentProperty> context, - ConversionService conversionService) { + ConversionService conversionService, EntityOperations entityOperations) { MongoPersistentEntity entity = context.getRequiredPersistentEntity(bean.getClass()); IdentifierAccessor identifierAccessor = entity.getIdentifierAccessor(bean); PersistentPropertyAccessor propertyAccessor = entity.getPropertyAccessor(bean); return new AdaptibleMappedEntity<>(entity, identifierAccessor, - new ConvertingPropertyAccessor<>(propertyAccessor, conversionService)); + new ConvertingPropertyAccessor<>(propertyAccessor, conversionService), entityOperations); } @Nullable diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java index 74e170312c..6107abaa30 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java @@ -17,10 +17,14 @@ import static org.assertj.core.api.Assertions.*; +import lombok.AllArgsConstructor; +import lombok.NoArgsConstructor; + import java.time.Instant; +import java.util.Map; +import org.bson.Document; import org.junit.jupiter.api.Test; - import org.springframework.core.convert.ConversionService; import org.springframework.core.convert.support.DefaultConversionService; import org.springframework.data.annotation.Id; @@ -61,6 +65,57 @@ void populateIdShouldReturnTargetBeanWhenIdIsNull() { assertThat(initAdaptibleEntity(new DomainTypeWithIdProperty()).populateIdIfNecessary(null)).isNotNull(); } + @Test // GH-4308 + void shouldExtractKeysFromEntity() { + + WithNestedDocument object = new WithNestedDocument("foo"); + + Map keys = operations.forEntity(object).extractKeys(new Document("id", 1)); + + assertThat(keys).containsEntry("id", "foo"); + } + + @Test // GH-4308 + void shouldExtractKeysFromDocument() { + + Document object = new Document("id", "foo"); + + Map keys = operations.forEntity(object).extractKeys(new Document("id", 1)); + + assertThat(keys).containsEntry("id", "foo"); + } + + @Test // GH-4308 + void shouldExtractKeysFromNestedEntity() { + + WithNestedDocument object = new WithNestedDocument("foo", new WithNestedDocument("bar"), null); + + Map keys = operations.forEntity(object).extractKeys(new Document("nested.id", 1)); + + assertThat(keys).containsEntry("nested.id", "bar"); + } + + @Test // GH-4308 + void shouldExtractKeysFromNestedEntityDocument() { + + WithNestedDocument object = new WithNestedDocument("foo", new WithNestedDocument("bar"), + new Document("john", "doe")); + + Map keys = operations.forEntity(object).extractKeys(new Document("document.john", 1)); + + assertThat(keys).containsEntry("document.john", "doe"); + } + + @Test // GH-4308 + void shouldExtractKeysFromNestedDocument() { + + Document object = new Document("document", new Document("john", "doe")); + + Map keys = operations.forEntity(object).extractKeys(new Document("document.john", 1)); + + assertThat(keys).containsEntry("document.john", "doe"); + } + EntityOperations.AdaptibleEntity initAdaptibleEntity(T source) { return operations.forEntity(source, conversionService); } @@ -80,4 +135,19 @@ static class InvalidTimeField { static class InvalidMetaField { Instant time; } + + @AllArgsConstructor + @NoArgsConstructor + class WithNestedDocument { + + String id; + + WithNestedDocument nested; + + Document document; + + public WithNestedDocument(String id) { + this.id = id; + } + } } diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java index 865f54f202..ef13509b23 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java @@ -18,18 +18,23 @@ import static org.springframework.data.mongodb.core.query.Criteria.*; import static org.springframework.data.mongodb.test.util.Assertions.*; +import lombok.Data; +import lombok.NoArgsConstructor; + import java.util.Arrays; import java.util.function.Function; import java.util.stream.Stream; import org.bson.Document; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.support.GenericApplicationContext; +import org.springframework.data.annotation.PersistenceCreator; import org.springframework.data.auditing.IsNewAwareAuditingHandler; import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.OffsetScrollPosition; @@ -69,7 +74,6 @@ class MongoTemplateScrollTests { cfg.configureMappingContext(it -> { it.autocreateIndex(false); - it.initialEntitySet(AuditablePerson.class); }); cfg.configureApplicationContext(it -> { @@ -87,6 +91,39 @@ class MongoTemplateScrollTests { @BeforeEach void setUp() { template.remove(Person.class).all(); + template.remove(WithNestedDocument.class).all(); + } + + @Test + void shouldUseKeysetScrollingWithNestedSort() { + + WithNestedDocument john20 = new WithNestedDocument(null, "John", 120, new WithNestedDocument("John", 20), + new Document("name", "bar")); + WithNestedDocument john40 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 40), + new Document("name", "baz")); + WithNestedDocument john41 = new WithNestedDocument(null, "John", 141, new WithNestedDocument("John", 41), + new Document("name", "foo")); + + template.insertAll(Arrays.asList(john20, john40, john41)); + + Query q = new Query(where("name").regex("J.*")).with(Sort.by("nested.name", "nested.age", "document.name")) + .limit(2); + q.with(KeysetScrollPosition.initial()); + + Scroll scroll = template.scroll(q, WithNestedDocument.class); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(2); + assertThat(scroll).containsOnly(john20, john40); + + scroll = template.scroll(q.with(scroll.lastPosition()), WithNestedDocument.class); + + assertThat(scroll.hasNext()).isFalse(); + assertThat(scroll.isLast()).isTrue(); + assertThat(scroll).hasSize(1); + assertThat(scroll).containsOnly(john41); + } @ParameterizedTest // GH-4308 @@ -144,4 +181,32 @@ static Document toDocument(Person person) { return new Document("_class", person.getClass().getName()).append("_id", person.getId()).append("active", true) .append("firstName", person.getFirstName()).append("age", person.getAge()); } + + @NoArgsConstructor + @Data + class WithNestedDocument { + + String id; + String name; + + int age; + + WithNestedDocument nested; + + Document document; + + public WithNestedDocument(String name, int age) { + this.name = name; + this.age = age; + } + + @PersistenceCreator + public WithNestedDocument(String id, String name, int age, WithNestedDocument nested, Document document) { + this.id = id; + this.name = name; + this.age = age; + this.nested = nested; + this.document = document; + } + } } From 1b85cfe3ab39d9805cdacbcda958a9061d0fc0c3 Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Wed, 15 Mar 2023 16:56:41 +0100 Subject: [PATCH 4/7] Prevent key extraction if a keyset value is null. Follow the changes in data commons that renamed scroll to window. Also error when a certain scroll position does not allow creating a query out of it because of null values. --- .../data/mongodb/core/EntityOperations.java | 27 ++++++++-- .../mongodb/core/ExecutableFindOperation.java | 9 ++-- .../core/ExecutableFindOperationSupport.java | 4 +- .../data/mongodb/core/MongoOperations.java | 20 +++---- .../data/mongodb/core/MongoTemplate.java | 12 ++--- .../mongodb/core/ReactiveFindOperation.java | 4 +- .../core/ReactiveFindOperationSupport.java | 4 +- .../mongodb/core/ReactiveMongoOperations.java | 14 ++--- .../mongodb/core/ReactiveMongoTemplate.java | 12 ++--- .../data/mongodb/core/ScrollUtils.java | 21 +++++--- .../QuerydslMongoPredicateExecutor.java | 4 +- ...eactiveQuerydslMongoPredicateExecutor.java | 4 +- .../ReactiveSpringDataMongodbQuery.java | 4 +- .../support/SimpleMongoRepository.java | 4 +- .../SimpleReactiveMongoRepository.java | 4 +- .../support/SpringDataMongodbQuery.java | 6 +-- .../core/MongoTemplateScrollTests.java | 52 ++++++++++++++++--- .../ReactiveMongoTemplateScrollTests.java | 10 ++-- ...tractPersonRepositoryIntegrationTests.java | 12 ++--- .../mongodb/repository/PersonRepository.java | 10 ++-- .../ReactiveMongoRepositoryTests.java | 18 +++---- 21 files changed, 159 insertions(+), 96 deletions(-) diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java index a8e0d8e170..cbf252f1dc 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Optional; +import org.bson.BsonNull; import org.bson.Document; import org.springframework.core.convert.ConversionService; import org.springframework.dao.InvalidDataAccessApiUsageException; @@ -471,6 +472,7 @@ default boolean isVersionedEntity() { * @param sortObject * @return * @since 3.1 + * @throws IllegalStateException if a sort key yields {@literal null}. */ Map extractKeys(Document sortObject); @@ -600,7 +602,14 @@ public Map extractKeys(Document sortObject) { keyset.put(ID_FIELD, getId()); for (String key : sortObject.keySet()) { - keyset.put(key, BsonUtils.resolveValue(map, key)); + Object value = BsonUtils.resolveValue(map, key); + + if (value == null) { + throw new IllegalStateException( + String.format("Cannot extract value for key %s because its value is null", key)); + } + + keyset.put(key, value); } return keyset; @@ -756,14 +765,22 @@ public Map extractKeys(Document sortObject) { for (String key : sortObject.keySet()) { + Object value; if (key.indexOf('.') != -1) { // follow the path across nested levels. // TODO: We should have a MongoDB-specific property path abstraction to allow diving into Document. - keyset.put(key, getNestedPropertyValue(key)); + value = getNestedPropertyValue(key); } else { - keyset.put(key, getPropertyValue(key)); + value = getPropertyValue(key); } + + if (value == null) { + throw new IllegalStateException( + String.format("Cannot extract value for key %s because its value is null", key)); + } + + keyset.put(key, value); } return keyset; @@ -774,7 +791,7 @@ private Object getNestedPropertyValue(String key) { String[] segments = key.split("\\."); Entity currentEntity = this; - Object currentValue = null; + Object currentValue = BsonNull.VALUE; for (int i = 0; i < segments.length; i++) { @@ -786,7 +803,7 @@ private Object getNestedPropertyValue(String key) { } } - return currentValue; + return currentValue != null ? currentValue : BsonNull.VALUE; } } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java index e8b38c8e81..5ae2010937 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java @@ -20,7 +20,7 @@ import java.util.stream.Stream; import org.springframework.dao.DataAccessException; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.geo.GeoResults; import org.springframework.data.mongodb.core.query.CriteriaDefinition; @@ -126,16 +126,17 @@ default Optional first() { Stream stream(); /** - * Return a scroll of elements either starting or resuming at + * Return a window of elements either starting or resuming at * {@link org.springframework.data.domain.ScrollPosition}. * * @param scrollPosition the scroll position. - * @return a scroll of the resulting elements. + * @return a window of the resulting elements. + * @throws IllegalStateException if a potential {@literal KeysetScrollPosition} contains an invalid position. * @since 4.1 * @see org.springframework.data.domain.OffsetScrollPosition * @see org.springframework.data.domain.KeysetScrollPosition */ - Scroll scroll(ScrollPosition scrollPosition); + Window scroll(ScrollPosition scrollPosition); /** * Get the number of matching elements.
diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java index 81d7557e78..d99cffbe37 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperationSupport.java @@ -21,7 +21,7 @@ import org.bson.Document; import org.springframework.dao.IncorrectResultSizeDataAccessException; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.query.NearQuery; import org.springframework.data.mongodb.core.query.Query; @@ -141,7 +141,7 @@ public Stream stream() { } @Override - public Scroll scroll(ScrollPosition scrollPosition) { + public Window scroll(ScrollPosition scrollPosition) { return template.doScroll(query.with(scrollPosition), domainType, returnType, getCollectionName()); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java index cd019aae6d..2d51a56c15 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java @@ -24,7 +24,7 @@ import org.bson.Document; import org.springframework.data.domain.KeysetScrollPosition; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.geo.GeoResults; import org.springframework.data.mongodb.core.BulkOperations.BulkMode; import org.springframework.data.mongodb.core.aggregation.Aggregation; @@ -807,7 +807,7 @@ MapReduceResults mapReduce(Query query, String inputCollectionName, Strin List find(Query query, Class entityClass, String collectionName); /** - * Query for a scroll window of objects of type T from the specified collection.
+ * Query for a window window of objects of type T from the specified collection.
* Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with * {@link Query#limit(int)} to limit large query results for efficient scrolling.
* Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. @@ -817,16 +817,17 @@ MapReduceResults mapReduce(Query query, String inputCollectionName, Strin * * @param query the query class that specifies the criteria used to find a record and also an optional fields * specification. Must not be {@literal null}. - * @param entityType the parametrized type of the returned list. - * @return the converted scroll. + * @param entityType the parametrized type of the returned window. + * @return the converted window. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) */ - Scroll scroll(Query query, Class entityType); + Window scroll(Query query, Class entityType); /** - * Query for a scroll of objects of type T from the specified collection.
+ * Query for a window of objects of type T from the specified collection.
* Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with * {@link Query#limit(int)} to limit large query results for efficient scrolling.
* Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. @@ -836,14 +837,15 @@ MapReduceResults mapReduce(Query query, String inputCollectionName, Strin * * @param query the query class that specifies the criteria used to find a record and also an optional fields * specification. Must not be {@literal null}. - * @param entityType the parametrized type of the returned list. + * @param entityType the parametrized type of the returned window. * @param collectionName name of the collection to retrieve the objects from. - * @return the converted scroll. + * @return the converted window. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) */ - Scroll scroll(Query query, Class entityType, String collectionName); + Window scroll(Query query, Class entityType, String collectionName); /** * Returns a document with the given id mapped onto the given class. The collection the query is ran against will be diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java index f3f951da97..c86c61cb69 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java @@ -45,7 +45,7 @@ import org.springframework.dao.support.PersistenceExceptionTranslator; import org.springframework.data.convert.EntityReader; import org.springframework.data.domain.OffsetScrollPosition; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.geo.Distance; import org.springframework.data.geo.GeoResult; import org.springframework.data.geo.GeoResults; @@ -66,7 +66,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; -import org.springframework.data.mongodb.core.ScrollUtils.KeySetCursorQuery; +import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -851,7 +851,7 @@ public List find(Query query, Class entityClass, String collectionName } @Override - public Scroll scroll(Query query, Class entityType) { + public Window scroll(Query query, Class entityType) { Assert.notNull(entityType, "Entity type must not be null"); @@ -859,11 +859,11 @@ public Scroll scroll(Query query, Class entityType) { } @Override - public Scroll scroll(Query query, Class entityType, String collectionName) { + public Window scroll(Query query, Class entityType, String collectionName) { return doScroll(query, entityType, entityType, collectionName); } - Scroll doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { + Window doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { Assert.notNull(query, "Query must not be null"); Assert.notNull(collectionName, "CollectionName must not be null"); @@ -875,7 +875,7 @@ Scroll doScroll(Query query, Class sourceClass, Class targetClass, if (query.hasKeyset()) { - KeySetCursorQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, operations.getIdPropertyName(sourceClass)); List result = doFind(collectionName, createDelegate(query), keysetPaginationQuery.query(), diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java index 1a81f92a9a..3786cdac03 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java @@ -18,7 +18,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.geo.GeoResult; import org.springframework.data.mongodb.core.query.CriteriaDefinition; @@ -98,7 +98,7 @@ interface TerminatingFind { * @see org.springframework.data.domain.OffsetScrollPosition * @see org.springframework.data.domain.KeysetScrollPosition */ - Mono> scroll(ScrollPosition scrollPosition); + Mono> scroll(ScrollPosition scrollPosition); /** * Get all matching elements using a {@link com.mongodb.CursorType#TailableAwait tailable cursor}. The stream will diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java index 13894c896f..30b8ab0921 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperationSupport.java @@ -20,7 +20,7 @@ import org.bson.Document; import org.springframework.dao.IncorrectResultSizeDataAccessException; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.CollectionPreparerSupport.ReactiveCollectionPreparerDelegate; import org.springframework.data.mongodb.core.query.NearQuery; @@ -140,7 +140,7 @@ public Flux all() { } @Override - public Mono> scroll(ScrollPosition scrollPosition) { + public Mono> scroll(ScrollPosition scrollPosition) { return template.doScroll(query.with(scrollPosition), domainType, returnType, getCollectionName()); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java index d252cb4a40..8030fb8a9d 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java @@ -26,7 +26,7 @@ import org.reactivestreams.Publisher; import org.reactivestreams.Subscription; import org.springframework.data.domain.KeysetScrollPosition; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.geo.GeoResult; import org.springframework.data.mongodb.ReactiveMongoDatabaseFactory; import org.springframework.data.mongodb.core.aggregation.Aggregation; @@ -477,15 +477,16 @@ Mono> createView(String name, String source, Aggregati * @param query the query class that specifies the criteria used to find a record and also an optional fields * specification. Must not be {@literal null}. * @param entityType the parametrized type of the returned list. - * @return {@link Mono} emitting the converted scroll. + * @return {@link Mono} emitting the converted window. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) */ - Mono> scroll(Query query, Class entityType); + Mono> scroll(Query query, Class entityType); /** - * Query for a scroll of objects of type T from the specified collection.
+ * Query for a window of objects of type T from the specified collection.
* Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with * {@link Query#limit(int)} to limit large query results for efficient scrolling.
* Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. @@ -497,12 +498,13 @@ Mono> createView(String name, String source, Aggregati * specification. Must not be {@literal null}. * @param entityType the parametrized type of the returned list. * @param collectionName name of the collection to retrieve the objects from. - * @return {@link Mono} emitting the converted scroll window. + * @return {@link Mono} emitting the converted window. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) */ - Mono> scroll(Query query, Class entityType, String collectionName); + Mono> scroll(Query query, Class entityType, String collectionName); /** * Returns a document with the given id mapped onto the given class. The collection the query is ran against will be diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java index 55f924e6e9..6bf38da1c9 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java @@ -59,7 +59,7 @@ import org.springframework.dao.support.PersistenceExceptionTranslator; import org.springframework.data.convert.EntityReader; import org.springframework.data.domain.OffsetScrollPosition; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.geo.Distance; import org.springframework.data.geo.GeoResult; import org.springframework.data.geo.Metric; @@ -80,7 +80,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext; import org.springframework.data.mongodb.core.QueryOperations.QueryContext; import org.springframework.data.mongodb.core.QueryOperations.UpdateContext; -import org.springframework.data.mongodb.core.ScrollUtils.KeySetCursorQuery; +import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery; import org.springframework.data.mongodb.core.aggregation.Aggregation; import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext; import org.springframework.data.mongodb.core.aggregation.AggregationOptions; @@ -830,7 +830,7 @@ public Flux find(@Nullable Query query, Class entityClass, String coll } @Override - public Mono> scroll(Query query, Class entityType) { + public Mono> scroll(Query query, Class entityType) { Assert.notNull(entityType, "Entity type must not be null"); @@ -838,11 +838,11 @@ public Mono> scroll(Query query, Class entityType) { } @Override - public Mono> scroll(Query query, Class entityType, String collectionName) { + public Mono> scroll(Query query, Class entityType, String collectionName) { return doScroll(query, entityType, entityType, collectionName); } - Mono> doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { + Mono> doScroll(Query query, Class sourceClass, Class targetClass, String collectionName) { Assert.notNull(query, "Query must not be null"); Assert.notNull(collectionName, "CollectionName must not be null"); @@ -853,7 +853,7 @@ Mono> doScroll(Query query, Class sourceClass, Class targetC if (query.hasKeyset()) { - KeySetCursorQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, + KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query, operations.getIdPropertyName(sourceClass)); Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java index 6458fa9d2a..320bafa4df 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java @@ -20,17 +20,19 @@ import java.util.Map; import java.util.function.IntFunction; +import org.bson.BsonNull; import org.bson.Document; import org.springframework.data.domain.KeysetScrollPosition; -import org.springframework.data.domain.Scroll; import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Window; import org.springframework.data.mongodb.core.EntityOperations.Entity; import org.springframework.data.mongodb.core.query.Query; /** - * Utilities to run scroll queries and create {@link Scroll} results. + * Utilities to run scroll queries and create {@link Window} results. * * @author Mark Paluch + * @author Christoph Strobl * @since 4.1 */ class ScrollUtils { @@ -42,7 +44,7 @@ class ScrollUtils { * @param idPropertyName * @return */ - static KeySetCursorQuery createKeysetPaginationQuery(Query query, String idPropertyName) { + static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) { Document sortObject = query.isSorted() ? query.getSortObject() : new Document(); sortObject.put(idPropertyName, 1); @@ -84,6 +86,9 @@ static KeySetCursorQuery createKeysetPaginationQuery(Query query, String idPrope Object o = keysetValues.get(sortSegment); if (j >= i) { // tail segment + if(o instanceof BsonNull) { + throw new IllegalStateException("Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment)); + } sortConstraint.put(sortSegment, new Document(sortOrder == 1 ? "$gt" : "$lt", o)); break; } @@ -104,10 +109,10 @@ static KeySetCursorQuery createKeysetPaginationQuery(Query query, String idPrope queryObject.put("$or", or); } - return new KeySetCursorQuery(queryObject, fieldsObject, sortObject); + return new KeySetScrollQuery(queryObject, fieldsObject, sortObject); } - static Scroll createWindow(Document sortObject, int limit, List result, EntityOperations operations) { + static Window createWindow(Document sortObject, int limit, List result, EntityOperations operations) { IntFunction positionFunction = value -> { @@ -121,8 +126,8 @@ static Scroll createWindow(Document sortObject, int limit, List result return createWindow(result, limit, positionFunction); } - static Scroll createWindow(List result, int limit, IntFunction positionFunction) { - return Scroll.from(getSubList(result, limit), positionFunction, hasMoreElements(result, limit)); + static Window createWindow(List result, int limit, IntFunction positionFunction) { + return Window.from(getSubList(result, limit), positionFunction, hasMoreElements(result, limit)); } static boolean hasMoreElements(List result, int limit) { @@ -138,7 +143,7 @@ static List getSubList(List result, int limit) { return result; } - record KeySetCursorQuery(Document query, Document fields, Document sort) { + record KeySetScrollQuery(Document query, Document fields, Document sort) { } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java index 4cf3ab3fe2..f95092adb8 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/QuerydslMongoPredicateExecutor.java @@ -25,7 +25,7 @@ import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoOperations; @@ -259,7 +259,7 @@ public List all() { } @Override - public Scroll scroll(ScrollPosition scrollPosition) { + public Window scroll(ScrollPosition scrollPosition) { return createQuery().scroll(scrollPosition); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java index ff2b442931..d2269fe69a 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveQuerydslMongoPredicateExecutor.java @@ -26,7 +26,7 @@ import org.reactivestreams.Publisher; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ReactiveMongoOperations; @@ -227,7 +227,7 @@ public Flux all() { } @Override - public Mono> scroll(ScrollPosition scrollPosition) { + public Mono> scroll(ScrollPosition scrollPosition) { return createQuery().scroll(scrollPosition); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java index cc27f81d68..e293414255 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/ReactiveSpringDataMongodbQuery.java @@ -26,7 +26,7 @@ import org.bson.Document; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.data.mongodb.core.ReactiveFindOperation; @@ -91,7 +91,7 @@ Flux fetch() { return createQuery().flatMapMany(it -> find.matching(it).all()); } - Mono> scroll(ScrollPosition scrollPosition) { + Mono> scroll(ScrollPosition scrollPosition) { return createQuery().flatMap(it -> find.matching(it).scroll(scrollPosition)); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java index 750be26aeb..6577a23e44 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleMongoRepository.java @@ -32,7 +32,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ExecutableFindOperation; @@ -392,7 +392,7 @@ public List all() { } @Override - public Scroll scroll(ScrollPosition scrollPosition) { + public Window scroll(ScrollPosition scrollPosition) { return createQuery().scroll(scrollPosition); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java index 9114f53139..bca6b9b793 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SimpleReactiveMongoRepository.java @@ -34,7 +34,7 @@ import org.springframework.data.domain.Example; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.ReactiveFindOperation; @@ -436,7 +436,7 @@ public Flux all() { } @Override - public Mono> scroll(ScrollPosition scrollPosition) { + public Mono> scroll(ScrollPosition scrollPosition) { return createQuery().scroll(scrollPosition); } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java index 2c073986d8..0ecff39583 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/repository/support/SpringDataMongodbQuery.java @@ -25,7 +25,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.mongodb.core.ExecutableFindOperation; import org.springframework.data.mongodb.core.MongoOperations; @@ -133,12 +133,12 @@ public void close() { } } - public Scroll scroll(ScrollPosition scrollPosition) { + public Window scroll(ScrollPosition scrollPosition) { try { return find.matching(createQuery()).scroll(scrollPosition); } catch (RuntimeException e) { - return handleException(e, Scroll.from(Collections.emptyList(), value -> { + return handleException(e, Window.from(Collections.emptyList(), value -> { throw new UnsupportedOperationException(); })); } diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java index ef13509b23..35c9b9d0cd 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java @@ -38,9 +38,9 @@ import org.springframework.data.auditing.IsNewAwareAuditingHandler; import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.OffsetScrollPosition; -import org.springframework.data.domain.Scroll; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; +import org.springframework.data.domain.Window; import org.springframework.data.mapping.context.PersistentEntities; import org.springframework.data.mongodb.core.MongoTemplateTests.PersonWithIdPropertyOfTypeUUIDListener; import org.springframework.data.mongodb.core.query.Query; @@ -51,9 +51,10 @@ import com.mongodb.client.MongoClient; /** - * Integration tests for {@link Scroll} queries. + * Integration tests for {@link org.springframework.data.domain.Window} queries. * * @author Mark Paluch + * @author Christoph Strobl */ @ExtendWith(MongoClientExtension.class) class MongoTemplateScrollTests { @@ -94,7 +95,7 @@ void setUp() { template.remove(WithNestedDocument.class).all(); } - @Test + @Test // GH-4308 void shouldUseKeysetScrollingWithNestedSort() { WithNestedDocument john20 = new WithNestedDocument(null, "John", 120, new WithNestedDocument("John", 20), @@ -110,20 +111,55 @@ void shouldUseKeysetScrollingWithNestedSort() { .limit(2); q.with(KeysetScrollPosition.initial()); - Scroll scroll = template.scroll(q, WithNestedDocument.class); + Window scroll = template.scroll(q, WithNestedDocument.class); assertThat(scroll.hasNext()).isTrue(); assertThat(scroll.isLast()).isFalse(); assertThat(scroll).hasSize(2); assertThat(scroll).containsOnly(john20, john40); - scroll = template.scroll(q.with(scroll.lastPosition()), WithNestedDocument.class); + scroll = template.scroll(q.with(scroll.positionAt(scroll.size()-1)), WithNestedDocument.class); assertThat(scroll.hasNext()).isFalse(); assertThat(scroll.isLast()).isTrue(); assertThat(scroll).hasSize(1); assertThat(scroll).containsOnly(john41); + } + + @Test // GH-4308 + void shouldErrorOnNullValueForQuery() { + + WithNestedDocument john20 = new WithNestedDocument(null, "John", 120, new WithNestedDocument("John", 20), + new Document("name", "bar")); + WithNestedDocument john40 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 41), + new Document()); + WithNestedDocument john41 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 41), + new Document()); + WithNestedDocument john42 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 41), + new Document()); + WithNestedDocument john43 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 41), + new Document()); + WithNestedDocument john44 = new WithNestedDocument(null, "John", 141, new WithNestedDocument("John", 41), + new Document("name", "foo")); + + template.insertAll(Arrays.asList(john20, john40, john41, john42, john43, john44)); + + Query q = new Query(where("name").regex("J.*")).with(Sort.by("nested.name", "nested.age", "document.name")) + .limit(2); + q.with(KeysetScrollPosition.initial()); + + Window scroll = template.scroll(q, WithNestedDocument.class); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(2); + assertThat(scroll).containsOnly(john20, john40); + + ScrollPosition startAfter = scroll.positionAt(scroll.size()-1); + assertThatExceptionOfType(IllegalStateException.class) + .isThrownBy(() -> template.scroll(q.with(startAfter), WithNestedDocument.class)) + .withMessageContaining("document.name"); } @ParameterizedTest // GH-4308 @@ -142,14 +178,14 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); q.with(scrollPosition); - Scroll scroll = template.scroll(q, resultType, "person"); + Window scroll = template.scroll(q, resultType, "person"); assertThat(scroll.hasNext()).isTrue(); assertThat(scroll.isLast()).isFalse(); assertThat(scroll).hasSize(2); assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); - scroll = template.scroll(q.with(scroll.lastPosition()).limit(3), resultType, "person"); + scroll = template.scroll(q.with(scroll.positionAt(scroll.size()-1)).limit(3), resultType, "person"); assertThat(scroll.hasNext()).isTrue(); assertThat(scroll.isLast()).isFalse(); @@ -157,7 +193,7 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); - scroll = template.scroll(q.with(scroll.lastPosition()).limit(1), resultType, "person"); + scroll = template.scroll(q.with(scroll.positionAt(scroll.size()-1)).limit(1), resultType, "person"); assertThat(scroll.hasNext()).isFalse(); assertThat(scroll.isLast()).isTrue(); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java index ede69c396f..d42d8d99f0 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java @@ -35,7 +35,7 @@ import org.springframework.context.support.GenericApplicationContext; import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.OffsetScrollPosition; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.query.Query; @@ -46,7 +46,7 @@ import com.mongodb.reactivestreams.client.MongoClient; /** - * Integration tests for {@link Scroll} queries. + * Integration tests for {@link Window} queries. * * @author Mark Paluch */ @@ -100,14 +100,14 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); q.with(scrollPosition); - Scroll scroll = template.scroll(q, resultType, "person").block(Duration.ofSeconds(10)); + Window scroll = template.scroll(q, resultType, "person").block(Duration.ofSeconds(10)); assertThat(scroll.hasNext()).isTrue(); assertThat(scroll.isLast()).isFalse(); assertThat(scroll).hasSize(2); assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); - scroll = template.scroll(q.limit(3).with(scroll.lastPosition()), resultType, "person") + scroll = template.scroll(q.limit(3).with(scroll.positionAt(scroll.size() - 1)), resultType, "person") .block(Duration.ofSeconds(10)); assertThat(scroll.hasNext()).isTrue(); @@ -116,7 +116,7 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); - scroll = template.scroll(q.limit(1).with(scroll.lastPosition()), resultType, "person") + scroll = template.scroll(q.limit(1).with(scroll.positionAt(scroll.size() - 1)), resultType, "person") .block(Duration.ofSeconds(10)); assertThat(scroll.hasNext()).isFalse(); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java index 697bf33b95..6a1e35d0be 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/AbstractPersonRepositoryIntegrationTests.java @@ -205,7 +205,7 @@ void findsPagedPersons() { @Test // GH-4308 void appliesScrollPositionCorrectly() { - Scroll page = repository.findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc("*a*", + Window page = repository.findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc("*a*", KeysetScrollPosition.initial()); assertThat(page.isLast()).isFalse(); @@ -216,7 +216,7 @@ void appliesScrollPositionCorrectly() { @Test // GH-4308 void appliesScrollPositionWithProjectionCorrectly() { - Scroll page = repository.findCursorProjectionByLastnameLike("*a*", + Window page = repository.findCursorProjectionByLastnameLike("*a*", PageRequest.of(0, 2, Sort.by(Direction.ASC, "lastname", "firstname"))); assertThat(page.isLast()).isFalse(); @@ -956,12 +956,12 @@ void shouldFindPersonsWhenUsingQueryDslPerdicatedOnIdProperty() { @Test // DATAMONGO-969 void shouldScrollPersonsWhenUsingQueryDslPerdicatedOnIdProperty() { - Scroll scroll = repository.findBy(person.id.in(asList(dave.id, carter.id, boyd.id)), // + Window scroll = repository.findBy(person.id.in(asList(dave.id, carter.id, boyd.id)), // q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())); assertThat(scroll).containsExactly(boyd, carter); - ScrollPosition resumeFrom = scroll.lastPosition(); + ScrollPosition resumeFrom = scroll.positionAt(scroll.size() - 1); scroll = repository.findBy(person.id.in(asList(dave.id, carter.id, boyd.id)), // q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(resumeFrom)); @@ -1185,14 +1185,14 @@ void scrollByExampleShouldReturnCorrectResult() { ReflectionTestUtils.setField(sample, "createdAt", null); ReflectionTestUtils.setField(sample, "email", null); - Scroll result = repository.findBy( + Window result = repository.findBy( Example.of(sample, ExampleMatcher.matching().withMatcher("lastname", GenericPropertyMatcher::startsWith)), q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())); assertThat(result).containsOnly(dave, leroi); assertThat(result.hasNext()).isTrue(); - ScrollPosition position = result.lastPosition(); + ScrollPosition position = result.positionAt(result.size() - 1); result = repository.findBy( Example.of(sample, ExampleMatcher.matching().withMatcher("lastname", GenericPropertyMatcher::startsWith)), q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(position)); diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java index b34b8cecde..1e6a37b0a9 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/PersonRepository.java @@ -23,11 +23,11 @@ import java.util.regex.Pattern; import java.util.stream.Stream; -import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Range; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Slice; import org.springframework.data.domain.Sort; import org.springframework.data.geo.Box; @@ -123,8 +123,8 @@ public interface PersonRepository extends MongoRepository, Query * @param scrollPosition * @return */ - Scroll findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc(String lastname, - KeysetScrollPosition scrollPosition); + Window findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc(String lastname, + ScrollPosition scrollPosition); /** * Returns a scroll of {@link Person}s applying projections with a lastname matching the given one (*-wildcards @@ -134,7 +134,7 @@ Scroll findTop2ByLastnameLikeOrderByLastnameAscFirstnameAsc(String lastn * @param pageable * @return */ - Scroll findCursorProjectionByLastnameLike(String lastname, Pageable pageable); + Window findCursorProjectionByLastnameLike(String lastname, Pageable pageable); /** * Returns a page of {@link Person}s with a lastname matching the given one (*-wildcards supported). diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java index a4b83c33bf..5d54eda624 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/repository/ReactiveMongoRepositoryTests.java @@ -50,10 +50,10 @@ import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Direction; +import org.springframework.data.domain.Window; import org.springframework.data.geo.Circle; import org.springframework.data.geo.Distance; import org.springframework.data.geo.GeoResult; @@ -293,15 +293,15 @@ void shouldUseTailableCursorWithDtoProjection() { @Test // GH-4308 void appliesScrollingCorrectly() { - Scroll scroll = repository + Window scroll = repository .findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc("*", KeysetScrollPosition.initial()).block(); assertThat(scroll).hasSize(2); assertThat(scroll).containsSequence(alicia, boyd); assertThat(scroll.isLast()).isFalse(); - Scroll nextScroll = repository - .findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc("*", scroll.lastPosition()).block(); + Window nextScroll = repository + .findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc("*", scroll.positionAt(scroll.size() - 1)).block(); assertThat(nextScroll).hasSize(2); assertThat(nextScroll).containsSequence(carter, dave); @@ -474,7 +474,7 @@ void shouldFindPersonsWhenUsingQueryDslPerdicatedOnIdProperty() { @Test // GH-4308 void shouldScrollWithId() { - List> capture = new ArrayList<>(); + List> capture = new ArrayList<>(); repository.findBy(person.id.in(Arrays.asList(dave.id, carter.id, boyd.id)), // q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(KeysetScrollPosition.initial())) // .as(StepVerifier::create) // @@ -482,10 +482,10 @@ void shouldScrollWithId() { assertThat(actual).hasSize(2).containsExactly(boyd, carter); }).verifyComplete(); - Scroll scroll = capture.get(0); + Window scroll = capture.get(0); repository.findBy(person.id.in(Arrays.asList(dave.id, carter.id, boyd.id)), // - q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(scroll.lastPosition())) // + q -> q.limit(2).sortBy(Sort.by("firstname")).scroll(scroll.positionAt(scroll.size() - 1))) // .as(StepVerifier::create) // .recordWith(() -> capture).assertNext(actual -> { assertThat(actual).containsOnly(dave); @@ -768,10 +768,10 @@ interface ReactivePersonRepository @Query("{ lastname: { $in: ?0 }, age: { $gt : ?1 } }") Flux findStringQuery(Flux lastname, Mono age); - Mono> findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc(String lastname, + Mono> findTop2ByLastnameLikeOrderByFirstnameAscLastnameAsc(String lastname, ScrollPosition scrollPosition); - Mono> findCursorProjectionByLastnameLike(String lastname, Pageable pageable); + Mono> findCursorProjectionByLastnameLike(String lastname, Pageable pageable); Flux findByLocationWithin(Circle circle); From c71cc051594d3189883f42b77d6aa3ccef895b79 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Thu, 16 Mar 2023 09:15:20 +0100 Subject: [PATCH 5/7] Add support for reverse scrolling. Closes #4325 --- .../data/mongodb/core/ScrollUtils.java | 33 +++++++++------ .../core/MongoTemplateScrollTests.java | 41 +++++++++++++++---- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java index 320bafa4df..7c0ae5da99 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java @@ -23,6 +23,7 @@ import org.bson.BsonNull; import org.bson.Document; import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.KeysetScrollPosition.Direction; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Window; import org.springframework.data.mongodb.core.EntityOperations.Entity; @@ -46,6 +47,10 @@ class ScrollUtils { */ static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) { + KeysetScrollPosition keyset = query.getKeyset(); + Map keysetValues = keyset.getKeys(); + Document queryObject = query.getQueryObject(); + Document sortObject = query.isSorted() ? query.getSortObject() : new Document(); sortObject.put(idPropertyName, 1); @@ -57,13 +62,7 @@ static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPrope } } - Document queryObject = query.getQueryObject(); - List or = (List) queryObject.getOrDefault("$or", new ArrayList<>()); - - // TODO: reverse scrolling - Map keysetValues = query.getKeyset().getKeys(); - Document keysetSort = new Document(); List sortKeys = new ArrayList<>(sortObject.keySet()); if (!keysetValues.isEmpty() && !keysetValues.keySet().containsAll(sortKeys)) { @@ -86,10 +85,11 @@ static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPrope Object o = keysetValues.get(sortSegment); if (j >= i) { // tail segment - if(o instanceof BsonNull) { - throw new IllegalStateException("Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment)); + if (o instanceof BsonNull) { + throw new IllegalStateException( + "Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment)); } - sortConstraint.put(sortSegment, new Document(sortOrder == 1 ? "$gt" : "$lt", o)); + sortConstraint.put(sortSegment, new Document(getComparator(sortOrder, keyset.getDirection()), o)); break; } @@ -102,9 +102,6 @@ static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPrope } } - if (!keysetSort.isEmpty()) { - or.add(keysetSort); - } if (!or.isEmpty()) { queryObject.put("$or", or); } @@ -112,6 +109,18 @@ static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPrope return new KeySetScrollQuery(queryObject, fieldsObject, sortObject); } + private static String getComparator(int sortOrder, Direction direction) { + + // use gte/lte to include the object at the cursor/keyset so that + // we can include it in the result to check whether there is a next object. + // It needs to be filtered out later on. + if (direction == Direction.Backward) { + return sortOrder == 0 ? "$gte" : "$lte"; + } + + return sortOrder == 1 ? "$gt" : "$lt"; + } + static Window createWindow(Document sortObject, int limit, List result, EntityOperations operations) { IntFunction positionFunction = value -> { diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java index 35c9b9d0cd..aca3ea4bc9 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java @@ -15,8 +15,8 @@ */ package org.springframework.data.mongodb.core; +import static org.assertj.core.api.Assertions.*; import static org.springframework.data.mongodb.core.query.Criteria.*; -import static org.springframework.data.mongodb.test.util.Assertions.*; import lombok.Data; import lombok.NoArgsConstructor; @@ -37,6 +37,7 @@ import org.springframework.data.annotation.PersistenceCreator; import org.springframework.data.auditing.IsNewAwareAuditingHandler; import org.springframework.data.domain.KeysetScrollPosition; +import org.springframework.data.domain.KeysetScrollPosition.Direction; import org.springframework.data.domain.OffsetScrollPosition; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; @@ -118,7 +119,7 @@ void shouldUseKeysetScrollingWithNestedSort() { assertThat(scroll).hasSize(2); assertThat(scroll).containsOnly(john20, john40); - scroll = template.scroll(q.with(scroll.positionAt(scroll.size()-1)), WithNestedDocument.class); + scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)), WithNestedDocument.class); assertThat(scroll.hasNext()).isFalse(); assertThat(scroll.isLast()).isTrue(); @@ -143,6 +144,19 @@ void shouldErrorOnNullValueForQuery() { new Document("name", "foo")); template.insertAll(Arrays.asList(john20, john40, john41, john42, john43, john44)); + } + + @Test // GH-4308 + void shouldAllowReverseSort() { + + WithNestedDocument john20 = new WithNestedDocument(null, "John", 120, new WithNestedDocument("John", 20), + new Document("name", "bar")); + WithNestedDocument john40 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 40), + new Document("name", "baz")); + WithNestedDocument john41 = new WithNestedDocument(null, "John", 141, new WithNestedDocument("John", 41), + new Document("name", "foo")); + + template.insertAll(Arrays.asList(john20, john40, john41)); Query q = new Query(where("name").regex("J.*")).with(Sort.by("nested.name", "nested.age", "document.name")) .limit(2); @@ -155,11 +169,22 @@ void shouldErrorOnNullValueForQuery() { assertThat(scroll).hasSize(2); assertThat(scroll).containsOnly(john20, john40); - ScrollPosition startAfter = scroll.positionAt(scroll.size()-1); + scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)), WithNestedDocument.class); - assertThatExceptionOfType(IllegalStateException.class) - .isThrownBy(() -> template.scroll(q.with(startAfter), WithNestedDocument.class)) - .withMessageContaining("document.name"); + assertThat(scroll.hasNext()).isFalse(); + assertThat(scroll.isLast()).isTrue(); + assertThat(scroll).hasSize(1); + assertThat(scroll).containsOnly(john41); + + KeysetScrollPosition scrollPosition = (KeysetScrollPosition) scroll.positionAt(0); + KeysetScrollPosition reversePosition = KeysetScrollPosition.of(scrollPosition.getKeys(), Direction.Backward); + + scroll = template.scroll(q.with(reversePosition), WithNestedDocument.class); + + assertThat(scroll.hasNext()).isTrue(); + assertThat(scroll.isLast()).isFalse(); + assertThat(scroll).hasSize(2); + assertThat(scroll).containsOnly(john20, john40); } @ParameterizedTest // GH-4308 @@ -185,7 +210,7 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla assertThat(scroll).hasSize(2); assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); - scroll = template.scroll(q.with(scroll.positionAt(scroll.size()-1)).limit(3), resultType, "person"); + scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)).limit(3), resultType, "person"); assertThat(scroll.hasNext()).isTrue(); assertThat(scroll.isLast()).isFalse(); @@ -193,7 +218,7 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); - scroll = template.scroll(q.with(scroll.positionAt(scroll.size()-1)).limit(1), resultType, "person"); + scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)).limit(1), resultType, "person"); assertThat(scroll.hasNext()).isFalse(); assertThat(scroll.isLast()).isTrue(); From ab4905f2cfbb247fabc7fbcbc536ac8a9ddf702a Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Thu, 16 Mar 2023 12:02:34 +0100 Subject: [PATCH 6/7] Document limitations around nullable properties. --- .../mongodb/core/ExecutableFindOperation.java | 8 ++++++-- .../data/mongodb/core/MongoOperations.java | 16 +++++++++++++--- .../data/mongodb/core/ReactiveFindOperation.java | 7 ++++++- .../mongodb/core/ReactiveMongoOperations.java | 14 ++++++++++++-- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java index 5ae2010937..d5d57e97ff 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ExecutableFindOperation.java @@ -20,8 +20,9 @@ import java.util.stream.Stream; import org.springframework.dao.DataAccessException; -import org.springframework.data.domain.Window; +import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Window; import org.springframework.data.geo.GeoResults; import org.springframework.data.mongodb.core.query.CriteriaDefinition; import org.springframework.data.mongodb.core.query.NearQuery; @@ -128,10 +129,13 @@ default Optional first() { /** * Return a window of elements either starting or resuming at * {@link org.springframework.data.domain.ScrollPosition}. + *

+ * When using {@link KeysetScrollPosition}, make sure to use non-nullable + * {@link org.springframework.data.domain.Sort sort properties} as MongoDB does not support criteria to reconstruct + * a query result from absent document fields or {@code null} values through {@code $gt/$lt} operators. * * @param scrollPosition the scroll position. * @return a window of the resulting elements. - * @throws IllegalStateException if a potential {@literal KeysetScrollPosition} contains an invalid position. * @since 4.1 * @see org.springframework.data.domain.OffsetScrollPosition * @see org.springframework.data.domain.KeysetScrollPosition diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java index 2d51a56c15..2f3c0dd926 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoOperations.java @@ -807,19 +807,24 @@ MapReduceResults mapReduce(Query query, String inputCollectionName, Strin List find(Query query, Class entityClass, String collectionName); /** - * Query for a window window of objects of type T from the specified collection.
+ * Query for a window of objects of type T from the specified collection.
* Make sure to either set {@link Query#skip(long)} or {@link Query#with(KeysetScrollPosition)} along with * {@link Query#limit(int)} to limit large query results for efficient scrolling.
* Result objects are converted from the MongoDB native representation using an instance of {@see MongoConverter}. * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
* If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way * to map objects since the test for class type is done in the client and not on the server. + *

+ * When using {@link KeysetScrollPosition}, make sure to use non-nullable {@link org.springframework.data.domain.Sort + * sort properties} as MongoDB does not support criteria to reconstruct a query result from absent document fields or + * {@code null} values through {@code $gt/$lt} operators. * * @param query the query class that specifies the criteria used to find a record and also an optional fields * specification. Must not be {@literal null}. * @param entityType the parametrized type of the returned window. * @return the converted window. - * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid + * position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) @@ -834,13 +839,18 @@ MapReduceResults mapReduce(Query query, String inputCollectionName, Strin * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
* If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way * to map objects since the test for class type is done in the client and not on the server. + *

+ * When using {@link KeysetScrollPosition}, make sure to use non-nullable {@link org.springframework.data.domain.Sort + * sort properties} as MongoDB does not support criteria to reconstruct a query result from absent document fields or + * {@code null} values through {@code $gt/$lt} operators. * * @param query the query class that specifies the criteria used to find a record and also an optional fields * specification. Must not be {@literal null}. * @param entityType the parametrized type of the returned window. * @param collectionName name of the collection to retrieve the objects from. * @return the converted window. - * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid + * position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java index 3786cdac03..4456ab6ac4 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveFindOperation.java @@ -18,8 +18,9 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import org.springframework.data.domain.Window; +import org.springframework.data.domain.KeysetScrollPosition; import org.springframework.data.domain.ScrollPosition; +import org.springframework.data.domain.Window; import org.springframework.data.geo.GeoResult; import org.springframework.data.mongodb.core.query.CriteriaDefinition; import org.springframework.data.mongodb.core.query.NearQuery; @@ -91,6 +92,10 @@ interface TerminatingFind { /** * Return a scroll of elements either starting or resuming at {@link ScrollPosition}. + *

+ * When using {@link KeysetScrollPosition}, make sure to use non-nullable + * {@link org.springframework.data.domain.Sort sort properties} as MongoDB does not support criteria to reconstruct + * a query result from absent document fields or {@code null} values through {@code $gt/$lt} operators. * * @param scrollPosition the scroll position. * @return a scroll of the resulting elements. diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java index 8030fb8a9d..af36989654 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoOperations.java @@ -473,12 +473,17 @@ Mono> createView(String name, String source, Aggregati * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
* If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way * to map objects since the test for class type is done in the client and not on the server. + *

+ * When using {@link KeysetScrollPosition}, make sure to use non-nullable {@link org.springframework.data.domain.Sort + * sort properties} as MongoDB does not support criteria to reconstruct a query result from absent document fields or + * {@code null} values through {@code $gt/$lt} operators. * * @param query the query class that specifies the criteria used to find a record and also an optional fields * specification. Must not be {@literal null}. * @param entityType the parametrized type of the returned list. * @return {@link Mono} emitting the converted window. - * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid + * position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) @@ -493,13 +498,18 @@ Mono> createView(String name, String source, Aggregati * Unless configured otherwise, an instance of {@link MappingMongoConverter} will be used.
* If your collection does not contain a homogeneous collection of types, this operation will not be an efficient way * to map objects since the test for class type is done in the client and not on the server. + *

+ * When using {@link KeysetScrollPosition}, make sure to use non-nullable {@link org.springframework.data.domain.Sort + * sort properties} as MongoDB does not support criteria to reconstruct a query result from absent document fields or + * {@code null} values through {@code $gt/$lt} operators. * * @param query the query class that specifies the criteria used to find a record and also an optional fields * specification. Must not be {@literal null}. * @param entityType the parametrized type of the returned list. * @param collectionName name of the collection to retrieve the objects from. * @return {@link Mono} emitting the converted window. - * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid position. + * @throws IllegalStateException if a potential {@link Query#getKeyset() KeysetScrollPosition} contains an invalid + * position. * @since 4.1 * @see Query#with(org.springframework.data.domain.OffsetScrollPosition) * @see Query#with(org.springframework.data.domain.KeysetScrollPosition) From 135634e9edb1651a43867f1bb76bda6f2888280c Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Fri, 17 Mar 2023 10:52:08 +0100 Subject: [PATCH 7/7] Use projecting read callback to allow interface projections. Along the lines fix entity operations proxy handling by reading the underlying map instead of inspecting the proxy interface. Also make sure to map potential raw fields back to the according property. --- .../data/mongodb/core/EntityOperations.java | 77 ++++-- .../data/mongodb/core/MongoTemplate.java | 5 +- .../mongodb/core/ReactiveMongoTemplate.java | 12 +- .../data/mongodb/core/ScrollUtils.java | 5 +- .../core/EntityOperationsUnitTests.java | 40 ++- .../core/MongoTemplateScrollTests.java | 239 ++++++++++++++---- .../ReactiveMongoTemplateScrollTests.java | 95 +++++-- 7 files changed, 382 insertions(+), 91 deletions(-) diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java index cbf252f1dc..9212e20ae2 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/EntityOperations.java @@ -30,6 +30,8 @@ import org.springframework.data.mapping.MappingException; import org.springframework.data.mapping.PersistentEntity; import org.springframework.data.mapping.PersistentPropertyAccessor; +import org.springframework.data.mapping.PersistentPropertyPath; +import org.springframework.data.mapping.PropertyPath; import org.springframework.data.mapping.context.MappingContext; import org.springframework.data.mapping.model.ConvertingPropertyAccessor; import org.springframework.data.mongodb.core.CollectionOptions.TimeSeriesOptions; @@ -50,6 +52,7 @@ import org.springframework.data.projection.EntityProjection; import org.springframework.data.projection.EntityProjectionIntrospector; import org.springframework.data.projection.ProjectionFactory; +import org.springframework.data.projection.TargetAware; import org.springframework.data.util.Optionals; import org.springframework.lang.Nullable; import org.springframework.util.Assert; @@ -117,12 +120,16 @@ Entity forEntity(T entity) { Assert.notNull(entity, "Bean must not be null"); + if (entity instanceof TargetAware targetAware) { + return new SimpleMappedEntity((Map) targetAware.getTarget(), this); + } + if (entity instanceof String) { - return new UnmappedEntity(parse(entity.toString())); + return new UnmappedEntity(parse(entity.toString()), this); } if (entity instanceof Map) { - return new SimpleMappedEntity((Map) entity); + return new SimpleMappedEntity((Map) entity, this); } return MappedEntity.of(entity, context, this); @@ -142,11 +149,11 @@ AdaptibleEntity forEntity(T entity, ConversionService conversionService) Assert.notNull(conversionService, "ConversionService must not be null"); if (entity instanceof String) { - return new UnmappedEntity(parse(entity.toString())); + return new UnmappedEntity(parse(entity.toString()), this); } if (entity instanceof Map) { - return new SimpleMappedEntity((Map) entity); + return new SimpleMappedEntity((Map) entity, this); } return AdaptibleMappedEntity.of(entity, context, conversionService, this); @@ -287,7 +294,8 @@ public TypedOperations forType(@Nullable Class entityClass) { */ public EntityProjection introspectProjection(Class resultType, Class entityType) { - if (!queryMapper.getMappingContext().hasPersistentEntityFor(entityType)) { + MongoPersistentEntity persistentEntity = queryMapper.getMappingContext().getPersistentEntity(entityType); + if (persistentEntity == null && !resultType.isInterface() || ClassUtils.isAssignable(Document.class, resultType)) { return (EntityProjection) EntityProjection.nonProjecting(resultType); } return introspector.introspect(resultType, entityType); @@ -369,6 +377,7 @@ private Document getMappedValidator(Validator validator, Class domainType) { * A representation of information about an entity. * * @author Oliver Gierke + * @author Christoph Strobl * @since 2.1 */ interface Entity { @@ -471,10 +480,10 @@ default boolean isVersionedEntity() { /** * @param sortObject * @return - * @since 3.1 + * @since 4.1 * @throws IllegalStateException if a sort key yields {@literal null}. */ - Map extractKeys(Document sortObject); + Map extractKeys(Document sortObject, Class sourceType); } @@ -523,9 +532,11 @@ interface AdaptibleEntity extends Entity { private static class UnmappedEntity> implements AdaptibleEntity { private final T map; + private final EntityOperations entityOperations; - protected UnmappedEntity(T map) { + protected UnmappedEntity(T map, EntityOperations entityOperations) { this.map = map; + this.entityOperations = entityOperations; } @Override @@ -596,13 +607,19 @@ public boolean isNew() { } @Override - public Map extractKeys(Document sortObject) { + public Map extractKeys(Document sortObject, Class sourceType) { Map keyset = new LinkedHashMap<>(); - keyset.put(ID_FIELD, getId()); + MongoPersistentEntity sourceEntity = entityOperations.context.getPersistentEntity(sourceType); + if (sourceEntity != null && sourceEntity.hasIdProperty()) { + keyset.put(sourceEntity.getRequiredIdProperty().getName(), getId()); + } else { + keyset.put(ID_FIELD, getId()); + } for (String key : sortObject.keySet()) { - Object value = BsonUtils.resolveValue(map, key); + + Object value = resolveValue(key, sourceEntity); if (value == null) { throw new IllegalStateException( @@ -614,12 +631,24 @@ public Map extractKeys(Document sortObject) { return keyset; } + + @Nullable + private Object resolveValue(String key, @Nullable MongoPersistentEntity sourceEntity) { + + if (sourceEntity == null) { + return BsonUtils.resolveValue(map, key); + } + PropertyPath from = PropertyPath.from(key, sourceEntity.getTypeInformation()); + PersistentPropertyPath persistentPropertyPath = entityOperations.context + .getPersistentPropertyPath(from); + return BsonUtils.resolveValue(map, persistentPropertyPath.toDotPath(p -> p.getFieldName())); + } } private static class SimpleMappedEntity> extends UnmappedEntity { - protected SimpleMappedEntity(T map) { - super(map); + protected SimpleMappedEntity(T map, EntityOperations entityOperations) { + super(map, entityOperations); } @Override @@ -758,10 +787,15 @@ public boolean isNew() { } @Override - public Map extractKeys(Document sortObject) { + public Map extractKeys(Document sortObject, Class sourceType) { Map keyset = new LinkedHashMap<>(); - keyset.put(entity.getRequiredIdProperty().getName(), getId()); + MongoPersistentEntity sourceEntity = entityOperations.context.getPersistentEntity(sourceType); + if (sourceEntity != null && sourceEntity.hasIdProperty()) { + keyset.put(sourceEntity.getRequiredIdProperty().getName(), getId()); + } else { + keyset.put(entity.getRequiredIdProperty().getName(), getId()); + } for (String key : sortObject.keySet()) { @@ -933,6 +967,14 @@ interface TypedOperations { * @since 3.3 */ TimeSeriesOptions mapTimeSeriesOptions(TimeSeriesOptions options); + + /** + * @return the name of the id field. + * @since 4.1 + */ + default String getIdKeyName() { + return ID_FIELD; + } } /** @@ -1055,6 +1097,11 @@ private String mappedNameOrDefault(String name) { MongoPersistentProperty persistentProperty = entity.getPersistentProperty(name); return persistentProperty != null ? persistentProperty.getFieldName() : name; } + + @Override + public String getIdKeyName() { + return entity.getIdProperty().getName(); + } } } diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java index c86c61cb69..00e8132de5 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java @@ -870,7 +870,8 @@ Window doScroll(Query query, Class sourceClass, Class targetClass, Assert.notNull(sourceClass, "Entity type must not be null"); Assert.notNull(targetClass, "Target type must not be null"); - ReadDocumentCallback callback = new ReadDocumentCallback<>(mongoConverter, targetClass, collectionName); + EntityProjection projection = operations.introspectProjection(targetClass, sourceClass); + ProjectingReadCallback callback = new ProjectingReadCallback<>(mongoConverter, projection, collectionName); int limit = query.isLimited() ? query.getLimit() + 1 : Integer.MAX_VALUE; if (query.hasKeyset()) { @@ -882,7 +883,7 @@ Window doScroll(Query query, Class sourceClass, Class targetClass, keysetPaginationQuery.fields(), sourceClass, new QueryCursorPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback); - return ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), result, operations); + return ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), result, sourceClass, operations); } List result = doFind(collectionName, createDelegate(query), query.getQueryObject(), query.getFieldsObject(), diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java index 6bf38da1c9..1e99adddbe 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java @@ -849,6 +849,8 @@ Mono> doScroll(Query query, Class sourceClass, Class targetC Assert.notNull(sourceClass, "Entity type must not be null"); Assert.notNull(targetClass, "Target type must not be null"); + EntityProjection projection = operations.introspectProjection(targetClass, sourceClass); + ProjectingReadCallback callback = new ProjectingReadCallback<>(mongoConverter, projection, collectionName); int limit = query.isLimited() ? query.getLimit() + 1 : Integer.MAX_VALUE; if (query.hasKeyset()) { @@ -857,15 +859,15 @@ Mono> doScroll(Query query, Class sourceClass, Class targetC operations.getIdPropertyName(sourceClass)); Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), - keysetPaginationQuery.query(), keysetPaginationQuery.fields(), targetClass, - new QueryFindPublisherPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass)).collectList(); + keysetPaginationQuery.query(), keysetPaginationQuery.fields(), sourceClass, + new QueryFindPublisherPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback).collectList(); - return result.map(it -> ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), it, operations)); + return result.map(it -> ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), it, sourceClass, operations)); } Mono> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), query.getQueryObject(), - query.getFieldsObject(), targetClass, - new QueryFindPublisherPreparer(query, query.getSortObject(), limit, query.getSkip(), sourceClass)) + query.getFieldsObject(), sourceClass, + new QueryFindPublisherPreparer(query, query.getSortObject(), limit, query.getSkip(), sourceClass), callback) .collectList(); return result.map( diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java index 7c0ae5da99..112f95270b 100644 --- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java +++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java @@ -121,14 +121,15 @@ private static String getComparator(int sortOrder, Direction direction) { return sortOrder == 1 ? "$gt" : "$lt"; } - static Window createWindow(Document sortObject, int limit, List result, EntityOperations operations) { + static Window createWindow(Document sortObject, int limit, List result, Class sourceType, + EntityOperations operations) { IntFunction positionFunction = value -> { T last = result.get(value); Entity entity = operations.forEntity(last); - Map keys = entity.extractKeys(sortObject); + Map keys = entity.extractKeys(sortObject, sourceType); return KeysetScrollPosition.of(keys); }; diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java index 6107abaa30..3635cf4e32 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/EntityOperationsUnitTests.java @@ -33,11 +33,13 @@ import org.springframework.data.mongodb.core.convert.NoOpDbRefResolver; import org.springframework.data.mongodb.core.mapping.TimeSeries; import org.springframework.data.mongodb.test.util.MongoTestMappingContext; +import org.springframework.data.projection.SpelAwareProxyProjectionFactory; /** * Unit tests for {@link EntityOperations}. * * @author Mark Paluch + * @author Christoph Strobl */ class EntityOperationsUnitTests { @@ -70,7 +72,8 @@ void shouldExtractKeysFromEntity() { WithNestedDocument object = new WithNestedDocument("foo"); - Map keys = operations.forEntity(object).extractKeys(new Document("id", 1)); + Map keys = operations.forEntity(object).extractKeys(new Document("id", 1), + WithNestedDocument.class); assertThat(keys).containsEntry("id", "foo"); } @@ -80,7 +83,7 @@ void shouldExtractKeysFromDocument() { Document object = new Document("id", "foo"); - Map keys = operations.forEntity(object).extractKeys(new Document("id", 1)); + Map keys = operations.forEntity(object).extractKeys(new Document("id", 1), Document.class); assertThat(keys).containsEntry("id", "foo"); } @@ -90,7 +93,8 @@ void shouldExtractKeysFromNestedEntity() { WithNestedDocument object = new WithNestedDocument("foo", new WithNestedDocument("bar"), null); - Map keys = operations.forEntity(object).extractKeys(new Document("nested.id", 1)); + Map keys = operations.forEntity(object).extractKeys(new Document("nested.id", 1), + WithNestedDocument.class); assertThat(keys).containsEntry("nested.id", "bar"); } @@ -101,7 +105,8 @@ void shouldExtractKeysFromNestedEntityDocument() { WithNestedDocument object = new WithNestedDocument("foo", new WithNestedDocument("bar"), new Document("john", "doe")); - Map keys = operations.forEntity(object).extractKeys(new Document("document.john", 1)); + Map keys = operations.forEntity(object).extractKeys(new Document("document.john", 1), + WithNestedDocument.class); assertThat(keys).containsEntry("document.john", "doe"); } @@ -111,11 +116,32 @@ void shouldExtractKeysFromNestedDocument() { Document object = new Document("document", new Document("john", "doe")); - Map keys = operations.forEntity(object).extractKeys(new Document("document.john", 1)); + Map keys = operations.forEntity(object).extractKeys(new Document("document.john", 1), + Document.class); assertThat(keys).containsEntry("document.john", "doe"); } + @Test // GH-4308 + void shouldExtractIdPropertyNameFromRawDocument() { + + Document object = new Document("_id", "id-1").append("value", "val"); + + Map keys = operations.forEntity(object).extractKeys(new Document("value", 1), DomainTypeWithIdProperty.class); + + assertThat(keys).containsEntry("id", "id-1"); + } + + @Test // GH-4308 + void shouldExtractValuesFromProxy() { + + ProjectionInterface source = new SpelAwareProxyProjectionFactory().createProjection(ProjectionInterface.class, new Document("_id", "id-1").append("value", "val")); + + Map keys = operations.forEntity(source).extractKeys(new Document("value", 1), DomainTypeWithIdProperty.class); + + assertThat(keys).isEqualTo(new Document("id", "id-1").append("value", "val")); + } + EntityOperations.AdaptibleEntity initAdaptibleEntity(T source) { return operations.forEntity(source, conversionService); } @@ -150,4 +176,8 @@ public WithNestedDocument(String id) { this.id = id; } } + + interface ProjectionInterface { + String getValue(); + } } diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java index aca3ea4bc9..88c03e9158 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java @@ -18,10 +18,13 @@ import static org.assertj.core.api.Assertions.*; import static org.springframework.data.mongodb.core.query.Criteria.*; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.lang.reflect.Proxy; import java.util.Arrays; +import java.util.Comparator; import java.util.function.Function; import java.util.stream.Stream; @@ -44,10 +47,13 @@ import org.springframework.data.domain.Window; import org.springframework.data.mapping.context.PersistentEntities; import org.springframework.data.mongodb.core.MongoTemplateTests.PersonWithIdPropertyOfTypeUUIDListener; +import org.springframework.data.mongodb.core.mapping.Field; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.test.util.Client; import org.springframework.data.mongodb.test.util.MongoClientExtension; import org.springframework.data.mongodb.test.util.MongoTestTemplate; +import org.springframework.lang.Nullable; +import org.springframework.util.ObjectUtils; import com.mongodb.client.MongoClient; @@ -90,10 +96,22 @@ class MongoTemplateScrollTests { }); }); + private static int compareProxies(PersonInterfaceProjection actual, PersonInterfaceProjection expected) { + if (actual.getAge() != expected.getAge()) { + return -1; + } + if (!ObjectUtils.nullSafeEquals(actual.getFirstName(), expected.getFirstName())) { + return -1; + } + + return 0; + } + @BeforeEach void setUp() { template.remove(Person.class).all(); template.remove(WithNestedDocument.class).all(); + template.remove(WithRenamedField.class).all(); } @Test // GH-4308 @@ -112,19 +130,19 @@ void shouldUseKeysetScrollingWithNestedSort() { .limit(2); q.with(KeysetScrollPosition.initial()); - Window scroll = template.scroll(q, WithNestedDocument.class); + Window window = template.scroll(q, WithNestedDocument.class); - assertThat(scroll.hasNext()).isTrue(); - assertThat(scroll.isLast()).isFalse(); - assertThat(scroll).hasSize(2); - assertThat(scroll).containsOnly(john20, john40); + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(2); + assertThat(window).containsOnly(john20, john40); - scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)), WithNestedDocument.class); + window = template.scroll(q.with(window.positionAt(window.size() - 1)), WithNestedDocument.class); - assertThat(scroll.hasNext()).isFalse(); - assertThat(scroll.isLast()).isTrue(); - assertThat(scroll).hasSize(1); - assertThat(scroll).containsOnly(john41); + assertThat(window.hasNext()).isFalse(); + assertThat(window.isLast()).isTrue(); + assertThat(window).hasSize(1); + assertThat(window).containsOnly(john41); } @Test // GH-4308 @@ -162,35 +180,35 @@ void shouldAllowReverseSort() { .limit(2); q.with(KeysetScrollPosition.initial()); - Window scroll = template.scroll(q, WithNestedDocument.class); + Window window = template.scroll(q, WithNestedDocument.class); - assertThat(scroll.hasNext()).isTrue(); - assertThat(scroll.isLast()).isFalse(); - assertThat(scroll).hasSize(2); - assertThat(scroll).containsOnly(john20, john40); + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(2); + assertThat(window).containsOnly(john20, john40); - scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)), WithNestedDocument.class); + window = template.scroll(q.with(window.positionAt(window.size() - 1)), WithNestedDocument.class); - assertThat(scroll.hasNext()).isFalse(); - assertThat(scroll.isLast()).isTrue(); - assertThat(scroll).hasSize(1); - assertThat(scroll).containsOnly(john41); + assertThat(window.hasNext()).isFalse(); + assertThat(window.isLast()).isTrue(); + assertThat(window).hasSize(1); + assertThat(window).containsOnly(john41); - KeysetScrollPosition scrollPosition = (KeysetScrollPosition) scroll.positionAt(0); + KeysetScrollPosition scrollPosition = (KeysetScrollPosition) window.positionAt(0); KeysetScrollPosition reversePosition = KeysetScrollPosition.of(scrollPosition.getKeys(), Direction.Backward); - scroll = template.scroll(q.with(reversePosition), WithNestedDocument.class); + window = template.scroll(q.with(reversePosition), WithNestedDocument.class); - assertThat(scroll.hasNext()).isTrue(); - assertThat(scroll.isLast()).isFalse(); - assertThat(scroll).hasSize(2); - assertThat(scroll).containsOnly(john20, john40); + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(2); + assertThat(window).containsOnly(john20, john40); } @ParameterizedTest // GH-4308 @MethodSource("positions") public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Class resultType, - Function assertionConverter) { + Function assertionConverter, @Nullable Comparator comparator) { Person john20 = new Person("John", 20); Person john40_1 = new Person("John", 40); @@ -201,53 +219,182 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42)); Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); - q.with(scrollPosition); - Window scroll = template.scroll(q, resultType, "person"); + Window window = template.query(Person.class).inCollection("person").as(resultType).matching(q) + .scroll(scrollPosition); + + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(2); + assertWindow(window, comparator).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); + + window = template.query(Person.class).inCollection("person").as(resultType).matching(q.limit(3)) + .scroll(window.positionAt(window.size() - 1)); + + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(3); + assertWindow(window, comparator).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); + assertWindow(window, comparator).containsAnyOf(assertionConverter.apply(john40_1), + assertionConverter.apply(john40_2)); + + window = template.query(Person.class).inCollection("person").as(resultType).matching(q.limit(1)) + .scroll(window.positionAt(window.size() - 1)); + + assertThat(window.hasNext()).isFalse(); + assertThat(window.isLast()).isTrue(); + assertThat(window).hasSize(1); + assertWindow(window, comparator).containsAnyOf(assertionConverter.apply(john40_1), + assertionConverter.apply(john40_2)); + } + + @ParameterizedTest // GH-4308 + @MethodSource("renamedFieldProjectTargets") + void scrollThroughResultsWithRenamedField(Class resultType, Function assertionConverter) { + + WithRenamedField one = new WithRenamedField("id-1", "v1", null); + WithRenamedField two = new WithRenamedField("id-2", "v2", null); + WithRenamedField three = new WithRenamedField("id-3", "v3", null); - assertThat(scroll.hasNext()).isTrue(); - assertThat(scroll.isLast()).isFalse(); - assertThat(scroll).hasSize(2); - assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); + template.insertAll(Arrays.asList(one, two, three)); - scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)).limit(3), resultType, "person"); + Query q = new Query(where("value").regex("v.*")).with(Sort.by(Sort.Direction.DESC, "value")).limit(2); + q.with(KeysetScrollPosition.initial()); + + Window window = template.query(WithRenamedField.class).as(resultType).matching(q) + .scroll(KeysetScrollPosition.initial()); - assertThat(scroll.hasNext()).isTrue(); - assertThat(scroll.isLast()).isFalse(); - assertThat(scroll).hasSize(3); - assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); - assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(2); + assertThat(window).containsOnly(assertionConverter.apply(three), assertionConverter.apply(two)); - scroll = template.scroll(q.with(scroll.positionAt(scroll.size() - 1)).limit(1), resultType, "person"); + window = template.query(WithRenamedField.class).as(resultType).matching(q) + .scroll(window.positionAt(window.size() - 1)); - assertThat(scroll.hasNext()).isFalse(); - assertThat(scroll.isLast()).isTrue(); - assertThat(scroll).hasSize(1); - assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + assertThat(window.hasNext()).isFalse(); + assertThat(window.isLast()).isTrue(); + assertThat(window).hasSize(1); + assertThat(window).containsOnly(assertionConverter.apply(one)); } static Stream positions() { return Stream.of(args(KeysetScrollPosition.initial(), Person.class, Function.identity()), // args(KeysetScrollPosition.initial(), Document.class, MongoTemplateScrollTests::toDocument), // - args(OffsetScrollPosition.initial(), Person.class, Function.identity())); + args(OffsetScrollPosition.initial(), Person.class, Function.identity()), // + args(OffsetScrollPosition.initial(), PersonDtoProjection.class, + MongoTemplateScrollTests::toPersonDtoProjection), // + args(OffsetScrollPosition.initial(), PersonInterfaceProjection.class, + MongoTemplateScrollTests::toPersonInterfaceProjection, MongoTemplateScrollTests::compareProxies)); + } + + static Stream renamedFieldProjectTargets() { + return Stream.of(Arguments.of(WithRenamedField.class, Function.identity()), + Arguments.of(Document.class, new Function() { + @Override + public Document apply(WithRenamedField withRenamedField) { + return new Document("_id", withRenamedField.getId()).append("_val", withRenamedField.getValue()) + .append("_class", WithRenamedField.class.getName()); + } + })); + } + + static org.assertj.core.api.IterableAssert assertWindow(Window window, @Nullable Comparator comparator) { + return comparator != null ? assertThat(window).usingElementComparator(comparator) : assertThat(window); } private static Arguments args(ScrollPosition scrollPosition, Class resultType, Function assertionConverter) { - return Arguments.of(scrollPosition, resultType, assertionConverter); + return args(scrollPosition, resultType, assertionConverter, null); + } + + private static Arguments args(ScrollPosition scrollPosition, Class resultType, + Function assertionConverter, @Nullable Comparator comparator) { + return Arguments.of(scrollPosition, resultType, assertionConverter, comparator); } static Document toDocument(Person person) { + return new Document("_class", person.getClass().getName()).append("_id", person.getId()).append("active", true) .append("firstName", person.getFirstName()).append("age", person.getAge()); } + static PersonDtoProjection toPersonDtoProjection(Person person) { + + PersonDtoProjection dto = new PersonDtoProjection(); + dto.firstName = person.getFirstName(); + dto.age = person.getAge(); + return dto; + } + + static PersonInterfaceProjection toPersonInterfaceProjection(Person person) { + + return new PersonInterfaceProjectionImpl(person); + } + + @Data + static class PersonDtoProjection { + String firstName; + int age; + } + + interface PersonInterfaceProjection { + String getFirstName(); + + int getAge(); + } + + static class PersonInterfaceProjectionImpl implements PersonInterfaceProjection { + + final Person delegate; + + public PersonInterfaceProjectionImpl(Person delegate) { + this.delegate = delegate; + } + + @Override + public String getFirstName() { + return delegate.getFirstName(); + } + + @Override + public int getAge() { + return delegate.getAge(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Proxy) { + return true; + } + return false; + } + + @Override + public int hashCode() { + return ObjectUtils.nullSafeHashCode(delegate); + } + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + static class WithRenamedField { + + String id; + + @Field("_val") String value; + + WithRenamedField nested; + } + @NoArgsConstructor @Data class WithNestedDocument { String id; + String name; int age; diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java index d42d8d99f0..35f67782f2 100644 --- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java +++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/ReactiveMongoTemplateScrollTests.java @@ -18,6 +18,10 @@ import static org.springframework.data.mongodb.core.query.Criteria.*; import static org.springframework.data.mongodb.test.util.Assertions.*; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.mongodb.core.mapping.Field; import reactor.test.StepVerifier; import java.time.Duration; @@ -49,6 +53,7 @@ * Integration tests for {@link Window} queries. * * @author Mark Paluch + * @author Christoph Strobl */ @ExtendWith(MongoClientExtension.class) class ReactiveMongoTemplateScrollTests { @@ -78,6 +83,11 @@ void setUp() { .as(StepVerifier::create) // .expectNextCount(1) // .verifyComplete(); + + template.remove(WithRenamedField.class).all() // + .as(StepVerifier::create) // + .expectNextCount(1) // + .verifyComplete(); } @ParameterizedTest // GH-4308 @@ -100,29 +110,59 @@ public void shouldApplyCursoringCorrectly(ScrollPosition scrollPosition, Cla Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age")).limit(2); q.with(scrollPosition); - Window scroll = template.scroll(q, resultType, "person").block(Duration.ofSeconds(10)); + Window window = template.scroll(q, resultType, "person").block(Duration.ofSeconds(10)); - assertThat(scroll.hasNext()).isTrue(); - assertThat(scroll.isLast()).isFalse(); - assertThat(scroll).hasSize(2); - assertThat(scroll).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(2); + assertThat(window).containsOnly(assertionConverter.apply(jane_20), assertionConverter.apply(jane_40)); - scroll = template.scroll(q.limit(3).with(scroll.positionAt(scroll.size() - 1)), resultType, "person") + window = template.scroll(q.limit(3).with(window.positionAt(window.size() - 1)), resultType, "person") .block(Duration.ofSeconds(10)); - assertThat(scroll.hasNext()).isTrue(); - assertThat(scroll.isLast()).isFalse(); - assertThat(scroll).hasSize(3); - assertThat(scroll).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); - assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(3); + assertThat(window).contains(assertionConverter.apply(jane_42), assertionConverter.apply(john20)); + assertThat(window).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); - scroll = template.scroll(q.limit(1).with(scroll.positionAt(scroll.size() - 1)), resultType, "person") + window = template.scroll(q.limit(1).with(window.positionAt(window.size() - 1)), resultType, "person") .block(Duration.ofSeconds(10)); - assertThat(scroll.hasNext()).isFalse(); - assertThat(scroll.isLast()).isTrue(); - assertThat(scroll).hasSize(1); - assertThat(scroll).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + assertThat(window.hasNext()).isFalse(); + assertThat(window.isLast()).isTrue(); + assertThat(window).hasSize(1); + assertThat(window).containsAnyOf(assertionConverter.apply(john40_1), assertionConverter.apply(john40_2)); + } + + @ParameterizedTest // GH-4308 + @MethodSource("renamedFieldProjectTargets") + void scrollThroughResultsWithRenamedField(Class resultType, Function assertionConverter) { + + WithRenamedField one = new WithRenamedField("id-1", "v1", null); + WithRenamedField two = new WithRenamedField("id-2", "v2", null); + WithRenamedField three = new WithRenamedField("id-3", "v3", null); + + template.insertAll(Arrays.asList(one, two, three)).as(StepVerifier::create).expectNextCount(3).verifyComplete(); + + Query q = new Query(where("value").regex("v.*")).with(Sort.by(Sort.Direction.DESC, "value")).limit(2); + q.with(KeysetScrollPosition.initial()); + + Window window = template.query(WithRenamedField.class).as(resultType).matching(q) + .scroll(KeysetScrollPosition.initial()).block(Duration.ofSeconds(10)); + + assertThat(window.hasNext()).isTrue(); + assertThat(window.isLast()).isFalse(); + assertThat(window).hasSize(2); + assertThat(window).containsOnly(assertionConverter.apply(three), assertionConverter.apply(two)); + + window = template.query(WithRenamedField.class).as(resultType).matching(q) + .scroll(window.positionAt(window.size() - 1)).block(Duration.ofSeconds(10)); + + assertThat(window.hasNext()).isFalse(); + assertThat(window.isLast()).isTrue(); + assertThat(window).hasSize(1); + assertThat(window).containsOnly(assertionConverter.apply(one)); } static Stream positions() { @@ -132,6 +172,17 @@ static Stream positions() { args(OffsetScrollPosition.initial(), Person.class, Function.identity())); } + static Stream renamedFieldProjectTargets() { + return Stream.of(Arguments.of(WithRenamedField.class, Function.identity()), + Arguments.of(Document.class, new Function() { + @Override + public Document apply(WithRenamedField withRenamedField) { + return new Document("_id", withRenamedField.getId()).append("_val", withRenamedField.getValue()) + .append("_class", WithRenamedField.class.getName()); + } + })); + } + private static Arguments args(ScrollPosition scrollPosition, Class resultType, Function assertionConverter) { return Arguments.of(scrollPosition, resultType, assertionConverter); @@ -141,4 +192,16 @@ static Document toDocument(Person person) { return new Document("_class", person.getClass().getName()).append("_id", person.getId()).append("active", true) .append("firstName", person.getFirstName()).append("age", person.getAge()); } + + @Data + @AllArgsConstructor + @NoArgsConstructor + static class WithRenamedField { + + String id; + + @Field("_val") String value; + + MongoTemplateScrollTests.WithRenamedField nested; + } }