From 3942244de0cc3f1868b4cc1b4777a87fb63e47e7 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Mon, 27 Feb 2023 10:27:50 +0100 Subject: [PATCH 1/4] Prepare issue branch --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 13bf0377b4..a8b89c32b4 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.data spring-data-commons - 3.1.0-SNAPSHOT + 3.1.0-GH-2151-SNAPSHOT Spring Data Core Core Spring concepts underpinning every Spring Data module. From 25eb5e3c09922918775eec7c8c44f8e9b9c0bc9d Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Mon, 27 Feb 2023 13:37:06 +0100 Subject: [PATCH 2/4] Introduce Scroll API. --- src/main/asciidoc/index.adoc | 1 + .../asciidoc/repositories-paging-sorting.adoc | 136 ++++++++++++++ src/main/asciidoc/repositories-scrolling.adoc | 83 +++++++++ src/main/asciidoc/repositories.adoc | 117 +----------- .../data/domain/KeysetScrollPosition.java | 144 +++++++++++++++ .../data/domain/OffsetScrollPosition.java | 127 +++++++++++++ .../springframework/data/domain/Pageable.java | 16 ++ .../springframework/data/domain/Scroll.java | 167 ++++++++++++++++++ .../data/domain/ScrollImpl.java | 119 +++++++++++++ .../data/domain/ScrollPosition.java | 33 ++++ .../data/repository/query/FluentQuery.java | 67 ++++++- .../data/repository/query/Parameter.java | 13 +- .../repository/query/ParameterAccessor.java | 9 + .../data/repository/query/Parameters.java | 48 ++++- .../query/ParametersParameterAccessor.java | 17 ++ .../data/repository/query/QueryMethod.java | 32 +++- .../repository/query/ResultProcessor.java | 7 +- .../util/QueryExecutionConverters.java | 2 + .../data/util/CustomCollections.java | 2 +- .../domain/KeysetScrollPositionUnitTests.java | 43 +++++ .../domain/OffsetScrollPositionUnitTests.java | 50 ++++++ .../data/domain/PageRequestUnitTests.java | 9 +- .../data/domain/ScrollUnitTests.java | 69 ++++++++ .../ParametersParameterAccessorUnitTests.java | 17 ++ .../repository/query/ParametersUnitTests.java | 13 +- .../query/QueryMethodUnitTests.java | 55 ++++++ .../SimpleParameterAccessorUnitTests.java | 18 +- 27 files changed, 1280 insertions(+), 134 deletions(-) create mode 100644 src/main/asciidoc/repositories-paging-sorting.adoc create mode 100644 src/main/asciidoc/repositories-scrolling.adoc create mode 100644 src/main/java/org/springframework/data/domain/KeysetScrollPosition.java create mode 100644 src/main/java/org/springframework/data/domain/OffsetScrollPosition.java create mode 100644 src/main/java/org/springframework/data/domain/Scroll.java create mode 100644 src/main/java/org/springframework/data/domain/ScrollImpl.java create mode 100644 src/main/java/org/springframework/data/domain/ScrollPosition.java create mode 100644 src/test/java/org/springframework/data/domain/KeysetScrollPositionUnitTests.java create mode 100644 src/test/java/org/springframework/data/domain/OffsetScrollPositionUnitTests.java create mode 100644 src/test/java/org/springframework/data/domain/ScrollUnitTests.java diff --git a/src/main/asciidoc/index.adoc b/src/main/asciidoc/index.adoc index 6cc84d4cfb..14fd59748d 100644 --- a/src/main/asciidoc/index.adoc +++ b/src/main/asciidoc/index.adoc @@ -2,6 +2,7 @@ Oliver Gierke; Thomas Darimont; Christoph Strobl; Mark Pollack; Thomas Risberg; Mark Paluch; Jay Bryant :revnumber: {version} :revdate: {localdate} +:feature-scroll: true ifdef::backend-epub3[:front-cover-image: image:epub-cover.png[Front Cover,1050,1600]] (C) 2008-2022 The original authors. diff --git a/src/main/asciidoc/repositories-paging-sorting.adoc b/src/main/asciidoc/repositories-paging-sorting.adoc new file mode 100644 index 0000000000..8bf940f474 --- /dev/null +++ b/src/main/asciidoc/repositories-paging-sorting.adoc @@ -0,0 +1,136 @@ +[[repositories.special-parameters]] +=== Special parameter handling + +To handle parameters in your query, define method parameters as already seen in the preceding examples. +Besides that, the infrastructure recognizes certain specific types like `Pageable` and `Sort`, to apply pagination and sorting to your queries dynamically. +The following example demonstrates these features: + +ifdef::feature-scroll[] +.Using `Pageable`, `Slice`, `ScrollPosition`, and `Sort` in query methods +==== +[source,java] +---- +Page findByLastname(String lastname, Pageable pageable); + +Slice findByLastname(String lastname, Pageable pageable); + +Scroll findTop10ByLastname(String lastname, ScrollPosition position, Sort sort); + +List findByLastname(String lastname, Sort sort); + +List findByLastname(String lastname, Pageable pageable); +---- +==== +endif::[] + +ifndef::feature-scroll[] +.Using `Pageable`, `Slice`, and `Sort` in query methods +==== +[source,java] +---- +Page findByLastname(String lastname, Pageable pageable); + +Slice findByLastname(String lastname, Pageable pageable); + +List findByLastname(String lastname, Sort sort); + +List findByLastname(String lastname, Pageable pageable); +---- +==== +endif::[] + +IMPORTANT: APIs taking `Sort` and `Pageable` expect non-`null` values to be handed into methods. +If you do not want to apply any sorting or pagination, use `Sort.unsorted()` and `Pageable.unpaged()`. + +The first method lets you pass an `org.springframework.data.domain.Pageable` instance to the query method to dynamically add paging to your statically defined query. +A `Page` knows about the total number of elements and pages available. +It does so by the infrastructure triggering a count query to calculate the overall number. +As this might be expensive (depending on the store used), you can instead return a `Slice`. +A `Slice` knows only about whether a next `Slice` is available, which might be sufficient when walking through a larger result set. + +Sorting options are handled through the `Pageable` instance, too. +If you need only sorting, add an `org.springframework.data.domain.Sort` parameter to your method. +As you can see, returning a `List` is also possible. +In this case, the additional metadata required to build the actual `Page` instance is not created (which, in turn, means that the additional count query that would have been necessary is not issued). +Rather, it restricts the query to look up only the given range of entities. + +NOTE: To find out how many pages you get for an entire query, you have to trigger an additional count query. +By default, this query is derived from the query you actually trigger. + +[[repositories.paging-and-sorting]] +==== Paging and Sorting + +You can define simple sorting expressions by using property names. +You can concatenate expressions to collect multiple criteria into one expression. + +.Defining sort expressions +==== +[source,java] +---- +Sort sort = Sort.by("firstname").ascending() + .and(Sort.by("lastname").descending()); +---- +==== + +For a more type-safe way to define sort expressions, start with the type for which to define the sort expression and use method references to define the properties on which to sort. + +.Defining sort expressions by using the type-safe API +==== +[source,java] +---- +TypedSort person = Sort.sort(Person.class); + +Sort sort = person.by(Person::getFirstname).ascending() + .and(person.by(Person::getLastname).descending()); +---- +==== + +NOTE: `TypedSort.by(…)` makes use of runtime proxies by (typically) using CGlib, which may interfere with native image compilation when using tools such as Graal VM Native. + +If your store implementation supports Querydsl, you can also use the generated metamodel types to define sort expressions: + +.Defining sort expressions by using the Querydsl API +==== +[source,java] +---- +QSort sort = QSort.by(QPerson.firstname.asc()) + .and(QSort.by(QPerson.lastname.desc())); +---- +==== + +ifdef::feature-scroll[] +include::repositories-scrolling.adoc[] +endif::[] + +[[repositories.limit-query-result]] +=== Limiting Query Results + +You can limit the results of query methods by using the `first` or `top` keywords, which you can use interchangeably. +You can append an optional numeric value to `top` or `first` to specify the maximum result size to be returned. +If the number is left out, a result size of 1 is assumed. +The following example shows how to limit the query size: + +.Limiting the result size of a query with `Top` and `First` +==== +[source,java] +---- +User findFirstByOrderByLastnameAsc(); + +User findTopByOrderByAgeDesc(); + +Page queryFirst10ByLastname(String lastname, Pageable pageable); + +Slice findTop3ByLastname(String lastname, Pageable pageable); + +List findFirst10ByLastname(String lastname, Sort sort); + +List findTop10ByLastname(String lastname, Pageable pageable); +---- +==== + +The limiting expressions also support the `Distinct` keyword for datastores that support distinct queries. +Also, for the queries that limit the result set to one instance, wrapping the result into with the `Optional` keyword is supported. + +If pagination or slicing is applied to a limiting query pagination (and the calculation of the number of available pages), it is applied within the limited result. + +NOTE: Limiting the results in combination with dynamic sorting by using a `Sort` parameter lets you express query methods for the 'K' smallest as well as for the 'K' biggest elements. diff --git a/src/main/asciidoc/repositories-scrolling.adoc b/src/main/asciidoc/repositories-scrolling.adoc new file mode 100644 index 0000000000..85c81f369d --- /dev/null +++ b/src/main/asciidoc/repositories-scrolling.adoc @@ -0,0 +1,83 @@ +[[repositories.scrolling]] +==== Scrolling + +Scrolling is a more fine-grained approach to iterate through larger results set chunks. +Scrolling consists of a stable sort, a scroll type (Offset- or Keyset-based scrolling) and result limiting. +You can define simple sorting expressions by using property names and define static result limiting using the <> through query derivation. +You can concatenate expressions to collect multiple criteria into one expression. + +Scroll queries return a `Scroll` that allows obtaining the scroll position to resume to obtain the next `Scroll` until your application has consumed the entire query result. +Similar to consuming a Java `Iterator>` by obtaining the next batch of results, query result scrolling lets you access the next `ScrollPosition` through `Scroll.lastScrollPosition()`. + +[[repositories.scrolling.offset]] +===== Scrolling using Offset + +Offset scrolling uses similar to pagination, an Offset counter to skip a number of results and let the data source only return results beginning at the given Offset. +This simple mechanism avoids large results being sent to the client application. +However, most databases require materializing the full query result before your server can return the results. + +.Using `OffsetScrollPosition` with Repository Query Methods +==== +[source,java] +---- +interface UserRepository extends Repository { + + Scroll findFirst10ByLastnameOrderByFirstname(String lastname, OffsetScrollPosition position); +} + +Scroll users = repository.findFirst10ByLastnameOrderByFirstname("Doe", OffsetScrollPosition.initial()); + +do { + + for (User u : users) { + // consume the user + } + + // obtain the next Scroll + users = repository.findFirst10ByLastnameOrderByFirstname("Doe", users.lastScrollPosition()); +} while (!users.isEmpty() && !users.isLast()); +---- +==== + +[[repositories.scrolling.keyset]] +===== Scrolling using Keyset-Filtering + +Offset-based requires most databases require materializing the entire result before your server can return the results. +So while the client only sees the portion of the requested results, your server needs to build the full result, which causes additional load. + +Keyset-Filtering approaches result subset retrieval by leveraging built-in capabilities of your database aiming to reduce the computation and I/O requirements for individual queries. +This approach maintains a set of keys to resume scrolling by passing keys into the query, effectively amending your filter criteria. + +The core idea of Keyset-Filtering is to start retrieving results using a stable sorting order. +Once you want to scroll to the next chunk, you obtain a `ScrollPosition` that is used to reconstruct the position within the sorted result. +The `ScrollPosition` captures the keyset of the last entity within the current `Scroll`. +To run the query, reconstruction rewrites the criteria clause to include all sort fields and the primary key so that the database can leverage potential indexes to run the query. +The database needs only constructing a much smaller result from the given keyset position without the need to fully materialize a large result and then skipping results until reaching a particular offset. + +.Using `KeysetScrollPosition` with Repository Query Methods +==== +[source,java] +---- +interface UserRepository extends Repository { + + Scroll findFirst10ByLastnameOrderByFirstname(String lastname, KeysetScrollPosition position); +} + +Scroll users = repository.findFirst10ByLastnameOrderByFirstname("Doe", KeysetScrollPosition.initial()); + +do { + + for (User u : users) { + // consume the user + } + + // obtain the next Scroll + users = repository.findFirst10ByLastnameOrderByFirstname("Doe", users.lastScrollPosition()); +} while (!users.isEmpty() && !users.isLast()); +---- +==== + +Keyset-Filtering works best when your database contains an index that matches the sort fields, hence a static sort works well. +Scroll queries applying Keyset-Filtering require to the properties used in the sort order to be returned by the query, and these must be mapped in the returned entity. + +You can use interface and DTO projections, however make sure to include all properties that you've sorted by to avoid keyset extraction failures. diff --git a/src/main/asciidoc/repositories.adoc b/src/main/asciidoc/repositories.adoc index c60fd8d691..2b2a520cd2 100644 --- a/src/main/asciidoc/repositories.adoc +++ b/src/main/asciidoc/repositories.adoc @@ -92,6 +92,10 @@ Page users = repository.findAll(PageRequest.of(1, 20)); ---- ==== +ifdef::feature-scroll[] +In addition to pagination, scrolling provides a more fine-grained access to iterate through chunks of larger result sets. +endif::[] + In addition to query methods, query derivation for both count and delete queries is available. The following list shows the interface definition for a derived count query: @@ -517,118 +521,7 @@ List findByAddress_ZipCode(ZipCode zipCode); Because we treat the underscore character as a reserved character, we strongly advise following standard Java naming conventions (that is, not using underscores in property names but using camel case instead). -[[repositories.special-parameters]] -=== Special parameter handling - -To handle parameters in your query, define method parameters as already seen in the preceding examples. -Besides that, the infrastructure recognizes certain specific types like `Pageable` and `Sort`, to apply pagination and sorting to your queries dynamically. -The following example demonstrates these features: - -.Using `Pageable`, `Slice`, and `Sort` in query methods -==== -[source,java] ----- -Page findByLastname(String lastname, Pageable pageable); - -Slice findByLastname(String lastname, Pageable pageable); - -List findByLastname(String lastname, Sort sort); - -List findByLastname(String lastname, Pageable pageable); ----- -==== - -IMPORTANT: APIs taking `Sort` and `Pageable` expect non-`null` values to be handed into methods. -If you do not want to apply any sorting or pagination, use `Sort.unsorted()` and `Pageable.unpaged()`. - -The first method lets you pass an `org.springframework.data.domain.Pageable` instance to the query method to dynamically add paging to your statically defined query. -A `Page` knows about the total number of elements and pages available. -It does so by the infrastructure triggering a count query to calculate the overall number. -As this might be expensive (depending on the store used), you can instead return a `Slice`. -A `Slice` knows only about whether a next `Slice` is available, which might be sufficient when walking through a larger result set. - -Sorting options are handled through the `Pageable` instance, too. -If you need only sorting, add an `org.springframework.data.domain.Sort` parameter to your method. -As you can see, returning a `List` is also possible. -In this case, the additional metadata required to build the actual `Page` instance is not created (which, in turn, means that the additional count query that would have been necessary is not issued). -Rather, it restricts the query to look up only the given range of entities. - -NOTE: To find out how many pages you get for an entire query, you have to trigger an additional count query. -By default, this query is derived from the query you actually trigger. - -[[repositories.paging-and-sorting]] -==== Paging and Sorting - -You can define simple sorting expressions by using property names. -You can concatenate expressions to collect multiple criteria into one expression. - -.Defining sort expressions -==== -[source,java] ----- -Sort sort = Sort.by("firstname").ascending() - .and(Sort.by("lastname").descending()); ----- -==== - -For a more type-safe way to define sort expressions, start with the type for which to define the sort expression and use method references to define the properties on which to sort. - -.Defining sort expressions by using the type-safe API -==== -[source,java] ----- -TypedSort person = Sort.sort(Person.class); - -Sort sort = person.by(Person::getFirstname).ascending() - .and(person.by(Person::getLastname).descending()); ----- -==== - -NOTE: `TypedSort.by(…)` makes use of runtime proxies by (typically) using CGlib, which may interfere with native image compilation when using tools such as Graal VM Native. - -If your store implementation supports Querydsl, you can also use the generated metamodel types to define sort expressions: - -.Defining sort expressions by using the Querydsl API -==== -[source,java] ----- -QSort sort = QSort.by(QPerson.firstname.asc()) - .and(QSort.by(QPerson.lastname.desc())); ----- -==== - -[[repositories.limit-query-result]] -=== Limiting Query Results - -You can limit the results of query methods by using the `first` or `top` keywords, which you can use interchangeably. -You can append an optional numeric value to `top` or `first` to specify the maximum result size to be returned. -If the number is left out, a result size of 1 is assumed. -The following example shows how to limit the query size: - -.Limiting the result size of a query with `Top` and `First` -==== -[source,java] ----- -User findFirstByOrderByLastnameAsc(); - -User findTopByOrderByAgeDesc(); - -Page queryFirst10ByLastname(String lastname, Pageable pageable); - -Slice findTop3ByLastname(String lastname, Pageable pageable); - -List findFirst10ByLastname(String lastname, Sort sort); - -List findTop10ByLastname(String lastname, Pageable pageable); ----- -==== - -The limiting expressions also support the `Distinct` keyword for datastores that support distinct queries. -Also, for the queries that limit the result set to one instance, wrapping the result into with the `Optional` keyword is supported. - -If pagination or slicing is applied to a limiting query pagination (and the calculation of the number of available pages), it is applied within the limited result. - -NOTE: Limiting the results in combination with dynamic sorting by using a `Sort` parameter lets you express query methods for the 'K' smallest as well as for the 'K' biggest elements. +include::repositories-paging-sorting.adoc[] [[repositories.collections-and-iterables]] === Repository Methods Returning Collections or Iterables diff --git a/src/main/java/org/springframework/data/domain/KeysetScrollPosition.java b/src/main/java/org/springframework/data/domain/KeysetScrollPosition.java new file mode 100644 index 0000000000..9063b6f3f8 --- /dev/null +++ b/src/main/java/org/springframework/data/domain/KeysetScrollPosition.java @@ -0,0 +1,144 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * A {@link ScrollPosition} based on the last seen keyset. Keyset scrolling must be associated with a {@link Sort + * well-defined sort} to be able to extract the keyset when resuming scrolling within the sorted result set. + * + * @author Mark Paluch + * @since 3.1 + */ +public final class KeysetScrollPosition implements ScrollPosition { + + private static final KeysetScrollPosition initial = new KeysetScrollPosition(Collections.emptyMap(), + Direction.Forward); + + private final Map keys; + + private final Direction direction; + + private KeysetScrollPosition(Map keys, Direction direction) { + this.keys = keys; + this.direction = direction; + } + + /** + * Creates a new initial {@link KeysetScrollPosition} to start scrolling using keyset-queries. + * + * @return a new initial {@link KeysetScrollPosition} to start scrolling using keyset-queries. + */ + public static KeysetScrollPosition initial() { + return initial; + } + + /** + * Creates a new {@link KeysetScrollPosition} from a keyset. + * + * @param keys must not be {@literal null}. + * @return a new {@link KeysetScrollPosition} for the given keyset. + */ + public static KeysetScrollPosition of(Map keys) { + return of(keys, Direction.Forward); + } + + /** + * Creates a new {@link KeysetScrollPosition} from a keyset and {@link Direction}. + * + * @param keys must not be {@literal null}. + * @param direction must not be {@literal null}. + * @return a new {@link KeysetScrollPosition} for the given keyset and {@link Direction}. + */ + public static KeysetScrollPosition of(Map keys, Direction direction) { + + Assert.notNull(keys, "Keys must not be null"); + Assert.notNull(direction, "Direction must not be null"); + + if (keys.isEmpty()) { + return initial(); + } + + return new KeysetScrollPosition(Collections.unmodifiableMap(new LinkedHashMap<>(keys)), direction); + } + + @Override + public boolean isInitial() { + return keys.isEmpty(); + } + + /** + * @return the keyset. + */ + public Map getKeys() { + return keys; + } + + /** + * @return the scroll direction. + */ + public Direction getDirection() { + return direction; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + KeysetScrollPosition that = (KeysetScrollPosition) o; + return ObjectUtils.nullSafeEquals(keys, that.keys) && direction == that.direction; + } + + @Override + public int hashCode() { + + int result = 17; + + result += 31 * ObjectUtils.nullSafeHashCode(keys); + result += 31 * ObjectUtils.nullSafeHashCode(direction); + + return result; + } + + @Override + public String toString() { + return String.format("KeysetScrollPosition [%s, %s]", direction, keys); + } + + /** + * Keyset scrolling direction. + */ + enum Direction { + + /** + * Forward (default) direction to scroll from the beginning of the results to their end. + */ + Forward, + + /** + * Backward direction to scroll from the end of the results to their beginning. + */ + Backward; + } +} diff --git a/src/main/java/org/springframework/data/domain/OffsetScrollPosition.java b/src/main/java/org/springframework/data/domain/OffsetScrollPosition.java new file mode 100644 index 0000000000..07000c7b8a --- /dev/null +++ b/src/main/java/org/springframework/data/domain/OffsetScrollPosition.java @@ -0,0 +1,127 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import java.util.function.IntFunction; + +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * A {@link ScrollPosition} based on the offsets within query results. + * + * @author Mark Paluch + * @since 3.1 + */ +public final class OffsetScrollPosition implements ScrollPosition { + + private static final OffsetScrollPosition initial = new OffsetScrollPosition(0); + + private final long offset; + + private OffsetScrollPosition(long offset) { + this.offset = offset; + } + + /** + * Creates a new initial {@link OffsetScrollPosition} to start scrolling using offset/limit. + * + * @return a new initial {@link OffsetScrollPosition} to start scrolling using offset/limit. + */ + public static OffsetScrollPosition initial() { + return initial; + } + + /** + * Creates a new {@link OffsetScrollPosition} from an {@code offset}. + * + * @param offset + * @return a new {@link OffsetScrollPosition} with the given {@code offset}. + */ + public static OffsetScrollPosition of(long offset) { + + if (offset == 0) { + return initial(); + } + + return new OffsetScrollPosition(offset); + } + + /** + * Returns the {@link IntFunction position function} to calculate. + * + * @param startOffset the start offset to be used. Must not be negative. + * @return the offset-based position function. + */ + public static IntFunction positionFunction(long startOffset) { + + Assert.isTrue(startOffset >= 0, "Start offset must not be negative"); + + return startOffset == 0 ? OffsetPositionFunction.ZERO : new OffsetPositionFunction(startOffset); + } + + @Override + public boolean isInitial() { + return offset == 0; + } + + /** + * @return the offset. + */ + public long getOffset() { + return offset; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + OffsetScrollPosition that = (OffsetScrollPosition) o; + return offset == that.offset; + } + + @Override + public int hashCode() { + + int result = 17; + + result += 31 * ObjectUtils.nullSafeHashCode(offset); + + return result; + } + + @Override + public String toString() { + return String.format("OffsetScrollPosition [%s]", offset); + } + + private record OffsetPositionFunction(long startOffset) implements IntFunction { + + static final OffsetPositionFunction ZERO = new OffsetPositionFunction(0); + + @Override + public OffsetScrollPosition apply(int offset) { + + if (offset < 0) { + throw new IndexOutOfBoundsException(offset); + } + + return of(startOffset + offset + 1); + } + } +} diff --git a/src/main/java/org/springframework/data/domain/Pageable.java b/src/main/java/org/springframework/data/domain/Pageable.java index 7fd873f721..4486394de6 100644 --- a/src/main/java/org/springframework/data/domain/Pageable.java +++ b/src/main/java/org/springframework/data/domain/Pageable.java @@ -161,4 +161,20 @@ default Optional toOptional() { return isUnpaged() ? Optional.empty() : Optional.of(this); } + /** + * Returns an {@link OffsetScrollPosition} from this pageable if the page request {@link #isPaged() is paged}. + * + * @return + * @throws IllegalStateException if the request is {@link #isUnpaged()} + * @since 3.1 + */ + default OffsetScrollPosition toScrollPosition() { + + if (isUnpaged()) { + throw new IllegalStateException("Cannot create OffsetScrollPosition from an unpaged instance"); + } + + return OffsetScrollPosition.of(getOffset()); + } + } diff --git a/src/main/java/org/springframework/data/domain/Scroll.java b/src/main/java/org/springframework/data/domain/Scroll.java new file mode 100644 index 0000000000..82271fc672 --- /dev/null +++ b/src/main/java/org/springframework/data/domain/Scroll.java @@ -0,0 +1,167 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; +import java.util.function.IntFunction; + +import org.springframework.data.util.Streamable; + +/** + * A scroll of data consumed from an underlying query result. A scroll is similar to {@link Slice} in the sense that it + * contains a subset of the actual query results for easier consumption of large result sets. The scroll is less + * opinionated about the actual data retrieval, whether the query has used index/offset, keyset-based pagination or + * cursor resume tokens. + * + * @author Mark Paluch + * @since 3.1 + * @see ScrollPosition + */ +public interface Scroll extends Streamable { + + /** + * Construct a {@link Scroll}. + * + * @param items the list of data. + * @param positionFunction the list of data. + * @return the {@link Scroll}. + * @param + */ + static Scroll from(List items, IntFunction positionFunction) { + return new ScrollImpl<>(items, positionFunction, false); + } + + /** + * Construct a {@link Scroll}. + * + * @param items the list of data. + * @param positionFunction the list of data. + * @param hasNext + * @return the {@link Scroll}. + * @param + */ + static Scroll from(List items, IntFunction positionFunction, boolean hasNext) { + return new ScrollImpl<>(items, positionFunction, hasNext); + } + + /** + * Returns the number of elements in this scroll. + * + * @return the number of elements in this scroll. + */ + int size(); + + /** + * Returns {@code true} if this scroll contains no elements. + * + * @return {@code true} if this scroll contains no elements + */ + boolean isEmpty(); + + /** + * Returns the scroll content as {@link List}. + * + * @return + */ + List getContent(); + + /** + * Returns whether the current scroll is the last one. + * + * @return + */ + default boolean isLast() { + return !hasNext(); + } + + /** + * Returns if there is a next scroll. + * + * @return if there is a next scroll window. + */ + boolean hasNext(); + + /** + * Returns the {@link ScrollPosition} at {@code index}. + * + * @param index + * @return + * @throws IndexOutOfBoundsException if the index is out of range ({@code index < 0 || index >= size()}). + */ + ScrollPosition positionAt(int index); + + /** + * Returns the {@link ScrollPosition} for {@code object}. + * + * @param object + * @return + * @throws NoSuchElementException if the object is not part of the result. + */ + default ScrollPosition positionAt(T object) { + + int index = getContent().indexOf(object); + + if (index == -1) { + throw new NoSuchElementException(); + } + + return positionAt(index); + } + + // TODO: First and last seem to conflict with first/last scroll or first/last position of elements. + // these methods should rather express the position of the first element within this scroll and the scroll position + // to be used to get the next Scroll. + /** + * Returns the first {@link ScrollPosition} or throw {@link NoSuchElementException} if the list is empty. + * + * @return the first {@link ScrollPosition}. + * @throws NoSuchElementException if this result is empty. + */ + default ScrollPosition firstPosition() { + + if (size() == 0) { + throw new NoSuchElementException(); + } + + return positionAt(0); + } + + /** + * Returns the last {@link ScrollPosition} or throw {@link NoSuchElementException} if the list is empty. + * + * @return the last {@link ScrollPosition}. + * @throws NoSuchElementException if this result is empty. + */ + default ScrollPosition lastPosition() { + + if (size() == 0) { + throw new NoSuchElementException(); + } + + return positionAt(size() - 1); + } + + /** + * Returns a new {@link Scroll} with the content of the current one mapped by the given {@code converter}. + * + * @param converter must not be {@literal null}. + * @return a new {@link Scroll} with the content of the current one mapped by the given {@code converter}. + */ + Scroll map(Function converter); + +} diff --git a/src/main/java/org/springframework/data/domain/ScrollImpl.java b/src/main/java/org/springframework/data/domain/ScrollImpl.java new file mode 100644 index 0000000000..538849670b --- /dev/null +++ b/src/main/java/org/springframework/data/domain/ScrollImpl.java @@ -0,0 +1,119 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; +import java.util.function.IntFunction; +import java.util.stream.Collectors; + +import org.jetbrains.annotations.NotNull; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * Default {@link Scroll} implementation. + * + * @author Mark Paluch + * @since 3.1 + */ +class ScrollImpl implements Scroll { + + private final List items; + private final IntFunction positionFunction; + + private final boolean hasNext; + + ScrollImpl(List items, IntFunction positionFunction, boolean hasNext) { + + Assert.notNull(items, "List of items must not be null"); + Assert.notNull(positionFunction, "Position function must not be null"); + + this.items = items; + this.positionFunction = positionFunction; + this.hasNext = hasNext; + } + + @Override + public int size() { + return items.size(); + } + + @Override + public boolean isEmpty() { + return items.isEmpty(); + } + + @Override + public List getContent() { + return items; + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public ScrollPosition positionAt(int index) { + + if (index < 0 || index >= size()) { + throw new IndexOutOfBoundsException(index); + } + + return positionFunction.apply(index); + } + + @Override + public Scroll map(Function converter) { + + Assert.notNull(converter, "Function must not be null"); + + return new ScrollImpl<>(stream().map(converter).collect(Collectors.toList()), positionFunction, hasNext); + } + + @NotNull + @Override + public Iterator iterator() { + return items.iterator(); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ScrollImpl that = (ScrollImpl) o; + return ObjectUtils.nullSafeEquals(items, that.items) + && ObjectUtils.nullSafeEquals(positionFunction, that.positionFunction) + && ObjectUtils.nullSafeEquals(hasNext, that.hasNext); + } + + @Override + public int hashCode() { + int result = ObjectUtils.nullSafeHashCode(items); + result = 31 * result + ObjectUtils.nullSafeHashCode(positionFunction); + result = 31 * result + ObjectUtils.nullSafeHashCode(hasNext); + return result; + } + + @Override + public String toString() { + return "Scroll " + items; + } +} diff --git a/src/main/java/org/springframework/data/domain/ScrollPosition.java b/src/main/java/org/springframework/data/domain/ScrollPosition.java new file mode 100644 index 0000000000..0eeba7530a --- /dev/null +++ b/src/main/java/org/springframework/data/domain/ScrollPosition.java @@ -0,0 +1,33 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +/** + * Interface to specify a position within a total query result. Scroll positions are used to start scrolling from the + * beginning of a query result or to resume scrolling from a given position within the query result. + * + * @author Mark Paluch + * @since 3.1 + */ +public interface ScrollPosition { + + /** + * Returns whether the current scroll position is the initial one. + * + * @return + */ + boolean isInitial(); +} diff --git a/src/main/java/org/springframework/data/repository/query/FluentQuery.java b/src/main/java/org/springframework/data/repository/query/FluentQuery.java index 909a0d7169..7d3513bf50 100644 --- a/src/main/java/org/springframework/data/repository/query/FluentQuery.java +++ b/src/main/java/org/springframework/data/repository/query/FluentQuery.java @@ -26,6 +26,8 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.lang.Nullable; @@ -44,10 +46,23 @@ public interface FluentQuery { * @param sort the {@link Sort} specification to sort the results by, may be {@link Sort#unsorted()}, must not be * {@literal null}. * @return a new instance of {@link FluentQuery}. - * @throws IllegalArgumentException if resultType is {@code null}. + * @throws IllegalArgumentException if {@code sort} is {@code null}. */ FluentQuery sortBy(Sort sort); + /** + * Define the query limit. + * + * @param limit the limit to apply to the query to limit results. Must not be negative. + * @return a new instance of {@link FluentQuery}. + * @throws IllegalArgumentException if {@code limit} is less than zero. + * @throws UnsupportedOperationException if not supported by the underlying implementation. + * @since 3.1 + */ + default FluentQuery limit(int limit) { + throw new UnsupportedOperationException("Limit not supported"); + } + /** * Define the target type the result should be mapped to. Skip this step if you are only interested in the original * domain type. @@ -55,7 +70,7 @@ public interface FluentQuery { * @param resultType must not be {@code null}. * @param result type. * @return a new instance of {@link FluentQuery}. - * @throws IllegalArgumentException if resultType is {@code null}. + * @throws IllegalArgumentException if {@code resultType} is {@code null}. */ FluentQuery as(Class resultType); @@ -64,7 +79,7 @@ public interface FluentQuery { * * @param properties must not be {@code null}. * @return a new instance of {@link FluentQuery}. - * @throws IllegalArgumentException if fields is {@code null}. + * @throws IllegalArgumentException if {@code properties} is {@code null}. */ default FluentQuery project(String... properties) { return project(Arrays.asList(properties)); @@ -75,7 +90,7 @@ default FluentQuery project(String... properties) { * * @param properties must not be {@code null}. * @return a new instance of {@link FluentQuery}. - * @throws IllegalArgumentException if fields is {@code null}. + * @throws IllegalArgumentException if {@code properties} is {@code null}. */ FluentQuery project(Collection properties); @@ -90,6 +105,11 @@ interface FetchableFluentQuery extends FluentQuery { @Override FetchableFluentQuery sortBy(Sort sort); + @Override + default FetchableFluentQuery limit(int limit) { + throw new UnsupportedOperationException("Limit not supported"); + } + @Override FetchableFluentQuery as(Class resultType); @@ -144,12 +164,27 @@ default Optional first() { */ List all(); + /** + * Get all matching elements as {@link Scroll} to start result scrolling or resume scrolling at + * {@code scrollPosition}. + * + * @param scrollPosition must not be {@literal null}. + * @return + * @throws IllegalArgumentException if {@code scrollPosition} is {@literal null}. + * @throws UnsupportedOperationException if not supported by the underlying implementation. + * @since 3.1 + */ + default Scroll scroll(ScrollPosition scrollPosition) { + throw new UnsupportedOperationException("Scrolling not supported"); + } + /** * Get a page of matching elements for {@link Pageable}. * * @param pageable the pageable to request a paged result, can be {@link Pageable#unpaged()}, must not be * {@literal null}. The given {@link Pageable} will override any previously specified {@link Sort sort} if - * the {@link Sort} object is not {@link Sort#isUnsorted()}. + * the {@link Sort} object is not {@link Sort#isUnsorted()}. Any potentially specified {@link #limit(int)} + * will be overridden by {@link Pageable#getPageSize()}. * @return */ Page page(Pageable pageable); @@ -187,6 +222,11 @@ interface ReactiveFluentQuery extends FluentQuery { @Override ReactiveFluentQuery sortBy(Sort sort); + @Override + default ReactiveFluentQuery limit(int limit) { + throw new UnsupportedOperationException("Limit not supported"); + } + @Override ReactiveFluentQuery as(Class resultType); @@ -220,12 +260,27 @@ default ReactiveFluentQuery project(String... properties) { */ Flux all(); + /** + * Get all matching elements as {@link Scroll} to start result scrolling or resume scrolling at + * {@code scrollPosition}. + * + * @param scrollPosition must not be {@literal null}. + * @return + * @throws IllegalArgumentException if {@code scrollPosition} is {@literal null}. + * @throws UnsupportedOperationException if not supported by the underlying implementation. + * @since 3.1 + */ + default Mono> scroll(ScrollPosition scrollPosition) { + throw new UnsupportedOperationException("Scrolling not supported"); + } + /** * Get a page of matching elements for {@link Pageable}. * * @param pageable the pageable to request a paged result, can be {@link Pageable#unpaged()}, must not be * {@literal null}. The given {@link Pageable} will override any previously specified {@link Sort sort} if - * the {@link Sort} object is not {@link Sort#isUnsorted()}. + * the {@link Sort} object is not {@link Sort#isUnsorted()}. Any potentially specified {@link #limit(int)} + * will be overridden by {@link Pageable#getPageSize()}. * @return */ Mono> page(Pageable pageable); diff --git a/src/main/java/org/springframework/data/repository/query/Parameter.java b/src/main/java/org/springframework/data/repository/query/Parameter.java index 304772e50c..c751db67c5 100644 --- a/src/main/java/org/springframework/data/repository/query/Parameter.java +++ b/src/main/java/org/springframework/data/repository/query/Parameter.java @@ -26,6 +26,7 @@ import org.springframework.core.MethodParameter; import org.springframework.core.ResolvableType; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.repository.util.ClassUtils; import org.springframework.data.repository.util.QueryExecutionConverters; @@ -57,7 +58,7 @@ public class Parameter { static { - List> types = new ArrayList<>(Arrays.asList(Pageable.class, Sort.class)); + List> types = new ArrayList<>(Arrays.asList(ScrollPosition.class, Pageable.class, Sort.class)); // consider Kotlin Coroutines Continuation a special parameter. That parameter is synthetic and should not get // bound to any query. @@ -192,6 +193,16 @@ public String toString() { return format("%s:%s", isNamedParameter() ? getName() : "#" + getIndex(), getType().getName()); } + /** + * Returns whether the {@link Parameter} is a {@link ScrollPosition} parameter. + * + * @return + * @since 3.1 + */ + boolean isScrollPosition() { + return ScrollPosition.class.isAssignableFrom(getType()); + } + /** * Returns whether the {@link Parameter} is a {@link Pageable} parameter. * diff --git a/src/main/java/org/springframework/data/repository/query/ParameterAccessor.java b/src/main/java/org/springframework/data/repository/query/ParameterAccessor.java index f5ccd7ec3d..369c781a56 100644 --- a/src/main/java/org/springframework/data/repository/query/ParameterAccessor.java +++ b/src/main/java/org/springframework/data/repository/query/ParameterAccessor.java @@ -18,6 +18,7 @@ import java.util.Iterator; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.lang.Nullable; @@ -29,6 +30,14 @@ */ public interface ParameterAccessor extends Iterable { + /** + * Returns the {@link ScrollPosition} of the parameters, if available. Returns {@code null} otherwise. + * + * @return + */ + @Nullable + ScrollPosition getScrollPosition(); + /** * Returns the {@link Pageable} of the parameters, if available. Returns {@link Pageable#unpaged()} otherwise. * diff --git a/src/main/java/org/springframework/data/repository/query/Parameters.java b/src/main/java/org/springframework/data/repository/query/Parameters.java index 24426f4bd0..523f127995 100644 --- a/src/main/java/org/springframework/data/repository/query/Parameters.java +++ b/src/main/java/org/springframework/data/repository/query/Parameters.java @@ -28,6 +28,7 @@ import org.springframework.core.MethodParameter; import org.springframework.core.ParameterNameDiscoverer; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.util.Lazy; import org.springframework.data.util.Streamable; @@ -42,7 +43,7 @@ */ public abstract class Parameters, T extends Parameter> implements Streamable { - public static final List> TYPES = Arrays.asList(Pageable.class, Sort.class); + public static final List> TYPES = Arrays.asList(ScrollPosition.class, Pageable.class, Sort.class); private static final String PARAM_ON_SPECIAL = format("You must not use @%s on a parameter typed %s or %s", Param.class.getSimpleName(), Pageable.class.getSimpleName(), Sort.class.getSimpleName()); @@ -52,6 +53,7 @@ public abstract class Parameters, T extends Parameter private static final ParameterNameDiscoverer PARAMETER_NAME_DISCOVERER = new DefaultParameterNameDiscoverer(); + private final int scrollPositionIndex; private final int pageableIndex; private final int sortIndex; private final List parameters; @@ -91,6 +93,7 @@ protected Parameters(Method method, Function parameterFactor this.parameters = new ArrayList<>(parameterCount); this.dynamicProjectionIndex = -1; + int scrollPositionIndex = -1; int pageableIndex = -1; int sortIndex = -1; @@ -111,6 +114,10 @@ protected Parameters(Method method, Function parameterFactor this.dynamicProjectionIndex = parameter.getIndex(); } + if (ScrollPosition.class.isAssignableFrom(parameter.getType())) { + scrollPositionIndex = i; + } + if (Pageable.class.isAssignableFrom(parameter.getType())) { pageableIndex = i; } @@ -122,6 +129,7 @@ protected Parameters(Method method, Function parameterFactor parameters.add(parameter); } + this.scrollPositionIndex = scrollPositionIndex; this.pageableIndex = pageableIndex; this.sortIndex = sortIndex; this.bindable = Lazy.of(this::getBindable); @@ -138,6 +146,7 @@ protected Parameters(List originals) { this.parameters = new ArrayList<>(originals.size()); + int scrollPositionIndexTemp = -1; int pageableIndexTemp = -1; int sortIndexTemp = -1; int dynamicProjectionTemp = -1; @@ -147,11 +156,13 @@ protected Parameters(List originals) { T original = originals.get(i); this.parameters.add(original); + scrollPositionIndexTemp = original.isScrollPosition() ? i : -1; pageableIndexTemp = original.isPageable() ? i : -1; sortIndexTemp = original.isSort() ? i : -1; dynamicProjectionTemp = original.isDynamicProjectionParameter() ? i : -1; } + this.scrollPositionIndex = scrollPositionIndexTemp; this.pageableIndex = pageableIndexTemp; this.sortIndex = sortIndexTemp; this.dynamicProjectionIndex = dynamicProjectionTemp; @@ -185,6 +196,27 @@ protected T createParameter(MethodParameter parameter) { return (T) new Parameter(parameter); } + /** + * Returns whether the method the {@link Parameters} was created for contains a {@link ScrollPosition} argument. + * + * @return + * @since 3.1 + */ + public boolean hasScrollPositionParameter() { + return scrollPositionIndex != -1; + } + + /** + * Returns the index of the {@link ScrollPosition} {@link Method} parameter if available. Will return {@literal -1} if + * there is no {@link ScrollPosition} argument in the {@link Method}'s parameter list. + * + * @return the scrollPositionIndex + * @since 3.1 + */ + public int getScrollPositionIndex() { + return scrollPositionIndex; + } + /** * Returns whether the method the {@link Parameters} was created for contains a {@link Pageable} argument. * @@ -195,8 +227,8 @@ public boolean hasPageableParameter() { } /** - * Returns the index of the {@link Pageable} {@link Method} parameter if available. Will return {@literal -1} if - * there is no {@link Pageable} argument in the {@link Method}'s parameter list. + * Returns the index of the {@link Pageable} {@link Method} parameter if available. Will return {@literal -1} if there + * is no {@link Pageable} argument in the {@link Method}'s parameter list. * * @return the pageableIndex */ @@ -205,8 +237,8 @@ public int getPageableIndex() { } /** - * Returns the index of the {@link Sort} {@link Method} parameter if available. Will return {@literal -1} if there - * is no {@link Sort} argument in the {@link Method}'s parameter list. + * Returns the index of the {@link Sort} {@link Method} parameter if available. Will return {@literal -1} if there is + * no {@link Sort} argument in the {@link Method}'s parameter list. * * @return */ @@ -288,7 +320,7 @@ public boolean hasParameterAt(int position) { * @return */ public boolean hasSpecialParameter() { - return hasSortParameter() || hasPageableParameter(); + return hasScrollPositionParameter() || hasSortParameter() || hasPageableParameter(); } /** @@ -315,8 +347,8 @@ public S getBindableParameters() { /** * Returns a bindable parameter with the given index. So for a method with a signature of - * {@code (Pageable pageable, String name)} a call to {@code #getBindableParameter(0)} will return the - * {@link String} parameter. + * {@code (Pageable pageable, String name)} a call to {@code #getBindableParameter(0)} will return the {@link String} + * parameter. * * @param bindableIndex * @return diff --git a/src/main/java/org/springframework/data/repository/query/ParametersParameterAccessor.java b/src/main/java/org/springframework/data/repository/query/ParametersParameterAccessor.java index 0b056142ba..045ab16b1f 100644 --- a/src/main/java/org/springframework/data/repository/query/ParametersParameterAccessor.java +++ b/src/main/java/org/springframework/data/repository/query/ParametersParameterAccessor.java @@ -18,6 +18,7 @@ import java.util.Iterator; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.data.repository.util.QueryExecutionConverters; import org.springframework.data.repository.util.ReactiveWrapperConverters; @@ -91,6 +92,22 @@ protected Object[] getValues() { return this.values; } + @Override + public ScrollPosition getScrollPosition() { + + if (!parameters.hasScrollPositionParameter()) { + + Pageable pageable = getPageable(); + if (pageable.isPaged()) { + return pageable.toScrollPosition(); + } + + return null; + } + + return (ScrollPosition) values[parameters.getScrollPositionIndex()]; + } + @Override public Pageable getPageable() { diff --git a/src/main/java/org/springframework/data/repository/query/QueryMethod.java b/src/main/java/org/springframework/data/repository/query/QueryMethod.java index 9a39f271ce..23db75a5d9 100644 --- a/src/main/java/org/springframework/data/repository/query/QueryMethod.java +++ b/src/main/java/org/springframework/data/repository/query/QueryMethod.java @@ -18,11 +18,14 @@ import static org.springframework.data.repository.util.ClassUtils.*; import java.lang.reflect.Method; +import java.util.Collections; import java.util.Set; import java.util.stream.Stream; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Slice; import org.springframework.data.domain.Sort; import org.springframework.data.projection.ProjectionFactory; @@ -31,6 +34,7 @@ import org.springframework.data.repository.util.QueryExecutionConverters; import org.springframework.data.repository.util.ReactiveWrapperConverters; import org.springframework.data.util.Lazy; +import org.springframework.data.util.ReactiveWrappers; import org.springframework.data.util.TypeInformation; import org.springframework.util.Assert; @@ -92,6 +96,10 @@ public QueryMethod(Method method, RepositoryMetadata metadata, ProjectionFactory } } + if (hasParameterOfType(method, ScrollPosition.class)) { + assertReturnTypeAssignable(method, Collections.singleton(Scroll.class)); + } + Assert.notNull(this.parameters, () -> String.format("Parameters extracted from method '%s' must not be null", method.getName())); @@ -100,6 +108,12 @@ public QueryMethod(Method method, RepositoryMetadata metadata, ProjectionFactory String.format("Paging query needs to have a Pageable parameter; Offending method: %s", method)); } + if (isScrollQuery()) { + + Assert.isTrue(this.parameters.hasScrollPositionParameter() || this.parameters.hasPageableParameter(), + String.format("Scroll query needs to have a ScrollPosition parameter; Offending method: %s", method)); + } + this.domainClass = Lazy.of(() -> { Class repositoryDomainClass = metadata.getDomainType(); @@ -188,6 +202,16 @@ public boolean isCollectionQuery() { return isCollectionQuery.get(); } + /** + * Returns whether the query method will return a {@link Scroll}. + * + * @return + * @since 3.1 + */ + public boolean isScrollQuery() { + return org.springframework.util.ClassUtils.isAssignable(Scroll.class, unwrappedReturnType); + } + /** * Returns whether the query method will return a {@link Slice}. * @@ -268,7 +292,7 @@ public String toString() { private boolean calculateIsCollectionQuery() { - if (isPageQuery() || isSliceQuery()) { + if (isPageQuery() || isSliceQuery() || isScrollQuery()) { return false; } @@ -314,6 +338,10 @@ private static void assertReturnTypeAssignable(Method method, Set> type // TODO: to resolve generics fully we'd need the actual repository interface here TypeInformation returnType = TypeInformation.fromReturnTypeOf(method); + returnType = ReactiveWrappers.isSingleValueType(returnType.getType()) // + ? returnType.getRequiredComponentType() // + : returnType; + returnType = QueryExecutionConverters.isSingleValue(returnType.getType()) // ? returnType.getRequiredComponentType() // : returnType; @@ -324,6 +352,6 @@ private static void assertReturnTypeAssignable(Method method, Set> type } } - throw new IllegalStateException("Method has to have one of the following return types" + types); + throw new IllegalStateException("Method has to have one of the following return types " + types); } } diff --git a/src/main/java/org/springframework/data/repository/query/ResultProcessor.java b/src/main/java/org/springframework/data/repository/query/ResultProcessor.java index b91449dac9..ce1cc2a39b 100644 --- a/src/main/java/org/springframework/data/repository/query/ResultProcessor.java +++ b/src/main/java/org/springframework/data/repository/query/ResultProcessor.java @@ -26,6 +26,7 @@ import org.springframework.core.convert.ConversionService; import org.springframework.core.convert.converter.Converter; import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.data.domain.Scroll; import org.springframework.data.domain.Slice; import org.springframework.data.projection.ProjectionFactory; import org.springframework.data.repository.util.ReactiveWrapperConverters; @@ -143,6 +144,10 @@ public T processResult(@Nullable Object source, Converter pr ChainingConverter converter = ChainingConverter.of(type.getReturnedType(), preparingConverter).and(this.converter); + if (source instanceof Scroll && method.isScrollQuery()) { + return (T) ((Scroll) source).map(converter::convert); + } + if (source instanceof Slice && (method.isPageQuery() || method.isSliceQuery())) { return (T) ((Slice) source).map(converter::convert); } @@ -163,7 +168,7 @@ public T processResult(@Nullable Object source, Converter pr } if (ReactiveWrapperConverters.supports(source.getClass())) { - return (T) ReactiveWrapperConverters.map(source, converter::convert); + return (T) ReactiveWrapperConverters.map(source, it -> processResult(it, preparingConverter)); } return (T) converter.convert(source); diff --git a/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java b/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java index 281c13a742..e14e8c654e 100644 --- a/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java +++ b/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java @@ -36,6 +36,7 @@ import org.springframework.core.convert.support.ConfigurableConversionService; import org.springframework.core.convert.support.DefaultConversionService; import org.springframework.data.domain.Page; +import org.springframework.data.domain.Scroll; import org.springframework.data.domain.Slice; import org.springframework.data.geo.GeoResults; import org.springframework.data.util.CustomCollections; @@ -97,6 +98,7 @@ public abstract class QueryExecutionConverters { ALLOWED_PAGEABLE_TYPES.add(Slice.class); ALLOWED_PAGEABLE_TYPES.add(Page.class); ALLOWED_PAGEABLE_TYPES.add(List.class); + ALLOWED_PAGEABLE_TYPES.add(Scroll.class); WRAPPER_TYPES.add(NullableWrapperToCompletableFutureConverter.getWrapperType()); diff --git a/src/main/java/org/springframework/data/util/CustomCollections.java b/src/main/java/org/springframework/data/util/CustomCollections.java index c0533c275d..21857bf006 100644 --- a/src/main/java/org/springframework/data/util/CustomCollections.java +++ b/src/main/java/org/springframework/data/util/CustomCollections.java @@ -223,7 +223,7 @@ public boolean hasSuperTypeFor(Class type) { /** * Returns whether the current's raw type is one of the given ones. * - * @param candidates must not be {@literal null}. + * @param type must not be {@literal null}. * @return */ public boolean has(Class type) { diff --git a/src/test/java/org/springframework/data/domain/KeysetScrollPositionUnitTests.java b/src/test/java/org/springframework/data/domain/KeysetScrollPositionUnitTests.java new file mode 100644 index 0000000000..6fe28d4598 --- /dev/null +++ b/src/test/java/org/springframework/data/domain/KeysetScrollPositionUnitTests.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import static org.assertj.core.api.Assertions.*; + +import java.util.Collections; + +import org.junit.jupiter.api.Test; +import org.springframework.data.domain.KeysetScrollPosition.Direction; + +/** + * Unit tests for {@link KeysetScrollPosition}. + * + * @author Mark Paluch + */ +class KeysetScrollPositionUnitTests { + + @Test // GH-2151 + void equalsAndHashCode() { + + KeysetScrollPosition foo1 = KeysetScrollPosition.of(Collections.singletonMap("k", "v")); + KeysetScrollPosition foo2 = KeysetScrollPosition.of(Collections.singletonMap("k", "v")); + KeysetScrollPosition bar = KeysetScrollPosition.of(Collections.singletonMap("k", "v"), Direction.Backward); + + assertThat(foo1).isEqualTo(foo2).hasSameClassAs(foo2); + assertThat(foo1).isNotEqualTo(bar).doesNotHaveSameHashCodeAs(bar); + } + +} diff --git a/src/test/java/org/springframework/data/domain/OffsetScrollPositionUnitTests.java b/src/test/java/org/springframework/data/domain/OffsetScrollPositionUnitTests.java new file mode 100644 index 0000000000..a04f57752a --- /dev/null +++ b/src/test/java/org/springframework/data/domain/OffsetScrollPositionUnitTests.java @@ -0,0 +1,50 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import static org.assertj.core.api.Assertions.*; +import static org.springframework.data.domain.OffsetScrollPosition.*; + +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link OffsetScrollPosition}. + * + * @author Mark Paluch + */ +class OffsetScrollPositionUnitTests { + + @Test // GH-2151 + void equalsAndHashCode() { + + OffsetScrollPosition foo1 = OffsetScrollPosition.of(1); + OffsetScrollPosition foo2 = OffsetScrollPosition.of(1); + OffsetScrollPosition bar = OffsetScrollPosition.of(2); + + assertThat(foo1).isEqualTo(foo2).hasSameClassAs(foo2); + assertThat(foo1).isNotEqualTo(bar).doesNotHaveSameHashCodeAs(bar); + } + + @Test // GH-2151 + void shouldCreateCorrectIndexPosition() { + + assertThat(positionFunction(0).apply(0)).isEqualTo(OffsetScrollPosition.of(1)); + assertThat(positionFunction(0).apply(1)).isEqualTo(OffsetScrollPosition.of(2)); + + assertThat(positionFunction(100).apply(0)).isEqualTo(OffsetScrollPosition.of(101)); + assertThat(positionFunction(100).apply(1)).isEqualTo(OffsetScrollPosition.of(102)); + } +} diff --git a/src/test/java/org/springframework/data/domain/PageRequestUnitTests.java b/src/test/java/org/springframework/data/domain/PageRequestUnitTests.java index 0727553d8e..fe0bfe31f6 100755 --- a/src/test/java/org/springframework/data/domain/PageRequestUnitTests.java +++ b/src/test/java/org/springframework/data/domain/PageRequestUnitTests.java @@ -19,7 +19,6 @@ import static org.springframework.data.domain.UnitTestUtils.*; import org.junit.jupiter.api.Test; - import org.springframework.data.domain.Sort.Direction; /** @@ -67,4 +66,12 @@ void rejectsNullSort() { assertThatIllegalArgumentException() // .isThrownBy(() -> PageRequest.of(0, 10, null)); } + + @Test // GH-2151 + void createsOffsetScrollPosition() { + + PageRequest request = PageRequest.of(1, 10); + + assertThat(request.toScrollPosition()).isEqualTo(OffsetScrollPosition.of(10)); + } } diff --git a/src/test/java/org/springframework/data/domain/ScrollUnitTests.java b/src/test/java/org/springframework/data/domain/ScrollUnitTests.java new file mode 100644 index 0000000000..5ffbc3f170 --- /dev/null +++ b/src/test/java/org/springframework/data/domain/ScrollUnitTests.java @@ -0,0 +1,69 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import static org.assertj.core.api.Assertions.*; + +import java.util.List; +import java.util.function.IntFunction; + +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link Scroll}. + * + * @author Mark Paluch + */ +class ScrollUnitTests { + + @Test // GH-2151 + void equalsAndHashCode() { + + IntFunction positionFunction = OffsetScrollPosition.positionFunction(0); + Scroll one = Scroll.from(List.of(1, 2, 3), positionFunction); + Scroll two = Scroll.from(List.of(1, 2, 3), positionFunction); + + assertThat(one).isEqualTo(two).hasSameHashCodeAs(two); + assertThat(one.equals(two)).isTrue(); + + assertThat(Scroll.from(List.of(1, 2, 3), positionFunction, true)).isNotEqualTo(two).doesNotHaveSameHashCodeAs(two); + } + + @Test // GH-2151 + void allowsIteration() { + + Scroll scroll = Scroll.from(List.of(1, 2, 3), OffsetScrollPosition.positionFunction(0)); + + for (Integer integer : scroll) { + assertThat(integer).isBetween(1, 3); + } + } + + @Test // GH-2151 + void shouldCreateCorrectPositions() { + + Scroll scroll = Scroll.from(List.of(1, 2, 3), OffsetScrollPosition.positionFunction(0)); + + assertThat(scroll.firstPosition()).isEqualTo(OffsetScrollPosition.of(1)); + assertThat(scroll.lastPosition()).isEqualTo(OffsetScrollPosition.of(3)); + + // by index + assertThat(scroll.positionAt(1)).isEqualTo(OffsetScrollPosition.of(2)); + + // by object + assertThat(scroll.positionAt(Integer.valueOf(1))).isEqualTo(OffsetScrollPosition.of(1)); + } +} diff --git a/src/test/java/org/springframework/data/repository/query/ParametersParameterAccessorUnitTests.java b/src/test/java/org/springframework/data/repository/query/ParametersParameterAccessorUnitTests.java index 2bb109e9c6..68c7b51684 100755 --- a/src/test/java/org/springframework/data/repository/query/ParametersParameterAccessorUnitTests.java +++ b/src/test/java/org/springframework/data/repository/query/ParametersParameterAccessorUnitTests.java @@ -19,14 +19,17 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.data.domain.OffsetScrollPosition; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; /** * Unit tests for {@link ParametersParameterAccessor}. * * @author Oliver Gierke * @author Greg Turnquist + * @author Mark Paluch */ class ParametersParameterAccessorUnitTests { @@ -75,6 +78,18 @@ void iteratesonlyOverBindableValues() throws Exception { assertThat(accessor.getBindableValue(0)).isEqualTo("Foo"); } + @Test // GH-2151 + void handlesScrollPositionAsAParameterType() throws NoSuchMethodException { + + var method = Sample.class.getMethod("method", ScrollPosition.class, String.class); + var parameters = new DefaultParameters(method); + + var accessor = new ParametersParameterAccessor(parameters, new Object[] { OffsetScrollPosition.of(1), "Foo" }); + + assertThat(accessor).hasSize(1); + assertThat(accessor.getBindableValue(0)).isEqualTo("Foo"); + } + @Test // #2626 void handlesPageRequestAsAParameterType() throws NoSuchMethodException { @@ -93,6 +108,8 @@ interface Sample { void method(Pageable pageable, String string); + void method(ScrollPosition scrollPosition, String string); + void methodWithPageRequest(PageRequest pageRequest, String string); } } diff --git a/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java b/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java index ca76a8c108..a62071f6ed 100755 --- a/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java +++ b/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java @@ -25,9 +25,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.reactivestreams.Publisher; - +import org.springframework.data.domain.OffsetScrollPosition; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; import org.springframework.data.domain.Sort; import org.springframework.test.util.ReflectionTestUtils; @@ -182,6 +183,14 @@ void acceptsCustomPageableParameter() throws Exception { assertThat(parameters.hasPageableParameter()).isTrue(); } + @Test // GH-2151 + void acceptsScrollPositionSubtypeParameter() throws Exception { + + var parameters = getParametersFor("customScrollPosition", OffsetScrollPosition.class); + + assertThat(parameters.hasScrollPositionParameter()).isTrue(); + } + private Parameters getParametersFor(String methodName, Class... parameterTypes) throws SecurityException, NoSuchMethodException { @@ -221,6 +230,8 @@ static interface SampleDao { void methodWithSingle(Single single); Page customPageable(SomePageable pageable); + + Scroll customScrollPosition(OffsetScrollPosition request); } interface SomePageable extends Pageable {} diff --git a/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java b/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java index 8aff2582b3..7d76e1cf53 100755 --- a/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java +++ b/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java @@ -19,6 +19,7 @@ import io.vavr.collection.Seq; import io.vavr.control.Option; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import java.io.Serializable; @@ -31,6 +32,8 @@ import org.junit.jupiter.api.Test; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Slice; import org.springframework.data.projection.ProjectionFactory; import org.springframework.data.projection.SpelAwareProxyProjectionFactory; @@ -88,6 +91,48 @@ void doesNotConsiderPageMethodCollectionQuery() throws Exception { assertThat(queryMethod.isCollectionQuery()).isFalse(); } + @Test // GH-2151 + void supportsImperativecursorQueries() throws Exception { + var method = SampleRepository.class.getMethod("cursorWindow", ScrollPosition.class); + var queryMethod = new QueryMethod(method, metadata, factory); + + assertThat(queryMethod.isPageQuery()).isFalse(); + assertThat(queryMethod.isScrollQuery()).isTrue(); + assertThat(queryMethod.isCollectionQuery()).isFalse(); + } + + @Test // GH-2151 + void supportsReactiveCursorQueries() throws Exception { + var method = SampleRepository.class.getMethod("reactiveCursorWindow", ScrollPosition.class); + var queryMethod = new QueryMethod(method, metadata, factory); + assertThat(queryMethod.isPageQuery()).isFalse(); + + assertThat(queryMethod.isScrollQuery()).isTrue(); + assertThat(queryMethod.isCollectionQuery()).isFalse(); + } + + @Test // GH-2151 + void rejectsInvalidReactiveCursorQueries() throws Exception { + var method = SampleRepository.class.getMethod("invalidReactiveCursorWindow", ScrollPosition.class); + + assertThatIllegalStateException().isThrownBy(() -> new QueryMethod(method, metadata, factory)); + } + + @Test // GH-2151 + void rejectsCursorWindowMethodWithoutPageable() throws Exception { + var method = SampleRepository.class.getMethod("cursorWindowWithoutScrollPosition"); + + assertThatIllegalArgumentException().isThrownBy(() -> new QueryMethod(method, metadata, factory)); + } + + @Test // GH-2151 + void rejectsCursorWindowMethodWithInvalidReturnType() throws Exception { + + var method = SampleRepository.class.getMethod("cursorWindowMethodWithInvalidReturnType", ScrollPosition.class); + + assertThatIllegalStateException().isThrownBy(() -> new QueryMethod(method, metadata, factory)); + } + @Test // DATACMNS-171 void detectsAnEntityBeingReturned() throws Exception { @@ -305,6 +350,16 @@ interface SampleRepository extends Repository { Mono> reactiveSlice(); ImmutableList returnsEclipseCollection(); + + Scroll cursorWindow(ScrollPosition cursorRequest); + + Mono> reactiveCursorWindow(ScrollPosition cursorRequest); + + Flux> invalidReactiveCursorWindow(ScrollPosition cursorRequest); + + Page cursorWindowMethodWithInvalidReturnType(ScrollPosition cursorRequest); + + Scroll cursorWindowWithoutScrollPosition(); } class User { diff --git a/src/test/java/org/springframework/data/repository/query/SimpleParameterAccessorUnitTests.java b/src/test/java/org/springframework/data/repository/query/SimpleParameterAccessorUnitTests.java index e9e504ecb4..f17f7264ac 100755 --- a/src/test/java/org/springframework/data/repository/query/SimpleParameterAccessorUnitTests.java +++ b/src/test/java/org/springframework/data/repository/query/SimpleParameterAccessorUnitTests.java @@ -19,23 +19,27 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.data.domain.OffsetScrollPosition; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; /** * Unit tests for {@link ParametersParameterAccessor}. * * @author Oliver Gierke + * @author Mark Paluch */ class SimpleParameterAccessorUnitTests { - Parameters parameters, sortParameters, pageableParameters; + Parameters parameters, cursorRequestParameters, sortParameters, pageableParameters; @BeforeEach void setUp() throws SecurityException, NoSuchMethodException { parameters = new DefaultParameters(Sample.class.getMethod("sample", String.class)); + cursorRequestParameters = new DefaultParameters(Sample.class.getMethod("sample", ScrollPosition.class)); sortParameters = new DefaultParameters(Sample.class.getMethod("sample1", String.class, Sort.class)); pageableParameters = new DefaultParameters(Sample.class.getMethod("sample2", String.class, Pageable.class)); } @@ -75,6 +79,16 @@ void returnsNullForPageableAndSortIfNoneAvailable() throws Exception { assertThat(accessor.getSort().isSorted()).isFalse(); } + @Test // GH-2151 + void returnsScrollPositionIfAvailable() { + + var cursorRequest = OffsetScrollPosition.of(1); + ParameterAccessor accessor = new ParametersParameterAccessor(cursorRequestParameters, + new Object[] { cursorRequest }); + + assertThat(accessor.getScrollPosition()).isEqualTo(cursorRequest); + } + @Test void returnsSortIfAvailable() { @@ -110,6 +124,8 @@ interface Sample { void sample(String firstname); + void sample(ScrollPosition scrollPosition); + void sample1(String firstname, Sort sort); void sample2(String firstname, Pageable pageable); From 74df309cc6dd57286ab1dd22cad760007c46e774 Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Wed, 15 Mar 2023 09:23:53 +0100 Subject: [PATCH 3/4] Add `WindowIterator` and rename `Scroll` to `Window`. The intend of WindowIterator is to support users who need to iterate multiple windows. It keeps track of the position and loads the next window if needed so that the user does not have to interact with the position at all. Also remove the Window methods to get the frist/last position and enforce the index based variant. Update the documentation to make use of the newly introduced API. --- src/main/asciidoc/repositories-scrolling.adoc | 73 ++++++---- .../data/domain/{Scroll.java => Window.java} | 78 +++-------- .../{ScrollImpl.java => WindowImpl.java} | 12 +- .../data/domain/WindowIterator.java | 117 ++++++++++++++++ .../data/repository/query/FluentQuery.java | 10 +- .../data/repository/query/QueryMethod.java | 8 +- .../repository/query/ResultProcessor.java | 6 +- .../util/QueryExecutionConverters.java | 4 +- .../data/domain/WindowIteratorUnitTests.java | 131 ++++++++++++++++++ ...ollUnitTests.java => WindowUnitTests.java} | 25 ++-- .../repository/query/ParametersUnitTests.java | 4 +- .../query/QueryMethodUnitTests.java | 10 +- 12 files changed, 357 insertions(+), 121 deletions(-) rename src/main/java/org/springframework/data/domain/{Scroll.java => Window.java} (51%) rename src/main/java/org/springframework/data/domain/{ScrollImpl.java => WindowImpl.java} (89%) create mode 100644 src/main/java/org/springframework/data/domain/WindowIterator.java create mode 100644 src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java rename src/test/java/org/springframework/data/domain/{ScrollUnitTests.java => WindowUnitTests.java} (66%) diff --git a/src/main/asciidoc/repositories-scrolling.adoc b/src/main/asciidoc/repositories-scrolling.adoc index 85c81f369d..3e06ed7795 100644 --- a/src/main/asciidoc/repositories-scrolling.adoc +++ b/src/main/asciidoc/repositories-scrolling.adoc @@ -6,8 +6,36 @@ Scrolling consists of a stable sort, a scroll type (Offset- or Keyset-based scro You can define simple sorting expressions by using property names and define static result limiting using the <> through query derivation. You can concatenate expressions to collect multiple criteria into one expression. -Scroll queries return a `Scroll` that allows obtaining the scroll position to resume to obtain the next `Scroll` until your application has consumed the entire query result. -Similar to consuming a Java `Iterator>` by obtaining the next batch of results, query result scrolling lets you access the next `ScrollPosition` through `Scroll.lastScrollPosition()`. +Scroll queries return a `Window` that allows obtaining the scroll position to resume to obtain the next `Window` until your application has consumed the entire query result. +Similar to consuming a Java `Iterator>` by obtaining the next batch of results, query result scrolling lets you access the a `ScrollPosition` through `Window.positionAt(...)`. + +[source,java] +---- +Window users = repository.findFirst10ByLastnameOrderByFirstname("Doe", OffsetScrollPosition.initial()); +do { + + for (User u : users) { + // consume the user + } + + // obtain the next Scroll + users = repository.findFirst10ByLastnameOrderByFirstname("Doe", users.positionAt(users.size() - 1)); +} while (!users.isEmpty() && !users.isLast()); +---- + +`WindowIterator` provides a specialized implementation that simplifies consumption of multiple `Window` instances by removing the need to check for the presence of a next `Window` and applying the `ScrollPosition`. + +[source,java] +---- +WindowIterator users = WindowIterator.of(position -> repository.findFirst10ByLastnameOrderByFirstname("Doe", position)) + .startingAt(OffsetScrollPosition.initial()); + +while (users.hasNext()) { + users.next().forEach(user -> { + // consume the user + }); +} +---- [[repositories.scrolling.offset]] ===== Scrolling using Offset @@ -22,21 +50,13 @@ However, most databases require materializing the full query result before your ---- interface UserRepository extends Repository { - Scroll findFirst10ByLastnameOrderByFirstname(String lastname, OffsetScrollPosition position); + Window findFirst10ByLastnameOrderByFirstname(String lastname, OffsetScrollPosition position); } -Scroll users = repository.findFirst10ByLastnameOrderByFirstname("Doe", OffsetScrollPosition.initial()); - -do { - - for (User u : users) { - // consume the user - } - - // obtain the next Scroll - users = repository.findFirst10ByLastnameOrderByFirstname("Doe", users.lastScrollPosition()); -} while (!users.isEmpty() && !users.isLast()); +WindowIterator users = WindowIterator.of(position -> repository.findFirst10ByLastnameOrderByFirstname("Doe", position)) + .startingAt(OffsetScrollPosition.initial()); <1> ---- +<1> Start from the initial offset at position 0; ==== [[repositories.scrolling.keyset]] @@ -50,31 +70,30 @@ This approach maintains a set of keys to resume scrolling by passing keys into t The core idea of Keyset-Filtering is to start retrieving results using a stable sorting order. Once you want to scroll to the next chunk, you obtain a `ScrollPosition` that is used to reconstruct the position within the sorted result. -The `ScrollPosition` captures the keyset of the last entity within the current `Scroll`. +The `ScrollPosition` captures the keyset of the last entity within the current `Window`. To run the query, reconstruction rewrites the criteria clause to include all sort fields and the primary key so that the database can leverage potential indexes to run the query. The database needs only constructing a much smaller result from the given keyset position without the need to fully materialize a large result and then skipping results until reaching a particular offset. +[WARNING] +==== +Keyset-Filtering requires properties used for sorting to be non nullable. +This limitation applies due to the store specific `null` value handling of comparison operators as well as the need to run queries against an indexed source. +Keyset-Filtering on nullable properties will lead to unexpected results. +==== + .Using `KeysetScrollPosition` with Repository Query Methods ==== [source,java] ---- interface UserRepository extends Repository { - Scroll findFirst10ByLastnameOrderByFirstname(String lastname, KeysetScrollPosition position); + Window findFirst10ByLastnameOrderByFirstname(String lastname, KeysetScrollPosition position); } -Scroll users = repository.findFirst10ByLastnameOrderByFirstname("Doe", KeysetScrollPosition.initial()); - -do { - - for (User u : users) { - // consume the user - } - - // obtain the next Scroll - users = repository.findFirst10ByLastnameOrderByFirstname("Doe", users.lastScrollPosition()); -} while (!users.isEmpty() && !users.isLast()); +WindowIterator users = WindowIterator.of(position -> repository.findFirst10ByLastnameOrderByFirstname("Doe", position)) + .startingAt(KeysetScrollPosition.initial()); <1> ---- +<1> Start at the very beginning and do not apply additional filtering. ==== Keyset-Filtering works best when your database contains an index that matches the sort fields, hence a static sort works well. diff --git a/src/main/java/org/springframework/data/domain/Scroll.java b/src/main/java/org/springframework/data/domain/Window.java similarity index 51% rename from src/main/java/org/springframework/data/domain/Scroll.java rename to src/main/java/org/springframework/data/domain/Window.java index 82271fc672..d27e2249e4 100644 --- a/src/main/java/org/springframework/data/domain/Scroll.java +++ b/src/main/java/org/springframework/data/domain/Window.java @@ -23,65 +23,66 @@ import org.springframework.data.util.Streamable; /** - * A scroll of data consumed from an underlying query result. A scroll is similar to {@link Slice} in the sense that it - * contains a subset of the actual query results for easier consumption of large result sets. The scroll is less + * A set of data consumed from an underlying query result. A {@link Window} is similar to {@link Slice} in the sense + * that it contains a subset of the actual query results for easier consumption of large result sets. The window is less * opinionated about the actual data retrieval, whether the query has used index/offset, keyset-based pagination or * cursor resume tokens. * * @author Mark Paluch + * @author Christoph Strobl * @since 3.1 * @see ScrollPosition */ -public interface Scroll extends Streamable { +public interface Window extends Streamable { /** - * Construct a {@link Scroll}. + * Construct a {@link Window}. * * @param items the list of data. * @param positionFunction the list of data. - * @return the {@link Scroll}. + * @return the {@link Window}. * @param */ - static Scroll from(List items, IntFunction positionFunction) { - return new ScrollImpl<>(items, positionFunction, false); + static Window from(List items, IntFunction positionFunction) { + return new WindowImpl<>(items, positionFunction, false); } /** - * Construct a {@link Scroll}. + * Construct a {@link Window}. * * @param items the list of data. * @param positionFunction the list of data. * @param hasNext - * @return the {@link Scroll}. + * @return the {@link Window}. * @param */ - static Scroll from(List items, IntFunction positionFunction, boolean hasNext) { - return new ScrollImpl<>(items, positionFunction, hasNext); + static Window from(List items, IntFunction positionFunction, boolean hasNext) { + return new WindowImpl<>(items, positionFunction, hasNext); } /** - * Returns the number of elements in this scroll. + * Returns the number of elements in this window. * - * @return the number of elements in this scroll. + * @return the number of elements in this window. */ int size(); /** - * Returns {@code true} if this scroll contains no elements. + * Returns {@code true} if this window contains no elements. * - * @return {@code true} if this scroll contains no elements + * @return {@code true} if this window contains no elements */ boolean isEmpty(); /** - * Returns the scroll content as {@link List}. + * Returns the windows content as {@link List}. * * @return */ List getContent(); /** - * Returns whether the current scroll is the last one. + * Returns whether the current window is the last one. * * @return */ @@ -90,9 +91,9 @@ default boolean isLast() { } /** - * Returns if there is a next scroll. + * Returns if there is a next window. * - * @return if there is a next scroll window. + * @return if there is a next window window. */ boolean hasNext(); @@ -123,45 +124,12 @@ default ScrollPosition positionAt(T object) { return positionAt(index); } - // TODO: First and last seem to conflict with first/last scroll or first/last position of elements. - // these methods should rather express the position of the first element within this scroll and the scroll position - // to be used to get the next Scroll. /** - * Returns the first {@link ScrollPosition} or throw {@link NoSuchElementException} if the list is empty. - * - * @return the first {@link ScrollPosition}. - * @throws NoSuchElementException if this result is empty. - */ - default ScrollPosition firstPosition() { - - if (size() == 0) { - throw new NoSuchElementException(); - } - - return positionAt(0); - } - - /** - * Returns the last {@link ScrollPosition} or throw {@link NoSuchElementException} if the list is empty. - * - * @return the last {@link ScrollPosition}. - * @throws NoSuchElementException if this result is empty. - */ - default ScrollPosition lastPosition() { - - if (size() == 0) { - throw new NoSuchElementException(); - } - - return positionAt(size() - 1); - } - - /** - * Returns a new {@link Scroll} with the content of the current one mapped by the given {@code converter}. + * Returns a new {@link Window} with the content of the current one mapped by the given {@code converter}. * * @param converter must not be {@literal null}. - * @return a new {@link Scroll} with the content of the current one mapped by the given {@code converter}. + * @return a new {@link Window} with the content of the current one mapped by the given {@code converter}. */ - Scroll map(Function converter); + Window map(Function converter); } diff --git a/src/main/java/org/springframework/data/domain/ScrollImpl.java b/src/main/java/org/springframework/data/domain/WindowImpl.java similarity index 89% rename from src/main/java/org/springframework/data/domain/ScrollImpl.java rename to src/main/java/org/springframework/data/domain/WindowImpl.java index 538849670b..2272447f4e 100644 --- a/src/main/java/org/springframework/data/domain/ScrollImpl.java +++ b/src/main/java/org/springframework/data/domain/WindowImpl.java @@ -26,19 +26,19 @@ import org.springframework.util.ObjectUtils; /** - * Default {@link Scroll} implementation. + * Default {@link Window} implementation. * * @author Mark Paluch * @since 3.1 */ -class ScrollImpl implements Scroll { +class WindowImpl implements Window { private final List items; private final IntFunction positionFunction; private final boolean hasNext; - ScrollImpl(List items, IntFunction positionFunction, boolean hasNext) { + WindowImpl(List items, IntFunction positionFunction, boolean hasNext) { Assert.notNull(items, "List of items must not be null"); Assert.notNull(positionFunction, "Position function must not be null"); @@ -79,11 +79,11 @@ public ScrollPosition positionAt(int index) { } @Override - public Scroll map(Function converter) { + public Window map(Function converter) { Assert.notNull(converter, "Function must not be null"); - return new ScrollImpl<>(stream().map(converter).collect(Collectors.toList()), positionFunction, hasNext); + return new WindowImpl<>(stream().map(converter).collect(Collectors.toList()), positionFunction, hasNext); } @NotNull @@ -98,7 +98,7 @@ public boolean equals(Object o) { return true; if (o == null || getClass() != o.getClass()) return false; - ScrollImpl that = (ScrollImpl) o; + WindowImpl that = (WindowImpl) o; return ObjectUtils.nullSafeEquals(items, that.items) && ObjectUtils.nullSafeEquals(positionFunction, that.positionFunction) && ObjectUtils.nullSafeEquals(hasNext, that.hasNext); diff --git a/src/main/java/org/springframework/data/domain/WindowIterator.java b/src/main/java/org/springframework/data/domain/WindowIterator.java new file mode 100644 index 0000000000..f4c06833bc --- /dev/null +++ b/src/main/java/org/springframework/data/domain/WindowIterator.java @@ -0,0 +1,117 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.Function; + +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * An {@link Iterator} over multiple {@link Window Windows} obtained via a {@link Function window function}, that keeps track of + * the current {@link ScrollPosition} returning the Window {@link Window#getContent() content} on {@link #next()}. + * 
+ * WindowIterator<User> users = WindowIterator.of(position -> repository.findFirst10By...("spring", position))
+ *   .startingAt(OffsetScrollPosition.initial());
+ * while (users.hasNext()) {
+ *   users.next().forEach(user -> {
+ *     // consume the user
+ *   });
+ * }
+ *
+ * + * @author Christoph Strobl + * @since 3.1 + */ +public class WindowIterator implements Iterator> { + + private final Function> windowFunction; + private ScrollPosition currentPosition; + + @Nullable // + private Window currentWindow; + + /** + * Entrypoint to create a new {@link WindowIterator} for the given windowFunction. + * + * @param windowFunction must not be {@literal null}. + * @param + * @return new instance of {@link WindowIteratorBuilder}. + */ + public static WindowIteratorBuilder of(Function> windowFunction) { + return new WindowIteratorBuilder(windowFunction); + } + + WindowIterator(Function> windowFunction, ScrollPosition position) { + + this.windowFunction = windowFunction; + this.currentPosition = position; + this.currentWindow = doScroll(); + } + + @Override + public boolean hasNext() { + return currentWindow != null; + } + + @Override + public List next() { + + List toReturn = new ArrayList<>(currentWindow.getContent()); + currentPosition = currentWindow.positionAt(currentWindow.size() -1); + currentWindow = doScroll(); + return toReturn; + } + + @Nullable + Window doScroll() { + + if (currentWindow != null && !currentWindow.hasNext()) { + return null; + } + + Window window = windowFunction.apply(currentPosition); + if (window.isEmpty() && window.isLast()) { + return null; + } + return window; + } + + /** + * Builder API to construct a {@link WindowIterator}. + * + * @param + * @author Christoph Strobl + * @since 3.1 + */ + public static class WindowIteratorBuilder { + + private Function> windowFunction; + + WindowIteratorBuilder(Function> windowFunction) { + this.windowFunction = windowFunction; + } + + public WindowIterator startingAt(ScrollPosition position) { + + Assert.state(windowFunction != null, "WindowFunction cannot not be null"); + return new WindowIterator<>(windowFunction, position); + } + } +} diff --git a/src/main/java/org/springframework/data/repository/query/FluentQuery.java b/src/main/java/org/springframework/data/repository/query/FluentQuery.java index 7d3513bf50..b36437dfbd 100644 --- a/src/main/java/org/springframework/data/repository/query/FluentQuery.java +++ b/src/main/java/org/springframework/data/repository/query/FluentQuery.java @@ -15,6 +15,7 @@ */ package org.springframework.data.repository.query; +import org.springframework.data.domain.Window; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -26,7 +27,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Sort; import org.springframework.lang.Nullable; @@ -165,7 +165,7 @@ default Optional first() { List all(); /** - * Get all matching elements as {@link Scroll} to start result scrolling or resume scrolling at + * Get all matching elements as {@link Window} to start result scrolling or resume scrolling at * {@code scrollPosition}. * * @param scrollPosition must not be {@literal null}. @@ -174,7 +174,7 @@ default Optional first() { * @throws UnsupportedOperationException if not supported by the underlying implementation. * @since 3.1 */ - default Scroll scroll(ScrollPosition scrollPosition) { + default Window scroll(ScrollPosition scrollPosition) { throw new UnsupportedOperationException("Scrolling not supported"); } @@ -261,7 +261,7 @@ default ReactiveFluentQuery project(String... properties) { Flux all(); /** - * Get all matching elements as {@link Scroll} to start result scrolling or resume scrolling at + * Get all matching elements as {@link Window} to start result scrolling or resume scrolling at * {@code scrollPosition}. * * @param scrollPosition must not be {@literal null}. @@ -270,7 +270,7 @@ default ReactiveFluentQuery project(String... properties) { * @throws UnsupportedOperationException if not supported by the underlying implementation. * @since 3.1 */ - default Mono> scroll(ScrollPosition scrollPosition) { + default Mono> scroll(ScrollPosition scrollPosition) { throw new UnsupportedOperationException("Scrolling not supported"); } diff --git a/src/main/java/org/springframework/data/repository/query/QueryMethod.java b/src/main/java/org/springframework/data/repository/query/QueryMethod.java index 23db75a5d9..1d12f533d8 100644 --- a/src/main/java/org/springframework/data/repository/query/QueryMethod.java +++ b/src/main/java/org/springframework/data/repository/query/QueryMethod.java @@ -24,7 +24,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Slice; import org.springframework.data.domain.Sort; @@ -97,7 +97,7 @@ public QueryMethod(Method method, RepositoryMetadata metadata, ProjectionFactory } if (hasParameterOfType(method, ScrollPosition.class)) { - assertReturnTypeAssignable(method, Collections.singleton(Scroll.class)); + assertReturnTypeAssignable(method, Collections.singleton(Window.class)); } Assert.notNull(this.parameters, @@ -203,13 +203,13 @@ public boolean isCollectionQuery() { } /** - * Returns whether the query method will return a {@link Scroll}. + * Returns whether the query method will return a {@link Window}. * * @return * @since 3.1 */ public boolean isScrollQuery() { - return org.springframework.util.ClassUtils.isAssignable(Scroll.class, unwrappedReturnType); + return org.springframework.util.ClassUtils.isAssignable(Window.class, unwrappedReturnType); } /** diff --git a/src/main/java/org/springframework/data/repository/query/ResultProcessor.java b/src/main/java/org/springframework/data/repository/query/ResultProcessor.java index ce1cc2a39b..4083df7c5e 100644 --- a/src/main/java/org/springframework/data/repository/query/ResultProcessor.java +++ b/src/main/java/org/springframework/data/repository/query/ResultProcessor.java @@ -26,7 +26,7 @@ import org.springframework.core.convert.ConversionService; import org.springframework.core.convert.converter.Converter; import org.springframework.core.convert.support.DefaultConversionService; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.Slice; import org.springframework.data.projection.ProjectionFactory; import org.springframework.data.repository.util.ReactiveWrapperConverters; @@ -144,8 +144,8 @@ public T processResult(@Nullable Object source, Converter pr ChainingConverter converter = ChainingConverter.of(type.getReturnedType(), preparingConverter).and(this.converter); - if (source instanceof Scroll && method.isScrollQuery()) { - return (T) ((Scroll) source).map(converter::convert); + if (source instanceof Window && method.isScrollQuery()) { + return (T) ((Window) source).map(converter::convert); } if (source instanceof Slice && (method.isPageQuery() || method.isSliceQuery())) { diff --git a/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java b/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java index e14e8c654e..b900480c21 100644 --- a/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java +++ b/src/main/java/org/springframework/data/repository/util/QueryExecutionConverters.java @@ -36,7 +36,7 @@ import org.springframework.core.convert.support.ConfigurableConversionService; import org.springframework.core.convert.support.DefaultConversionService; import org.springframework.data.domain.Page; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.Slice; import org.springframework.data.geo.GeoResults; import org.springframework.data.util.CustomCollections; @@ -98,7 +98,7 @@ public abstract class QueryExecutionConverters { ALLOWED_PAGEABLE_TYPES.add(Slice.class); ALLOWED_PAGEABLE_TYPES.add(Page.class); ALLOWED_PAGEABLE_TYPES.add(List.class); - ALLOWED_PAGEABLE_TYPES.add(Scroll.class); + ALLOWED_PAGEABLE_TYPES.add(Window.class); WRAPPER_TYPES.add(NullableWrapperToCompletableFutureConverter.getWrapperType()); diff --git a/src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java b/src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java new file mode 100644 index 0000000000..a8f525df72 --- /dev/null +++ b/src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java @@ -0,0 +1,131 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.domain; + +import static org.assertj.core.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.data.domain.WindowIterator.WindowIteratorBuilder; + +/** + * @author Christoph Strobl + */ +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class WindowIteratorUnitTests { + + @Mock Function> fkt; + + @Mock Window window; + + @Mock ScrollPosition scrollPosition; + + @Captor ArgumentCaptor scrollCaptor; + + @BeforeEach + void beforeEach() { + when(fkt.apply(any())).thenReturn(window); + } + + @Test // GH-2151 + void loadsDataOnCreation() { + + WindowIteratorBuilder of = WindowIterator.of(fkt); + verifyNoInteractions(fkt); + + of.startingAt(scrollPosition); + verify(fkt).apply(eq(scrollPosition)); + } + + @Test // GH-2151 + void hasNextReturnsFalseIfNoDataAvailable() { + + when(window.isLast()).thenReturn(true); + when(window.isEmpty()).thenReturn(true); + + assertThat(WindowIterator.of(fkt).startingAt(scrollPosition).hasNext()).isFalse(); + } + + @Test // GH-2151 + void hasNextReturnsTrueIfDataAvailableButOnlyOnePage() { + + when(window.isLast()).thenReturn(true); + when(window.isEmpty()).thenReturn(false); + + assertThat(WindowIterator.of(fkt).startingAt(scrollPosition).hasNext()).isTrue(); + } + + @Test // GH-2151 + void allowsToIterateAllWindows() { + + ScrollPosition p1 = mock(ScrollPosition.class); + ScrollPosition p2 = mock(ScrollPosition.class); + + when(window.isEmpty()).thenReturn(false, false, false); + when(window.isLast()).thenReturn(false, false, true); + when(window.hasNext()).thenReturn(true, true, false); + when(window.size()).thenReturn(1, 1, 1); + when(window.positionAt(anyInt())).thenReturn(p1, p2); + when(window.getContent()).thenReturn(List.of((T) "0"), List.of((T) "1"), List.of((T) "2")); + + WindowIterator iterator = WindowIterator.of(fkt).startingAt(scrollPosition); + List capturedResult = new ArrayList<>(3); + while (iterator.hasNext()) { + capturedResult.addAll(iterator.next()); + } + + verify(fkt, times(3)).apply(scrollCaptor.capture()); + assertThat(scrollCaptor.getAllValues()).containsExactly(scrollPosition, p1, p2); + assertThat(capturedResult).containsExactly((T) "0", (T) "1", (T) "2"); + } + + @Test // GH-2151 + void stopsAfterFirstPageIfOnlyOneWindowAvailable() { + + ScrollPosition p1 = mock(ScrollPosition.class); + + when(window.isEmpty()).thenReturn(false); + when(window.isLast()).thenReturn(true); + when(window.hasNext()).thenReturn(false); + when(window.size()).thenReturn(1); + when(window.positionAt(anyInt())).thenReturn(p1); + when(window.getContent()).thenReturn(List.of((T) "0")); + + WindowIterator iterator = WindowIterator.of(fkt).startingAt(scrollPosition); + List capturedResult = new ArrayList<>(1); + while (iterator.hasNext()) { + capturedResult.addAll(iterator.next()); + } + + verify(fkt).apply(scrollCaptor.capture()); + assertThat(scrollCaptor.getAllValues()).containsExactly(scrollPosition); + assertThat(capturedResult).containsExactly((T) "0"); + } +} diff --git a/src/test/java/org/springframework/data/domain/ScrollUnitTests.java b/src/test/java/org/springframework/data/domain/WindowUnitTests.java similarity index 66% rename from src/test/java/org/springframework/data/domain/ScrollUnitTests.java rename to src/test/java/org/springframework/data/domain/WindowUnitTests.java index 5ffbc3f170..210fc9327e 100644 --- a/src/test/java/org/springframework/data/domain/ScrollUnitTests.java +++ b/src/test/java/org/springframework/data/domain/WindowUnitTests.java @@ -23,31 +23,32 @@ import org.junit.jupiter.api.Test; /** - * Unit tests for {@link Scroll}. + * Unit tests for {@link Window}. * * @author Mark Paluch + * @author Christoph Strobl */ -class ScrollUnitTests { +class WindowUnitTests { @Test // GH-2151 void equalsAndHashCode() { IntFunction positionFunction = OffsetScrollPosition.positionFunction(0); - Scroll one = Scroll.from(List.of(1, 2, 3), positionFunction); - Scroll two = Scroll.from(List.of(1, 2, 3), positionFunction); + Window one = Window.from(List.of(1, 2, 3), positionFunction); + Window two = Window.from(List.of(1, 2, 3), positionFunction); assertThat(one).isEqualTo(two).hasSameHashCodeAs(two); assertThat(one.equals(two)).isTrue(); - assertThat(Scroll.from(List.of(1, 2, 3), positionFunction, true)).isNotEqualTo(two).doesNotHaveSameHashCodeAs(two); + assertThat(Window.from(List.of(1, 2, 3), positionFunction, true)).isNotEqualTo(two).doesNotHaveSameHashCodeAs(two); } @Test // GH-2151 void allowsIteration() { - Scroll scroll = Scroll.from(List.of(1, 2, 3), OffsetScrollPosition.positionFunction(0)); + Window window = Window.from(List.of(1, 2, 3), OffsetScrollPosition.positionFunction(0)); - for (Integer integer : scroll) { + for (Integer integer : window) { assertThat(integer).isBetween(1, 3); } } @@ -55,15 +56,15 @@ void allowsIteration() { @Test // GH-2151 void shouldCreateCorrectPositions() { - Scroll scroll = Scroll.from(List.of(1, 2, 3), OffsetScrollPosition.positionFunction(0)); + Window window = Window.from(List.of(1, 2, 3), OffsetScrollPosition.positionFunction(0)); - assertThat(scroll.firstPosition()).isEqualTo(OffsetScrollPosition.of(1)); - assertThat(scroll.lastPosition()).isEqualTo(OffsetScrollPosition.of(3)); + assertThat(window.positionAt(0)).isEqualTo(OffsetScrollPosition.of(1)); + assertThat(window.positionAt(window.size() - 1)).isEqualTo(OffsetScrollPosition.of(3)); // by index - assertThat(scroll.positionAt(1)).isEqualTo(OffsetScrollPosition.of(2)); + assertThat(window.positionAt(1)).isEqualTo(OffsetScrollPosition.of(2)); // by object - assertThat(scroll.positionAt(Integer.valueOf(1))).isEqualTo(OffsetScrollPosition.of(1)); + assertThat(window.positionAt(Integer.valueOf(1))).isEqualTo(OffsetScrollPosition.of(1)); } } diff --git a/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java b/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java index a62071f6ed..bea7d7b780 100755 --- a/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java +++ b/src/test/java/org/springframework/data/repository/query/ParametersUnitTests.java @@ -28,7 +28,7 @@ import org.springframework.data.domain.OffsetScrollPosition; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; +import org.springframework.data.domain.Window; import org.springframework.data.domain.Sort; import org.springframework.test.util.ReflectionTestUtils; @@ -231,7 +231,7 @@ static interface SampleDao { Page customPageable(SomePageable pageable); - Scroll customScrollPosition(OffsetScrollPosition request); + Window customScrollPosition(OffsetScrollPosition request); } interface SomePageable extends Pageable {} diff --git a/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java b/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java index 7d76e1cf53..e5ad94eefb 100755 --- a/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java +++ b/src/test/java/org/springframework/data/repository/query/QueryMethodUnitTests.java @@ -19,6 +19,7 @@ import io.vavr.collection.Seq; import io.vavr.control.Option; +import org.springframework.data.domain.Window; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -32,7 +33,6 @@ import org.junit.jupiter.api.Test; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Scroll; import org.springframework.data.domain.ScrollPosition; import org.springframework.data.domain.Slice; import org.springframework.data.projection.ProjectionFactory; @@ -351,15 +351,15 @@ interface SampleRepository extends Repository { ImmutableList returnsEclipseCollection(); - Scroll cursorWindow(ScrollPosition cursorRequest); + Window cursorWindow(ScrollPosition cursorRequest); - Mono> reactiveCursorWindow(ScrollPosition cursorRequest); + Mono> reactiveCursorWindow(ScrollPosition cursorRequest); - Flux> invalidReactiveCursorWindow(ScrollPosition cursorRequest); + Flux> invalidReactiveCursorWindow(ScrollPosition cursorRequest); Page cursorWindowMethodWithInvalidReturnType(ScrollPosition cursorRequest); - Scroll cursorWindowWithoutScrollPosition(); + Window cursorWindowWithoutScrollPosition(); } class User { From 6669ea67badaab07f5e19d7185aeba5347fa4708 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Thu, 16 Mar 2023 11:54:06 +0100 Subject: [PATCH 4/4] Polishing. Refactor WindowIterator to return individual objects during scrolling. --- src/main/asciidoc/repositories-scrolling.adoc | 14 +- .../data/domain/KeysetScrollPosition.java | 2 +- .../springframework/data/domain/Window.java | 25 +++- .../data/domain/WindowIterator.java | 92 +++++++++----- .../data/domain/WindowIteratorUnitTests.java | 120 +++++++++--------- 5 files changed, 149 insertions(+), 104 deletions(-) diff --git a/src/main/asciidoc/repositories-scrolling.adoc b/src/main/asciidoc/repositories-scrolling.adoc index 3e06ed7795..5f90fadfc3 100644 --- a/src/main/asciidoc/repositories-scrolling.adoc +++ b/src/main/asciidoc/repositories-scrolling.adoc @@ -20,10 +20,10 @@ do { // obtain the next Scroll users = repository.findFirst10ByLastnameOrderByFirstname("Doe", users.positionAt(users.size() - 1)); -} while (!users.isEmpty() && !users.isLast()); +} while (!users.isEmpty() && users.hasNext()); ---- -`WindowIterator` provides a specialized implementation that simplifies consumption of multiple `Window` instances by removing the need to check for the presence of a next `Window` and applying the `ScrollPosition`. +`WindowIterator` provides a utility to simplify scrolling across ``Window``s by removing the need to check for the presence of a next `Window` and applying the `ScrollPosition`. [source,java] ---- @@ -31,9 +31,8 @@ WindowIterator users = WindowIterator.of(position -> repository.findFirst1 .startingAt(OffsetScrollPosition.initial()); while (users.hasNext()) { - users.next().forEach(user -> { - // consume the user - }); + User u = users.next(); + // consume the user } ---- @@ -56,7 +55,8 @@ interface UserRepository extends Repository { WindowIterator users = WindowIterator.of(position -> repository.findFirst10ByLastnameOrderByFirstname("Doe", position)) .startingAt(OffsetScrollPosition.initial()); <1> ---- -<1> Start from the initial offset at position 0; + +<1> Start from the initial offset at position `0`. ==== [[repositories.scrolling.keyset]] @@ -76,7 +76,7 @@ The database needs only constructing a much smaller result from the given keyset [WARNING] ==== -Keyset-Filtering requires properties used for sorting to be non nullable. +Keyset-Filtering requires the keyset properties (those used for sorting) to be non-nullable. This limitation applies due to the store specific `null` value handling of comparison operators as well as the need to run queries against an indexed source. Keyset-Filtering on nullable properties will lead to unexpected results. ==== diff --git a/src/main/java/org/springframework/data/domain/KeysetScrollPosition.java b/src/main/java/org/springframework/data/domain/KeysetScrollPosition.java index 9063b6f3f8..81b3058e21 100644 --- a/src/main/java/org/springframework/data/domain/KeysetScrollPosition.java +++ b/src/main/java/org/springframework/data/domain/KeysetScrollPosition.java @@ -129,7 +129,7 @@ public String toString() { /** * Keyset scrolling direction. */ - enum Direction { + public enum Direction { /** * Forward (default) direction to scroll from the beginning of the results to their end. diff --git a/src/main/java/org/springframework/data/domain/Window.java b/src/main/java/org/springframework/data/domain/Window.java index d27e2249e4..536e58f08d 100644 --- a/src/main/java/org/springframework/data/domain/Window.java +++ b/src/main/java/org/springframework/data/domain/Window.java @@ -24,8 +24,8 @@ /** * A set of data consumed from an underlying query result. A {@link Window} is similar to {@link Slice} in the sense - * that it contains a subset of the actual query results for easier consumption of large result sets. The window is less - * opinionated about the actual data retrieval, whether the query has used index/offset, keyset-based pagination or + * that it contains a subset of the actual query results for easier scrolling across large result sets. The window is + * less opinionated about the actual data retrieval, whether the query has used index/offset, keyset-based pagination or * cursor resume tokens. * * @author Mark Paluch @@ -93,16 +93,33 @@ default boolean isLast() { /** * Returns if there is a next window. * - * @return if there is a next window window. + * @return if there is a next window. */ boolean hasNext(); + /** + * Returns whether the underlying scroll mechanism can provide a {@link ScrollPosition} at {@code index}. + * + * @param index + * @return {@code true} if a {@link ScrollPosition} can be created; {@code false} otherwise. + * @see #positionAt(int) + */ + default boolean hasPosition(int index) { + try { + return positionAt(index) != null; + } catch (IllegalStateException e) { + return false; + } + } + /** * Returns the {@link ScrollPosition} at {@code index}. * * @param index * @return * @throws IndexOutOfBoundsException if the index is out of range ({@code index < 0 || index >= size()}). + * @throws IllegalStateException if the underlying scroll mechanism cannot provide a scroll position for the given + * object. */ ScrollPosition positionAt(int index); @@ -112,6 +129,8 @@ default boolean isLast() { * @param object * @return * @throws NoSuchElementException if the object is not part of the result. + * @throws IllegalStateException if the underlying scroll mechanism cannot provide a scroll position for the given + * object. */ default ScrollPosition positionAt(T object) { diff --git a/src/main/java/org/springframework/data/domain/WindowIterator.java b/src/main/java/org/springframework/data/domain/WindowIterator.java index f4c06833bc..923dd44cf7 100644 --- a/src/main/java/org/springframework/data/domain/WindowIterator.java +++ b/src/main/java/org/springframework/data/domain/WindowIterator.java @@ -15,37 +15,40 @@ */ package org.springframework.data.domain; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; +import java.util.NoSuchElementException; import java.util.function.Function; import org.springframework.lang.Nullable; import org.springframework.util.Assert; /** - * An {@link Iterator} over multiple {@link Window Windows} obtained via a {@link Function window function}, that keeps track of - * the current {@link ScrollPosition} returning the Window {@link Window#getContent() content} on {@link #next()}. + * An {@link Iterator} over multiple {@link Window Windows} obtained via a {@link Function window function}, that keeps + * track of the current {@link ScrollPosition} allowing scrolling across all result elements. + * * 
- * WindowIterator<User> users = WindowIterator.of(position -> repository.findFirst10By...("spring", position))
+ * WindowIterator<User> users = WindowIterator.of(position -> repository.findFirst10By…("spring", position))
  *   .startingAt(OffsetScrollPosition.initial());
+ *
  * while (users.hasNext()) {
- *   users.next().forEach(user -> {
- *     // consume the user
- *   });
+ *   User u = users.next();
+ *   // consume user
  * }
  *
* * @author Christoph Strobl + * @author Mark Paluch * @since 3.1 */ -public class WindowIterator implements Iterator> { +public class WindowIterator implements Iterator { private final Function> windowFunction; + private ScrollPosition currentPosition; - @Nullable // - private Window currentWindow; + private @Nullable Window currentWindow; + + private @Nullable Iterator currentIterator; /** * Entrypoint to create a new {@link WindowIterator} for the given windowFunction. @@ -55,42 +58,57 @@ public class WindowIterator implements Iterator> { * @return new instance of {@link WindowIteratorBuilder}. */ public static WindowIteratorBuilder of(Function> windowFunction) { - return new WindowIteratorBuilder(windowFunction); + return new WindowIteratorBuilder<>(windowFunction); } WindowIterator(Function> windowFunction, ScrollPosition position) { this.windowFunction = windowFunction; this.currentPosition = position; - this.currentWindow = doScroll(); } @Override public boolean hasNext() { - return currentWindow != null; - } - @Override - public List next() { + // use while loop instead of recursion to fetch the next window. + do { + if (currentWindow == null) { + currentWindow = windowFunction.apply(currentPosition); + } + + if (currentIterator == null) { + if (currentWindow != null) { + currentIterator = currentWindow.iterator(); + } + } + + if (currentIterator != null) { - List toReturn = new ArrayList<>(currentWindow.getContent()); - currentPosition = currentWindow.positionAt(currentWindow.size() -1); - currentWindow = doScroll(); - return toReturn; + if (currentIterator.hasNext()) { + return true; + } + + if (currentWindow != null && currentWindow.hasNext()) { + + currentPosition = currentWindow.positionAt(currentWindow.size() - 1); + currentIterator = null; + currentWindow = null; + continue; + } + } + + return false; + } while (true); } - @Nullable - Window doScroll() { + @Override + public T next() { - if (currentWindow != null && !currentWindow.hasNext()) { - return null; + if (!hasNext()) { + throw new NoSuchElementException(); } - Window window = windowFunction.apply(currentPosition); - if (window.isEmpty() && window.isLast()) { - return null; - } - return window; + return currentIterator.next(); } /** @@ -102,15 +120,25 @@ Window doScroll() { */ public static class WindowIteratorBuilder { - private Function> windowFunction; + private final Function> windowFunction; WindowIteratorBuilder(Function> windowFunction) { + + Assert.notNull(windowFunction, "WindowFunction must not be null"); + this.windowFunction = windowFunction; } + /** + * Create a {@link WindowIterator} given {@link ScrollPosition}. + * + * @param position + * @return + */ public WindowIterator startingAt(ScrollPosition position) { - Assert.state(windowFunction != null, "WindowFunction cannot not be null"); + Assert.notNull(position, "ScrollPosition must not be null"); + return new WindowIterator<>(windowFunction, position); } } diff --git a/src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java b/src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java index a8f525df72..9b27d15fe1 100644 --- a/src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java +++ b/src/test/java/org/springframework/data/domain/WindowIteratorUnitTests.java @@ -20,112 +20,110 @@ import static org.mockito.Mockito.*; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.NoSuchElementException; import java.util.function.Function; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import org.springframework.data.domain.WindowIterator.WindowIteratorBuilder; /** + * Unit tests for {@link WindowIterator}. + * * @author Christoph Strobl + * @author Mark Paluch */ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -class WindowIteratorUnitTests { - - @Mock Function> fkt; +class WindowIteratorUnitTests { - @Mock Window window; + @Test // GH-2151 + void loadsDataOnNext() { - @Mock ScrollPosition scrollPosition; + Function> fkt = mock(Function.class); + WindowIterator iterator = WindowIterator.of(fkt).startingAt(OffsetScrollPosition.initial()); + verifyNoInteractions(fkt); - @Captor ArgumentCaptor scrollCaptor; + when(fkt.apply(any())).thenReturn(Window.from(Collections.emptyList(), value -> OffsetScrollPosition.initial())); - @BeforeEach - void beforeEach() { - when(fkt.apply(any())).thenReturn(window); + iterator.hasNext(); + verify(fkt).apply(OffsetScrollPosition.initial()); } @Test // GH-2151 - void loadsDataOnCreation() { + void hasNextReturnsFalseIfNoDataAvailable() { - WindowIteratorBuilder of = WindowIterator.of(fkt); - verifyNoInteractions(fkt); + Window window = Window.from(Collections.emptyList(), value -> OffsetScrollPosition.initial()); + WindowIterator iterator = WindowIterator.of(it -> window).startingAt(OffsetScrollPosition.initial()); - of.startingAt(scrollPosition); - verify(fkt).apply(eq(scrollPosition)); + assertThat(iterator.hasNext()).isFalse(); } @Test // GH-2151 - void hasNextReturnsFalseIfNoDataAvailable() { + void nextThrowsExceptionIfNoElementAvailable() { - when(window.isLast()).thenReturn(true); - when(window.isEmpty()).thenReturn(true); + Window window = Window.from(Collections.emptyList(), value -> OffsetScrollPosition.initial()); + WindowIterator iterator = WindowIterator.of(it -> window).startingAt(OffsetScrollPosition.initial()); - assertThat(WindowIterator.of(fkt).startingAt(scrollPosition).hasNext()).isFalse(); + assertThatExceptionOfType(NoSuchElementException.class).isThrownBy(iterator::next); } @Test // GH-2151 void hasNextReturnsTrueIfDataAvailableButOnlyOnePage() { - when(window.isLast()).thenReturn(true); - when(window.isEmpty()).thenReturn(false); + Window window = Window.from(List.of("a", "b"), value -> OffsetScrollPosition.initial()); + WindowIterator iterator = WindowIterator.of(it -> window).startingAt(OffsetScrollPosition.initial()); - assertThat(WindowIterator.of(fkt).startingAt(scrollPosition).hasNext()).isTrue(); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo("a"); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo("b"); + assertThat(iterator.hasNext()).isFalse(); + assertThat(iterator.hasNext()).isFalse(); } @Test // GH-2151 - void allowsToIterateAllWindows() { - - ScrollPosition p1 = mock(ScrollPosition.class); - ScrollPosition p2 = mock(ScrollPosition.class); - - when(window.isEmpty()).thenReturn(false, false, false); - when(window.isLast()).thenReturn(false, false, true); - when(window.hasNext()).thenReturn(true, true, false); - when(window.size()).thenReturn(1, 1, 1); - when(window.positionAt(anyInt())).thenReturn(p1, p2); - when(window.getContent()).thenReturn(List.of((T) "0"), List.of((T) "1"), List.of((T) "2")); - - WindowIterator iterator = WindowIterator.of(fkt).startingAt(scrollPosition); - List capturedResult = new ArrayList<>(3); - while (iterator.hasNext()) { - capturedResult.addAll(iterator.next()); - } - - verify(fkt, times(3)).apply(scrollCaptor.capture()); - assertThat(scrollCaptor.getAllValues()).containsExactly(scrollPosition, p1, p2); - assertThat(capturedResult).containsExactly((T) "0", (T) "1", (T) "2"); + void hasNextReturnsCorrectlyIfNextPageIsEmpty() { + + Window window = Window.from(List.of("a", "b"), value -> OffsetScrollPosition.initial()); + WindowIterator iterator = WindowIterator.of(it -> { + if (it.isInitial()) { + return window; + } + + return Window.from(Collections.emptyList(), OffsetScrollPosition::of, false); + }).startingAt(OffsetScrollPosition.initial()); + + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo("a"); + assertThat(iterator.hasNext()).isTrue(); + assertThat(iterator.next()).isEqualTo("b"); + assertThat(iterator.hasNext()).isFalse(); + assertThat(iterator.hasNext()).isFalse(); } @Test // GH-2151 - void stopsAfterFirstPageIfOnlyOneWindowAvailable() { + void allowsToIterateAllWindows() { - ScrollPosition p1 = mock(ScrollPosition.class); + Window window1 = Window.from(List.of("a", "b"), OffsetScrollPosition::of, true); + Window window2 = Window.from(List.of("c", "d"), value -> OffsetScrollPosition.of(2 + value)); + WindowIterator iterator = WindowIterator.of(it -> { + if (it.isInitial()) { + return window1; + } - when(window.isEmpty()).thenReturn(false); - when(window.isLast()).thenReturn(true); - when(window.hasNext()).thenReturn(false); - when(window.size()).thenReturn(1); - when(window.positionAt(anyInt())).thenReturn(p1); - when(window.getContent()).thenReturn(List.of((T) "0")); + return window2; + }).startingAt(OffsetScrollPosition.initial()); - WindowIterator iterator = WindowIterator.of(fkt).startingAt(scrollPosition); - List capturedResult = new ArrayList<>(1); + List capturedResult = new ArrayList<>(4); while (iterator.hasNext()) { - capturedResult.addAll(iterator.next()); + capturedResult.add(iterator.next()); } - verify(fkt).apply(scrollCaptor.capture()); - assertThat(scrollCaptor.getAllValues()).containsExactly(scrollPosition); - assertThat(capturedResult).containsExactly((T) "0"); + assertThat(capturedResult).containsExactly("a", "b", "c", "d"); } }