microsoft
/
spring-data-cosmosdb
зеркало из https://github.com/microsoft/spring-data-cosmosdb.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Pagination Bug Fix (#453) 

* Pagination bug fix
* Disable telemetry by default
* Updated cosmos db to 3.4.0
* Updated emulator tests to exlcude PageablePersonRepositoryIT.java
* Improved test to check correctness of new pagination implementation
* Updated readme to provide more clarity
This commit is contained in:
Kushagra Thapar 2019-11-07 09:53:23 -08:00 коммит произвёл GitHub
Родитель b4167d61be
Коммит 840072ed13
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
22 изменённых файлов: 566 добавлений и 38 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

12
README.md
Просмотреть файл

@ -64,6 +64,18 @@ class MyDocument {
- Supports [Azure Cosmos DB partition](https://docs.microsoft.com/en-us/azure/cosmos-db/partition-data). To specify a field of domain class to be partition key field, just annotate it with `@PartitionKey`. When you do CRUD operation, pls specify your partition value. For more sample on partition CRUD, pls refer to [test here](./src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/integration/AddressRepositoryIT.java)
- Supports [Spring Data custom query](https://docs.spring.io/spring-data/commons/docs/current/reference/html/#repositories.query-methods.details) find operation, e.g., `findByAFieldAndBField`
- Supports [Spring Data pagable and sort](https://docs.spring.io/spring-data/commons/docs/current/reference/html/#repositories.special-parameters).
- Based on available RUs on the database account, cosmosDB can return documents less than or equal to the requested size.
- Due to this variable number of returned documents in every iteration, user should not rely on the totalPageSize, and instead iterating over pageable should be done in this way.
```java
final CosmosPageRequest pageRequest = new CosmosPageRequest(0, pageSize, null);
Page<T> page = tRepository.findAll(pageRequest);
List<T> pageContent = page.getContent();
while(page.hasNext()) {
Pageable nextPageable = page.nextPageable();
page = repository.findAll(nextPageable);
pageContent = page.getContent();
}
```
- Supports [spring-boot-starter-data-rest](https://projects.spring.io/spring-data-rest/).
- Supports List and nested type in domain class.
- Configurable ObjectMapper bean with unique name `cosmosdbObjectMapper`, only configure customized ObjectMapper if you really need to. e.g.,

16
pom.xml
Просмотреть файл

@ -58,7 +58,7 @@
<gson.version>2.8.4</gson.version>
<project.reactor.test.version>3.3.0.RELEASE</project.reactor.test.version>
<azure.cosmos.version>3.3.1</azure.cosmos.version>
<azure.cosmos.version>3.4.0</azure.cosmos.version>
<azure.test.resourcegroup>spring-data-cosmosdb-test</azure.test.resourcegroup>
<azure.test.dbname>testdb-${maven.build.timestamp}</azure.test.dbname>
<skip.integration.tests>true</skip.integration.tests>
@ -483,6 +483,20 @@
<test.on.azure>false</test.on.azure>
<test.on.emulator>true</test.on.emulator>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<excludes>
<!--Excluding PageablePersonRepositoryIT test class, as it creates large documents which are not suitable for emulator-->
<exclude>**/PageablePersonRepositoryIT.java</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>release</id>

3
src/main/java/com/microsoft/azure/spring/data/cosmosdb/common/PropertyLoader.java
Просмотреть файл

@ -39,8 +39,9 @@ public class PropertyLoader {
allowed = getPropertyByName("telemetryAllowed", APPLICATION_YML_FILE);
}
// Default, no telemetry
if (allowed == null) {
return true;
return false;
} else {
return !allowed.equalsIgnoreCase("false");
}

29
src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplate.java
Просмотреть файл

@ -23,6 +23,7 @@ import com.microsoft.azure.spring.data.cosmosdb.common.Memoizer;
import com.microsoft.azure.spring.data.cosmosdb.core.convert.MappingCosmosConverter;
import com.microsoft.azure.spring.data.cosmosdb.core.generator.CountQueryGenerator;
import com.microsoft.azure.spring.data.cosmosdb.core.generator.FindQuerySpecGenerator;
import com.microsoft.azure.spring.data.cosmosdb.core.query.CosmosPageImpl;
import com.microsoft.azure.spring.data.cosmosdb.core.query.CosmosPageRequest;
import com.microsoft.azure.spring.data.cosmosdb.core.query.Criteria;
import com.microsoft.azure.spring.data.cosmosdb.core.query.CriteriaType;
@ -34,7 +35,6 @@ import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.lang.NonNull;
import org.springframework.util.Assert;
@ -431,12 +431,29 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware
result.add(entity);
}
final CosmosPageRequest pageRequest = CosmosPageRequest.of(pageable.getPageNumber(),
pageable.getPageSize(),
feedResponse.continuationToken(),
query.getSort());
final long total = count(query, domainClass, collectionName);
final int contentSize = result.size();
return new PageImpl<>(result, pageRequest, count(query, domainClass, collectionName));
int pageSize;
if (contentSize < pageable.getPageSize() && contentSize > 0) {
// If the content size is less than page size,
// this means, cosmosDB is returning less documents than page size,
// because of either RU limit, or payload limit
// Set the page size to content size.
pageSize = contentSize;
} else {
pageSize = pageable.getPageSize();
}
final CosmosPageRequest pageRequest = CosmosPageRequest.of(pageable.getOffset(),
pageable.getPageNumber(),
pageSize,
feedResponse.continuationToken(),
query.getSort());
return new CosmosPageImpl<>(result, pageRequest, total);
}
@Override

7
src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/generator/AbstractQueryGenerator.java
Просмотреть файл

@ -5,6 +5,7 @@
*/
package com.microsoft.azure.spring.data.cosmosdb.core.generator;
import com.azure.data.cosmos.SqlParameterList;
import com.azure.data.cosmos.SqlQuerySpec;
import com.microsoft.azure.spring.data.cosmosdb.core.query.Criteria;
import com.microsoft.azure.spring.data.cosmosdb.core.query.CriteriaType;
@ -204,8 +205,8 @@ public abstract class AbstractQueryGenerator {
final Pair<String, List<Pair<String, Object>>> queryBody = generateQueryBody(query);
final String queryString = String.join(" ", queryHead, queryBody.getValue0(), generateQueryTail(query));
final List<Pair<String, Object>> parameters = queryBody.getValue1();
final com.azure.data.cosmos.SqlParameterList sqlParameters =
new com.azure.data.cosmos.SqlParameterList();
final SqlParameterList sqlParameters =
new SqlParameterList();
sqlParameters.addAll(
parameters.stream()
@ -214,6 +215,6 @@ public abstract class AbstractQueryGenerator {
.collect(Collectors.toList())
);
return new com.azure.data.cosmos.SqlQuerySpec(queryString, sqlParameters);
return new SqlQuerySpec(queryString, sqlParameters);
}
}

3
src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/generator/CountQueryGenerator.java
Просмотреть файл

@ -5,12 +5,13 @@
*/
package com.microsoft.azure.spring.data.cosmosdb.core.generator;
import com.azure.data.cosmos.SqlQuerySpec;
import com.microsoft.azure.spring.data.cosmosdb.core.query.DocumentQuery;
public class CountQueryGenerator extends AbstractQueryGenerator implements QuerySpecGenerator {
@Override
public com.azure.data.cosmos.SqlQuerySpec generateCosmos(DocumentQuery query) {
public SqlQuerySpec generateCosmos(DocumentQuery query) {
return super.generateCosmosQuery(query, "SELECT VALUE COUNT(1) FROM r");
}
}

3
src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/generator/FindQuerySpecGenerator.java
Просмотреть файл

@ -5,6 +5,7 @@
*/
package com.microsoft.azure.spring.data.cosmosdb.core.generator;
import com.azure.data.cosmos.SqlQuerySpec;
import com.microsoft.azure.spring.data.cosmosdb.core.query.DocumentQuery;
import lombok.NoArgsConstructor;
@ -12,7 +13,7 @@ import lombok.NoArgsConstructor;
public class FindQuerySpecGenerator extends AbstractQueryGenerator implements QuerySpecGenerator {
@Override
public com.azure.data.cosmos.SqlQuerySpec generateCosmos(DocumentQuery query) {
public SqlQuerySpec generateCosmos(DocumentQuery query) {
return super.generateCosmosQuery(query, "SELECT * FROM ROOT r");
}
}

3
src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/generator/QuerySpecGenerator.java
Просмотреть файл

@ -5,6 +5,7 @@
*/
package com.microsoft.azure.spring.data.cosmosdb.core.generator;
import com.azure.data.cosmos.SqlQuerySpec;
import com.microsoft.azure.spring.data.cosmosdb.core.query.DocumentQuery;
public interface QuerySpecGenerator {
@ -14,5 +15,5 @@ public interface QuerySpecGenerator {
* @param query tree structured query condition.
* @return SqlQuerySpec executed by cosmos client.
*/
com.azure.data.cosmos.SqlQuerySpec generateCosmos(DocumentQuery query);
SqlQuerySpec generateCosmos(DocumentQuery query);
}

67
src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/query/CosmosPageImpl.java Normal file
Просмотреть файл

@ -0,0 +1,67 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See LICENSE in the project root for
* license information.
*/
package com.microsoft.azure.spring.data.cosmosdb.core.query;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import java.util.List;
import java.util.Objects;
public class CosmosPageImpl<T> extends PageImpl<T> {
private static final long serialVersionUID = 5294396337522314504L;
// For any query, CosmosDB returns documents less than or equal to page size
// Depending on the number of RUs, the number of returned documents can change
// Storing the offset of current page, helps to check hasNext and next values
private long offset;
public CosmosPageImpl(List<T> content, Pageable pageable, long total) {
super(content, pageable, total);
this.offset = pageable.getOffset();
}
@Override
public int getTotalPages() {
return super.getTotalPages();
}
@Override
public long getTotalElements() {
return super.getTotalElements();
}
@Override
public boolean hasNext() {
return this.offset + getContent().size() < getTotalElements();
}
@Override
public boolean isLast() {
return super.isLast();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
if (!super.equals(o)) {
return false;
}
final CosmosPageImpl<?> that = (CosmosPageImpl<?>) o;
return offset == that.offset;
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), offset);
}
}

37
src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/query/CosmosPageRequest.java
Просмотреть файл

@ -7,6 +7,7 @@ package com.microsoft.azure.spring.data.cosmosdb.core.query;
import com.azure.data.cosmos.FeedResponse;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
/**
@ -19,6 +20,8 @@ import org.springframework.data.domain.Sort;
public class CosmosPageRequest extends PageRequest {
private static final long serialVersionUID = 6093304300037688375L;
private long offset;
// Request continuation token used to resume query
private String requestContinuation;
@ -27,17 +30,41 @@ public class CosmosPageRequest extends PageRequest {
this.requestContinuation = requestContinuation;
}
public static CosmosPageRequest of(int page, int size, String requestContinuation) {
return new CosmosPageRequest(page, size, requestContinuation);
}
public CosmosPageRequest(int page, int size, String requestContinuation, Sort sort) {
super(page, size, sort);
this.requestContinuation = requestContinuation;
}
private CosmosPageRequest(long offset, int page, int size, String requestContinuation) {
super(page, size, Sort.unsorted());
this.offset = offset;
this.requestContinuation = requestContinuation;
}
private CosmosPageRequest(long offset, int page, int size, String requestContinuation,
Sort sort) {
super(page, size, sort);
this.offset = offset;
this.requestContinuation = requestContinuation;
}
public static CosmosPageRequest of(int page, int size, String requestContinuation, Sort sort) {
return new CosmosPageRequest(page, size, requestContinuation, sort);
return new CosmosPageRequest(0, page, size, requestContinuation, sort);
}
public static CosmosPageRequest of(long offset, int page, int size, String requestContinuation, Sort sort) {
return new CosmosPageRequest(offset, page, size, requestContinuation, sort);
}
@Override
public Pageable next() {
return new CosmosPageRequest(this.offset + (long) this.getPageSize(),
this.getPageNumber() + 1, getPageSize(), this.requestContinuation);
}
@Override
public long getOffset() {
return offset;
}
public String getRequestContinuation() {

2
src/test/java/com/microsoft/azure/spring/data/cosmosdb/common/PropertyLoaderUnitTest.java
Просмотреть файл

@ -22,6 +22,6 @@ public class PropertyLoaderUnitTest {
public void testGetApplicationTelemetryAllowed() {
final boolean isAllowed = PropertyLoader.isApplicationTelemetryAllowed();
Assert.assertTrue(isAllowed);
Assert.assertFalse(isAllowed);
}
}

10
src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplateIT.java
Просмотреть файл

@ -333,7 +333,7 @@ public class CosmosTemplateIT {
final Page<Person> page = cosmosTemplate.paginationQuery(query, Person.class, collectionName);
assertThat(page.getContent().size()).isEqualTo(1);
validateLastPage(page, PAGE_SIZE_2);
validateLastPage(page, page.getContent().size());
assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull();
assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull();
@ -350,7 +350,7 @@ public class CosmosTemplateIT {
assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull();
final Sort sort = Sort.by(Sort.Direction.DESC, "firstName");
final PageRequest pageRequest = CosmosPageRequest.of(0, PAGE_SIZE_3, null, sort);
final PageRequest pageRequest = new CosmosPageRequest(0, PAGE_SIZE_3, null, sort);
final Page<Person> page = cosmosTemplate.findAll(pageRequest, Person.class, collectionName);
assertThat(page.getContent().size()).isEqualTo(3);
@ -381,13 +381,13 @@ public class CosmosTemplateIT {
new PartitionKey(personInfo.getPartitionKeyFieldValue(testPerson5)));
final Sort sort = Sort.by(Sort.Direction.ASC, "firstName");
final PageRequest pageRequest = CosmosPageRequest.of(0, PAGE_SIZE_3, null, sort);
final PageRequest pageRequest = new CosmosPageRequest(0, PAGE_SIZE_3, null, sort);
final Page<Person> firstPage = cosmosTemplate.findAll(pageRequest, Person.class,
collectionName);
assertThat(firstPage.getContent().size()).isEqualTo(3);
validateNonLastPage(firstPage, PAGE_SIZE_3);
validateNonLastPage(firstPage, firstPage.getContent().size());
final List<Person> firstPageResults = firstPage.getContent();
assertThat(firstPageResults.get(0).getFirstName()).isEqualTo(testPerson4.getFirstName());
@ -398,7 +398,7 @@ public class CosmosTemplateIT {
collectionName);
assertThat(secondPage.getContent().size()).isEqualTo(2);
validateLastPage(secondPage, PAGE_SIZE_3);
validateLastPage(secondPage, secondPage.getContent().size());
final List<Person> secondPageResults = secondPage.getContent();
assertThat(secondPageResults.get(0).getFirstName()).isEqualTo(NEW_FIRST_NAME);

2
src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplatePartitionIT.java
Просмотреть файл

@ -241,6 +241,6 @@ public class CosmosTemplatePartitionIT {
final Page<PartitionPerson> page = cosmosTemplate.paginationQuery(query, PartitionPerson.class, collectionName);
assertThat(page.getContent().size()).isEqualTo(1);
validateLastPage(page, PAGE_SIZE_2);
validateLastPage(page, page.getContent().size());
}
}

23
src/test/java/com/microsoft/azure/spring/data/cosmosdb/domain/Address.java
Просмотреть файл

@ -11,7 +11,9 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import org.springframework.data.annotation.Id;
@Document(ru = "1000")
import java.util.Objects;
@Document(ru = "400")
@Data
@AllArgsConstructor
public class Address {
@ -20,4 +22,23 @@ public class Address {
String street;
@PartitionKey
String city;
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
final Address address = (Address) o;
return Objects.equals(postalCode, address.postalCode) &&
Objects.equals(street, address.street) &&
Objects.equals(city, address.city);
}
@Override
public int hashCode() {
return Objects.hash(postalCode, street, city);
}
}

2
src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/TestRepositoryConfig.java
Просмотреть файл

@ -36,7 +36,7 @@ public class TestRepositoryConfig extends AbstractCosmosConfiguration {
private RequestOptions getRequestOptions() {
final RequestOptions options = new RequestOptions();
options.setConsistencyLevel(ConsistencyLevel.CONSISTENT_PREFIX);
options.setConsistencyLevel(ConsistencyLevel.SESSION);
// options.setDisableRUPerMinuteUsage(true);
options.setScriptLoggingEnabled(true);

55
src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/integration/PageableAddressRepositoryIT.java
Просмотреть файл

@ -5,6 +5,10 @@
*/
package com.microsoft.azure.spring.data.cosmosdb.repository.integration;
import com.azure.data.cosmos.CosmosClient;
import com.azure.data.cosmos.CosmosItemProperties;
import com.azure.data.cosmos.FeedOptions;
import com.azure.data.cosmos.FeedResponse;
import com.microsoft.azure.spring.data.cosmosdb.common.TestConstants;
import com.microsoft.azure.spring.data.cosmosdb.common.TestUtils;
import com.microsoft.azure.spring.data.cosmosdb.core.CosmosTemplate;
@ -18,16 +22,20 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.data.domain.Page;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import reactor.core.publisher.Flux;
import reactor.test.StepVerifier;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import static com.microsoft.azure.spring.data.cosmosdb.common.PageTestUtils.validateLastPage;
import static com.microsoft.azure.spring.data.cosmosdb.common.PageTestUtils.validateNonLastPage;
import static com.microsoft.azure.spring.data.cosmosdb.common.TestConstants.DB_NAME;
import static com.microsoft.azure.spring.data.cosmosdb.common.TestConstants.PAGE_SIZE_1;
import static com.microsoft.azure.spring.data.cosmosdb.common.TestConstants.PAGE_SIZE_3;
import static org.assertj.core.api.Assertions.assertThat;
@ -53,6 +61,9 @@ public class PageableAddressRepositoryIT {
@Autowired
private PageableAddressRepository repository;
@Autowired
private ApplicationContext applicationContext;
@Before
public void setup() {
repository.save(TEST_ADDRESS1_PARTITION1);
@ -88,7 +99,7 @@ public class PageableAddressRepositoryIT {
final Page<Address> nextPage = repository.findAll(page.getPageable());
assertThat(nextPage.getContent().size()).isEqualTo(1);
validateLastPage(nextPage, PAGE_SIZE_3);
validateLastPage(nextPage, nextPage.getContent().size());
}
@Test
@ -98,7 +109,7 @@ public class PageableAddressRepositoryIT {
assertThat(page.getContent().size()).isEqualTo(2);
validateResultCityMatch(page, TestConstants.CITY);
validateLastPage(page, PAGE_SIZE_3);
validateLastPage(page, page.getContent().size());
}
@Test
@ -124,7 +135,7 @@ public class PageableAddressRepositoryIT {
assertThat(page.getContent().size()).isEqualTo(2);
validateResultStreetMatch(page, TestConstants.STREET);
validateLastPage(page, PAGE_SIZE_3);
validateLastPage(page, page.getContent().size());
}
@Test
@ -143,13 +154,41 @@ public class PageableAddressRepositoryIT {
validateLastPage(nextPage, PAGE_SIZE_1);
}
@Test
public void testOffsetAndLimit() {
final int skipCount = 2;
final int takeCount = 2;
final List<CosmosItemProperties> results = new ArrayList<>();
final FeedOptions options = new FeedOptions();
options.enableCrossPartitionQuery(true);
options.maxDegreeOfParallelism(2);
final String query = "SELECT * from c OFFSET " + skipCount + " LIMIT " + takeCount;
final CosmosClient cosmosClient = applicationContext.getBean(CosmosClient.class);
final Flux<FeedResponse<CosmosItemProperties>> feedResponseFlux =
cosmosClient.getDatabase(DB_NAME)
.getContainer(entityInformation.getCollectionName())
.queryItems(query, options);
StepVerifier.create(feedResponseFlux)
.consumeNextWith(cosmosItemPropertiesFeedResponse ->
results.addAll(cosmosItemPropertiesFeedResponse.results()))
.verifyComplete();
assertThat(results.size()).isEqualTo(takeCount);
}
private void validateResultCityMatch(Page<Address> page, String city) {
assertThat(page.getContent().stream().filter(address -> address.getCity().equals(city))
.collect(Collectors.toList()).size()).isEqualTo(page.getContent().size());
assertThat((int) page.getContent()
.stream()
.filter(address -> address.getCity().equals(city))
.count()).isEqualTo(page.getContent().size());
}
private void validateResultStreetMatch(Page<Address> page, String street) {
assertThat(page.getContent().stream().filter(address -> address.getStreet().equals(street))
.collect(Collectors.toList()).size()).isEqualTo(page.getContent().size());
assertThat((int) page.getContent()
.stream()
.filter(address -> address.getStreet().equals(street))
.count()).isEqualTo(page.getContent().size());
}
}

170
src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/integration/PageableMemoRepositoryIT.java Normal file
Просмотреть файл

@ -0,0 +1,170 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See LICENSE in the project root for
* license information.
*/
package com.microsoft.azure.spring.data.cosmosdb.repository.integration;
import com.azure.data.cosmos.CosmosClient;
import com.azure.data.cosmos.CosmosItemProperties;
import com.azure.data.cosmos.FeedOptions;
import com.azure.data.cosmos.FeedResponse;
import com.microsoft.azure.spring.data.cosmosdb.core.CosmosTemplate;
import com.microsoft.azure.spring.data.cosmosdb.core.query.CosmosPageRequest;
import com.microsoft.azure.spring.data.cosmosdb.domain.Importance;
import com.microsoft.azure.spring.data.cosmosdb.domain.Memo;
import com.microsoft.azure.spring.data.cosmosdb.repository.TestRepositoryConfig;
import com.microsoft.azure.spring.data.cosmosdb.repository.repository.PageableMemoRepository;
import com.microsoft.azure.spring.data.cosmosdb.repository.support.CosmosEntityInformation;
import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import reactor.core.publisher.Flux;
import reactor.test.StepVerifier;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import static com.microsoft.azure.spring.data.cosmosdb.common.TestConstants.DB_NAME;
import static org.assertj.core.api.Assertions.assertThat;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = TestRepositoryConfig.class)
@Slf4j
public class PageableMemoRepositoryIT {
private static final int TOTAL_CONTENT_SIZE = 500;
private final CosmosEntityInformation<Memo, String> entityInformation =
new CosmosEntityInformation<>(Memo.class);
@Autowired
private CosmosTemplate template;
@Autowired
private PageableMemoRepository repository;
@Autowired
private ApplicationContext applicationContext;
private static Set<Memo> memoSet;
private static boolean isSetupDone;
@Before
public void setup() {
if (isSetupDone) {
return;
}
memoSet = new HashSet<>();
final Random random = new Random();
final Importance[] importanceValues = Importance.values();
// Create larger documents with size more than 10 kb
for (int i = 0; i < TOTAL_CONTENT_SIZE; i++) {
final String id = UUID.randomUUID().toString();
final String message = UUID.randomUUID().toString();
final int randomIndex = random.nextInt(3);
final Memo memo = new Memo(id, message, new Date(), importanceValues[randomIndex]);
repository.save(memo);
memoSet.add(memo);
}
isSetupDone = true;
}
@PreDestroy
public void cleanUpCollection() {
template.deleteCollection(entityInformation.getCollectionName());
}
@Test
public void testFindAllWithPageSizeLessThanReturned() {
final Set<Memo> memos = findAllWithPageSize(20);
assertThat(memos).isEqualTo(memoSet);
}
@Test
public void testFindAllWithPageSizeLessThanTotal() {
final Set<Memo> memos = findAllWithPageSize(200);
assertThat(memos).isEqualTo(memoSet);
}
@Test
public void testFindAllWithPageSizeGreaterThanTotal() {
final Set<Memo> memos = findAllWithPageSize(10000);
assertThat(memos).isEqualTo(memoSet);
}
@Test
public void testOffsetAndLimitLessThanTotal() {
final int skipCount = 50;
final int takeCount = 200;
verifyItemsWithOffsetAndLimit(skipCount, takeCount, takeCount);
}
@Test
public void testOffsetAndLimitEqualToTotal() {
final int skipCount = 100;
final int takeCount = 300;
verifyItemsWithOffsetAndLimit(skipCount, takeCount, takeCount);
}
@Test
public void testOffsetAndLimitGreaterThanTotal() {
final int skipCount = 300;
final int takeCount = 300;
verifyItemsWithOffsetAndLimit(skipCount, takeCount, TOTAL_CONTENT_SIZE - skipCount);
}
private Flux<FeedResponse<CosmosItemProperties>> getItemsWithOffsetAndLimit(int skipCount, int takeCount) {
final FeedOptions options = new FeedOptions();
options.enableCrossPartitionQuery(true);
options.maxDegreeOfParallelism(2);
final String query = "SELECT * from c OFFSET " + skipCount + " LIMIT " + takeCount;
final CosmosClient cosmosClient = applicationContext.getBean(CosmosClient.class);
return cosmosClient.getDatabase(DB_NAME)
.getContainer(entityInformation.getCollectionName())
.queryItems(query, options);
}
private void verifyItemsWithOffsetAndLimit(int skipCount, int takeCount, int verifyCount) {
final List<CosmosItemProperties> itemsWithOffsetAndLimit = new ArrayList<>();
final Flux<FeedResponse<CosmosItemProperties>> itemsWithOffsetAndLimitFlux =
getItemsWithOffsetAndLimit(skipCount, takeCount);
StepVerifier.create(itemsWithOffsetAndLimitFlux)
.thenConsumeWhile(cosmosItemPropertiesFeedResponse -> {
itemsWithOffsetAndLimit.addAll(cosmosItemPropertiesFeedResponse.results());
return true;
})
.verifyComplete();
assertThat(itemsWithOffsetAndLimit.size()).isEqualTo(verifyCount);
}
private Set<Memo> findAllWithPageSize(int pageSize) {
final CosmosPageRequest pageRequest = new CosmosPageRequest(0, pageSize, null);
Page<Memo> page = repository.findAll(pageRequest);
final Set<Memo> outputSet = new HashSet<>(page.getContent());
while (page.hasNext()) {
final Pageable pageable = page.nextPageable();
page = repository.findAll(pageable);
outputSet.addAll((page.getContent()));
}
return outputSet;
}
}

128
src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/integration/PageablePersonRepositoryIT.java Normal file
Просмотреть файл

@ -0,0 +1,128 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See LICENSE in the project root for
* license information.
*/
package com.microsoft.azure.spring.data.cosmosdb.repository.integration;
import com.microsoft.azure.spring.data.cosmosdb.core.CosmosTemplate;
import com.microsoft.azure.spring.data.cosmosdb.core.query.CosmosPageRequest;
import com.microsoft.azure.spring.data.cosmosdb.domain.Address;
import com.microsoft.azure.spring.data.cosmosdb.domain.Person;
import com.microsoft.azure.spring.data.cosmosdb.repository.TestRepositoryConfig;
import com.microsoft.azure.spring.data.cosmosdb.repository.repository.PageablePersonRepository;
import com.microsoft.azure.spring.data.cosmosdb.repository.support.CosmosEntityInformation;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = TestRepositoryConfig.class)
@Slf4j
public class PageablePersonRepositoryIT {
private static final int TOTAL_CONTENT_SIZE = 50;
private final CosmosEntityInformation<Person, String> entityInformation =
new CosmosEntityInformation<>(Person.class);
@Autowired
private CosmosTemplate template;
@Autowired
private PageablePersonRepository repository;
private static Set<Person> personSet;
private static boolean isSetupDone;
@Before
public void setup() {
if (isSetupDone) {
return;
}
personSet = new HashSet<>();
// Create larger documents with size more than 10 kb
for (int i = 0; i < TOTAL_CONTENT_SIZE; i++) {
final List<String> hobbies = new ArrayList<>();
hobbies.add(StringUtils.repeat("hobbies-" + UUID.randomUUID().toString(),
(int) FileUtils.ONE_KB * 10));
final List<Address> address = new ArrayList<>();
address.add(new Address("postalCode-" + UUID.randomUUID().toString(),
"street-" + UUID.randomUUID().toString(),
"city-" + UUID.randomUUID().toString()));
final Person person = new Person(UUID.randomUUID().toString(),
UUID.randomUUID().toString(), UUID.randomUUID().toString(),
hobbies, address);
repository.save(person);
personSet.add(person);
}
isSetupDone = true;
}
@PreDestroy
public void cleanUpCollection() {
template.deleteCollection(entityInformation.getCollectionName());
}
// Cosmos DB can return any number of documents less than or equal to requested page size
// Because of available RUs, the number of return documents vary.
// With documents more than 10 KB, and collection RUs 400,
// it usually return documents less than total content size.
// This test covers the case where page size is greater than returned documents
@Test
public void testFindAllWithPageSizeGreaterThanReturned() {
final Set<Person> outputSet = findAllWithPageSize(30, false);
assertThat(outputSet).isEqualTo(personSet);
}
// This test covers the case where page size is less than returned documents
@Test
public void testFindAllWithPageSizeLessThanReturned() {
final Set<Person> outputSet = findAllWithPageSize(5, false);
assertThat(outputSet).isEqualTo(personSet);
}
// This test covers the case where page size is greater than total number of documents
@Test
public void testFindAllWithPageSizeGreaterThanTotal() {
final Set<Person> outputSet = findAllWithPageSize(120, true);
assertThat(outputSet).isEqualTo(personSet);
}
private Set<Person> findAllWithPageSize(int pageSize, boolean checkContentLimit) {
final CosmosPageRequest pageRequest = new CosmosPageRequest(0, pageSize, null);
Page<Person> page = repository.findAll(pageRequest);
final Set<Person> outputSet = new HashSet<>(page.getContent());
if (checkContentLimit) {
// Make sure CosmosDB returns less number of documents than requested
// This will verify the functionality of new pagination implementation
assertThat(page.getContent().size()).isLessThan(pageSize);
}
while (page.hasNext()) {
final Pageable pageable = page.nextPageable();
page = repository.findAll(pageable);
outputSet.addAll((page.getContent()));
}
return outputSet;
}
}

2
src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/integration/ProjectRepositorySortIT.java
Просмотреть файл

@ -223,7 +223,7 @@ public class ProjectRepositorySortIT {
Assert.assertEquals(references.size(), result.getContent().size());
Assert.assertEquals(references, result.getContent());
validateLastPage(result, 5);
validateLastPage(result, result.getContent().size());
}
}

14
src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/repository/PageableMemoRepository.java Normal file
Просмотреть файл

@ -0,0 +1,14 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See LICENSE in the project root for
* license information.
*/
package com.microsoft.azure.spring.data.cosmosdb.repository.repository;
import com.microsoft.azure.spring.data.cosmosdb.domain.Memo;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PageableMemoRepository extends PagingAndSortingRepository<Memo, String> {
}

14
src/test/java/com/microsoft/azure/spring/data/cosmosdb/repository/repository/PageablePersonRepository.java Normal file
Просмотреть файл

@ -0,0 +1,14 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See LICENSE in the project root for
* license information.
*/
package com.microsoft.azure.spring.data.cosmosdb.repository.repository;
import com.microsoft.azure.spring.data.cosmosdb.domain.Person;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PageablePersonRepository extends PagingAndSortingRepository<Person, String> {
}

2
src/test/resources/application.yml
Просмотреть файл

@ -1,4 +1,4 @@
cosmosdb:
key: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==
uri: ${DOCUMENTDB_URI}
telemetryAllowed: true
telemetryAllowed: false