From 18dbdc054c94848d86a06ab1152e801671c0a567 Mon Sep 17 00:00:00 2001 From: Kushagra Thapar Date: Mon, 21 Oct 2019 07:35:53 -0700 Subject: [PATCH] Diagnostics string feature (#445) * Added Response Diagnostics String, and Query Metrics feature. Updated versions to new spring-boot, spring-data versions. Updated sample application with response diagnostics string processor. Updated README on the usage of diagnostics information * Added populateQueryMetrics to application.properties * Added logback-test.xml for test logging * Removed diagnostics logging from tests --- README.md | 24 +++- pom.xml | 10 +- .../cosmosdb/CosmosDbProperties.java | 2 + .../cosmosdb/UserRepositoryConfiguration.java | 20 ++- .../src/main/resources/application.properties | 1 + .../src/test/resources/application.properties | 1 + samplecode/pom.xml | 2 +- .../azure/spring/data/cosmosdb/Constants.java | 2 +- .../data/cosmosdb/common/CosmosdbUtils.java | 33 +++++ .../data/cosmosdb/config/CosmosDBConfig.java | 8 ++ .../data/cosmosdb/core/CosmosTemplate.java | 93 ++++++++++---- .../cosmosdb/core/ReactiveCosmosTemplate.java | 117 ++++++++++++++---- .../cosmosdb/core/ResponseDiagnostics.java | 38 ++++++ .../core/ResponseDiagnosticsProcessor.java | 24 ++++ .../core/convert/MappingCosmosConverter.java | 45 +++---- .../common/ResponseDiagnosticsTestUtils.java | 36 ++++++ .../data/cosmosdb/core/CosmosTemplateIT.java | 68 +++++++++- .../core/ReactiveCosmosTemplateIT.java | 58 ++++++++- src/test/resources/application.properties | 2 + src/test/resources/logback-test.xml | 15 +++ 20 files changed, 503 insertions(+), 96 deletions(-) create mode 100644 src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnostics.java create mode 100644 src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnosticsProcessor.java create mode 100644 src/test/java/com/microsoft/azure/spring/data/cosmosdb/common/ResponseDiagnosticsTestUtils.java create mode 100644 src/test/resources/logback-test.xml diff --git a/README.md b/README.md index 4242dc4..8233710 100644 --- a/README.md +++ b/README.md @@ -101,9 +101,15 @@ Use `@EnableCosmosRepositories` to enable sync repository support. For reactive repository support, use `@EnableReactiveCosmosRepositories` +### Response Diagnostics String and Query Metrics +2.2.x supports Response Diagnostics String and Query Metrics. +Set `populateQueryMetrics` flag to true in application.properties to enable query metrics. +In addition to setting the flag, implement `ResponseDiagnosticsProcessor` to log diagnostics information. + ```java @Configuration @EnableCosmosRepositories +@Slf4j public class AppConfiguration extends AbstractCosmosConfiguration { @Value("${azure.cosmosdb.uri}") @@ -117,17 +123,33 @@ public class AppConfiguration extends AbstractCosmosConfiguration { @Value("${azure.cosmosdb.database}") private String dbName; + + @Value("${azure.cosmosdb.populateQueryMetrics}") + private boolean populateQueryMetrics; private CosmosKeyCredential cosmosKeyCredential; public CosmosDBConfig getConfig() { this.cosmosKeyCredential = new CosmosKeyCredential(key); - return CosmosDBConfig.builder(uri, this.cosmosKeyCredential, dbName).build(); + CosmosDbConfig cosmosdbConfig = CosmosDBConfig.builder(uri, + this.cosmosKeyCredential, dbName).build(); + cosmosdbConfig.setPopulateQueryMetrics(populateQueryMetrics); + cosmosdbConfig.setResponseDiagnosticsProcessor(new ResponseDiagnosticsProcessorImplementation()); + return cosmosdbConfig; } public void switchToSecondaryKey() { this.cosmosKeyCredential.key(secondaryKey); } + + private static class ResponseDiagnosticsProcessorImplementation implements ResponseDiagnosticsProcessor { + + @Override + public void processResponseDiagnostics(@Nullable ResponseDiagnostics responseDiagnostics) { + log.info("Response Diagnostics {}", responseDiagnostics); + } + } + } ``` Or if you want to customize your config: diff --git a/pom.xml b/pom.xml index 309930b..ae9fa2d 100644 --- a/pom.xml +++ b/pom.xml @@ -45,20 +45,20 @@ UTF-8 MM-dd-HH-mm-ss - 5.2.0.RC2 - 2.2.0.RC3 + 5.2.0.RELEASE + 2.2.0.RELEASE 2.9.5 2.8.9 1.7.1 - 1.5.11.RELEASE + 2.2.0.RELEASE 1.16.18 1.2 1.7.25 2.8.4 - 3.2.5.RELEASE + 3.3.0.RELEASE - 3.2.0 + 3.3.0 spring-data-cosmosdb-test testdb-${maven.build.timestamp} true diff --git a/samplecode/example/src/main/java/example/springdata/cosmosdb/CosmosDbProperties.java b/samplecode/example/src/main/java/example/springdata/cosmosdb/CosmosDbProperties.java index 97dc709..39af397 100644 --- a/samplecode/example/src/main/java/example/springdata/cosmosdb/CosmosDbProperties.java +++ b/samplecode/example/src/main/java/example/springdata/cosmosdb/CosmosDbProperties.java @@ -31,4 +31,6 @@ public class CosmosDbProperties { private String secondaryKey; private String database; + + private boolean populateQueryMetrics; } diff --git a/samplecode/example/src/main/java/example/springdata/cosmosdb/UserRepositoryConfiguration.java b/samplecode/example/src/main/java/example/springdata/cosmosdb/UserRepositoryConfiguration.java index 07b661a..86d10bc 100644 --- a/samplecode/example/src/main/java/example/springdata/cosmosdb/UserRepositoryConfiguration.java +++ b/samplecode/example/src/main/java/example/springdata/cosmosdb/UserRepositoryConfiguration.java @@ -18,18 +18,24 @@ package example.springdata.cosmosdb; import com.azure.data.cosmos.CosmosKeyCredential; import com.microsoft.azure.spring.data.cosmosdb.config.AbstractCosmosConfiguration; import com.microsoft.azure.spring.data.cosmosdb.config.CosmosDBConfig; +import com.microsoft.azure.spring.data.cosmosdb.core.ResponseDiagnostics; +import com.microsoft.azure.spring.data.cosmosdb.core.ResponseDiagnosticsProcessor; import com.microsoft.azure.spring.data.cosmosdb.repository.config.EnableReactiveCosmosRepositories; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +import javax.annotation.Nullable; + @Configuration @EnableConfigurationProperties(CosmosDbProperties.class) @EnableReactiveCosmosRepositories @PropertySource("classpath:application.properties") +@Slf4j public class UserRepositoryConfiguration extends AbstractCosmosConfiguration { @Autowired @@ -40,7 +46,11 @@ public class UserRepositoryConfiguration extends AbstractCosmosConfiguration { @Bean public CosmosDBConfig cosmosDbConfig() { this.cosmosKeyCredential = new CosmosKeyCredential(properties.getKey()); - return CosmosDBConfig.builder(properties.getUri(), cosmosKeyCredential, properties.getDatabase()).build(); + CosmosDBConfig cosmosDBConfig = CosmosDBConfig.builder(properties.getUri(), cosmosKeyCredential, + properties.getDatabase()).build(); + cosmosDBConfig.setPopulateQueryMetrics(properties.isPopulateQueryMetrics()); + cosmosDBConfig.setResponseDiagnosticsProcessor(new ResponseDiagnosticsProcessorImplementation()); + return cosmosDBConfig; } public void switchToSecondaryKey() { @@ -54,4 +64,12 @@ public class UserRepositoryConfiguration extends AbstractCosmosConfiguration { public void switchKey(String key) { this.cosmosKeyCredential.key(key); } + + private static class ResponseDiagnosticsProcessorImplementation implements ResponseDiagnosticsProcessor { + + @Override + public void processResponseDiagnostics(@Nullable ResponseDiagnostics responseDiagnostics) { + log.info("Response Diagnostics {}", responseDiagnostics); + } + } } diff --git a/samplecode/example/src/main/resources/application.properties b/samplecode/example/src/main/resources/application.properties index 97f179b..d54c504 100644 --- a/samplecode/example/src/main/resources/application.properties +++ b/samplecode/example/src/main/resources/application.properties @@ -2,3 +2,4 @@ azure.cosmosdb.uri=your-cosmosDb-uri azure.cosmosdb.key=your-cosmosDb-key azure.cosmosdb.secondaryKey=your-cosmosDb-secondary-key azure.cosmosdb.database=your-cosmosDb-dbName +azure.cosmosdb.populateQueryMetrics=if-populate-query-metrics \ No newline at end of file diff --git a/samplecode/example/src/test/resources/application.properties b/samplecode/example/src/test/resources/application.properties index 97f179b..d54c504 100644 --- a/samplecode/example/src/test/resources/application.properties +++ b/samplecode/example/src/test/resources/application.properties @@ -2,3 +2,4 @@ azure.cosmosdb.uri=your-cosmosDb-uri azure.cosmosdb.key=your-cosmosDb-key azure.cosmosdb.secondaryKey=your-cosmosDb-secondary-key azure.cosmosdb.database=your-cosmosDb-dbName +azure.cosmosdb.populateQueryMetrics=if-populate-query-metrics \ No newline at end of file diff --git a/samplecode/pom.xml b/samplecode/pom.xml index c2dd30f..0812cd2 100644 --- a/samplecode/pom.xml +++ b/samplecode/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 2.1.0.RELEASE + 2.2.0.RELEASE diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/Constants.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/Constants.java index 16278d8..8151d3a 100644 --- a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/Constants.java +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/Constants.java @@ -29,6 +29,6 @@ public class Constants { public static final String OBJECTMAPPER_BEAN_NAME = "cosmosdbObjectMapper"; - public static final String ISO_8601_COMPATIBLE_DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:s:SSSXXX"; + public static final String ISO_8601_COMPATIBLE_DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss:SSSXXX"; } diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/common/CosmosdbUtils.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/common/CosmosdbUtils.java index 64cef47..849a340 100644 --- a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/common/CosmosdbUtils.java +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/common/CosmosdbUtils.java @@ -5,13 +5,21 @@ */ package com.microsoft.azure.spring.data.cosmosdb.common; +import com.azure.data.cosmos.CosmosResponse; +import com.azure.data.cosmos.CosmosResponseDiagnostics; +import com.azure.data.cosmos.FeedResponse; +import com.azure.data.cosmos.FeedResponseDiagnostics; import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.azure.spring.data.cosmosdb.core.ResponseDiagnostics; +import com.microsoft.azure.spring.data.cosmosdb.core.ResponseDiagnosticsProcessor; import com.microsoft.azure.spring.data.cosmosdb.core.convert.ObjectMapperFactory; import com.microsoft.azure.spring.data.cosmosdb.exception.ConfigurationException; import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; +@Slf4j public class CosmosdbUtils { @SuppressWarnings("unchecked") @@ -25,4 +33,29 @@ public class CosmosdbUtils { throw new ConfigurationException("failed to get copy from " + instance.getClass().getName(), e); } } + + public static void fillAndProcessResponseDiagnostics(ResponseDiagnosticsProcessor responseDiagnosticsProcessor, + CosmosResponse cosmosResponse, FeedResponse feedResponse) { + if (responseDiagnosticsProcessor == null) { + return; + } + CosmosResponseDiagnostics cosmosResponseDiagnostics = null; + if (cosmosResponse != null) { + cosmosResponseDiagnostics = cosmosResponse.cosmosResponseDiagnosticsString(); + } + FeedResponseDiagnostics feedResponseDiagnostics = null; + if (feedResponse != null) { + feedResponseDiagnostics = feedResponse.feedResponseDiagnostics(); + } + if (cosmosResponseDiagnostics == null && + (feedResponseDiagnostics == null || feedResponseDiagnostics.toString().isEmpty())) { + log.debug("Empty response diagnostics"); + return; + } + final ResponseDiagnostics responseDiagnostics = + new ResponseDiagnostics(cosmosResponseDiagnostics, feedResponseDiagnostics); + + // Process response diagnostics + responseDiagnosticsProcessor.processResponseDiagnostics(responseDiagnostics); + } } diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/config/CosmosDBConfig.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/config/CosmosDBConfig.java index a36607c..154a9ef 100644 --- a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/config/CosmosDBConfig.java +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/config/CosmosDBConfig.java @@ -9,9 +9,11 @@ import com.azure.data.cosmos.ConnectionPolicy; import com.azure.data.cosmos.ConsistencyLevel; import com.azure.data.cosmos.CosmosKeyCredential; import com.azure.data.cosmos.internal.RequestOptions; +import com.microsoft.azure.spring.data.cosmosdb.core.ResponseDiagnosticsProcessor; import com.microsoft.azure.spring.data.cosmosdb.exception.CosmosDBAccessException; import lombok.Builder; import lombok.Getter; +import lombok.Setter; import org.springframework.util.Assert; @Getter @@ -33,6 +35,12 @@ public class CosmosDBConfig { private CosmosKeyCredential cosmosKeyCredential; + @Setter + private ResponseDiagnosticsProcessor responseDiagnosticsProcessor; + + @Setter + private boolean populateQueryMetrics; + public static CosmosDBConfigBuilder builder(String uri, CosmosKeyCredential cosmosKeyCredential, String database) { return defaultBuilder() diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplate.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplate.java index 4603ced..c60c0b5 100644 --- a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplate.java +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplate.java @@ -30,9 +30,6 @@ import com.microsoft.azure.spring.data.cosmosdb.core.query.DocumentQuery; import com.microsoft.azure.spring.data.cosmosdb.exception.CosmosDBAccessException; import com.microsoft.azure.spring.data.cosmosdb.repository.support.CosmosEntityInformation; import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -42,6 +39,8 @@ import org.springframework.data.domain.Pageable; import org.springframework.lang.NonNull; import org.springframework.util.Assert; import org.springframework.util.StringUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; import java.util.ArrayList; import java.util.Collections; @@ -50,6 +49,8 @@ import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; +import static com.microsoft.azure.spring.data.cosmosdb.common.CosmosdbUtils.fillAndProcessResponseDiagnostics; + @Slf4j public class CosmosTemplate implements CosmosOperations, ApplicationContextAware { @@ -57,6 +58,8 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware private final MappingCosmosConverter mappingCosmosConverter; private final String databaseName; + private final ResponseDiagnosticsProcessor responseDiagnosticsProcessor; + private final boolean isPopulateQueryMetrics; private final CosmosClient cosmosClient; private Function, CosmosEntityInformation> entityInfoCreator = @@ -72,6 +75,8 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware this.databaseName = dbName; this.cosmosClient = cosmosDbFactory.getCosmosClient(); + this.responseDiagnosticsProcessor = cosmosDbFactory.getConfig().getResponseDiagnosticsProcessor(); + this.isPopulateQueryMetrics = cosmosDbFactory.getConfig().isPopulateQueryMetrics(); } public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { @@ -101,6 +106,8 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware final CosmosItemResponse response = cosmosClient.getDatabase(this.databaseName) .getContainer(collectionName) .createItem(originalItem, options) + .doOnNext(cosmosItemResponse -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null)) .onErrorResume(Mono::error) .block(); @@ -134,8 +141,12 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware .getContainer(collectionName) .getItem(id.toString(), partitionKey) .read() - .flatMap(cosmosItemResponse -> Mono.justOrEmpty(toDomainObject(entityClass, - cosmosItemResponse.properties()))) + .flatMap(cosmosItemResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null); + return Mono.justOrEmpty(toDomainObject(entityClass, + cosmosItemResponse.properties())); + }) .onErrorResume(Mono::error) .block(); @@ -154,15 +165,20 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware final String query = String.format("select * from root where root.id = '%s'", id.toString()); final FeedOptions options = new FeedOptions(); options.enableCrossPartitionQuery(true); + options.populateQueryMetrics(isPopulateQueryMetrics); return cosmosClient .getDatabase(databaseName) .getContainer(collectionName) .queryItems(query, options) - .flatMap(cosmosItemFeedResponse -> Mono.justOrEmpty(cosmosItemFeedResponse + .flatMap(cosmosItemFeedResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, cosmosItemFeedResponse); + return Mono.justOrEmpty(cosmosItemFeedResponse .results() .stream() .map(cosmosItem -> mappingCosmosConverter.read(domainClass, cosmosItem)) - .findFirst())) + .findFirst()); + }) .onErrorResume(Mono::error) .blockFirst(); @@ -193,12 +209,15 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware final CosmosItemResponse cosmosItemResponse = cosmosClient.getDatabase(this.databaseName) .getContainer(collectionName) .upsertItem(originalItem, options) + .doOnNext(response -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + response, null)) .onErrorResume(Mono::error) .block(); if (cosmosItemResponse == null) { throw new CosmosDBAccessException("Failed to upsert item"); } + } catch (Exception ex) { throw new CosmosDBAccessException("Failed to upsert document to database.", ex); } @@ -234,7 +253,13 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware public void deleteCollection(@NonNull String collectionName) { Assert.hasText(collectionName, "collectionName should have text."); try { - cosmosClient.getDatabase(this.databaseName).getContainer(collectionName).delete().block(); + cosmosClient + .getDatabase(this.databaseName) + .getContainer(collectionName) + .delete() + .doOnNext(response -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + response, null)) + .block(); } catch (Exception e) { throw new CosmosDBAccessException("failed to delete collection: " + collectionName, e); @@ -251,11 +276,17 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware public CosmosContainerProperties createCollectionIfNotExists(@NonNull CosmosEntityInformation information) { final CosmosContainerResponse response = cosmosClient .createDatabaseIfNotExists(this.databaseName) - .flatMap(cosmosDatabaseResponse -> cosmosDatabaseResponse + .flatMap(cosmosDatabaseResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosDatabaseResponse, null); + return cosmosDatabaseResponse .database() .createContainerIfNotExists(information.getCollectionName(), - "/" + information.getPartitionKeyFieldName(), information.getRequestUnit()) - .map(cosmosContainerResponse -> cosmosContainerResponse)) + "/" + information.getPartitionKeyFieldName(), information.getRequestUnit()) + .doOnNext(cosmosContainerResponse -> + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosContainerResponse, null)); + }) .block(); if (response == null) { throw new CosmosDBAccessException("Failed to create collection"); @@ -276,12 +307,14 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware final CosmosItemRequestOptions options = new CosmosItemRequestOptions(); options.partitionKey(partitionKey); cosmosClient.getDatabase(this.databaseName) - .getContainer(collectionName) - .getItem(id.toString(), partitionKey) - .delete(options) - .onErrorResume(Mono::error) - .then() - .block(); + .getContainer(collectionName) + .getItem(id.toString(), partitionKey) + .delete(options) + .doOnNext(response -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + response, null)) + .onErrorResume(Mono::error) + .block(); + } catch (Exception e) { throw new CosmosDBAccessException("deleteById exception", e); } @@ -368,12 +401,15 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware feedOptions.maxItemCount(pageable.getPageSize()); feedOptions.enableCrossPartitionQuery(query.isCrossPartitionQuery(getPartitionKeyNames(domainClass))); + feedOptions.populateQueryMetrics(isPopulateQueryMetrics); final SqlQuerySpec sqlQuerySpec = new FindQuerySpecGenerator().generateCosmos(query); final FeedResponse feedResponse = cosmosClient.getDatabase(this.databaseName) .getContainer(collectionName) .queryItems(sqlQuerySpec, feedOptions) + .doOnNext(propertiesFeedResponse -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, propertiesFeedResponse)) .next() .block(); @@ -439,9 +475,12 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware final FeedOptions options = new FeedOptions(); options.enableCrossPartitionQuery(isCrossPartitionQuery); + options.populateQueryMetrics(isPopulateQueryMetrics); return executeQuery(querySpec, containerName, options) .onErrorResume(this::databaseAccessExceptionHandler) + .doOnNext(response -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, response)) .next() .map(r -> r.results().get(0).getLong(COUNT_VALUE_KEY)) .block(); @@ -483,11 +522,17 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware query.isCrossPartitionQuery(getPartitionKeyNames(domainClass)); final FeedOptions feedOptions = new FeedOptions(); feedOptions.enableCrossPartitionQuery(isCrossPartitionQuery); + feedOptions.populateQueryMetrics(isPopulateQueryMetrics); + return cosmosClient .getDatabase(this.databaseName) .getContainer(containerName) .queryItems(sqlQuerySpec, feedOptions) - .flatMap(cosmosItemFeedResponse -> Flux.fromIterable(cosmosItemFeedResponse.results())) + .flatMap(cosmosItemFeedResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, cosmosItemFeedResponse); + return Flux.fromIterable(cosmosItemFeedResponse.results()); + }) .collectList() .block(); } @@ -512,11 +557,13 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware applyVersioning(domainClass, cosmosItemProperties, options); return cosmosClient - .getDatabase(this.databaseName) - .getContainer(containerName) - .getItem(cosmosItemProperties.id(), partitionKey) - .delete(options) - .block(); + .getDatabase(this.databaseName) + .getContainer(containerName) + .getItem(cosmosItemProperties.id(), partitionKey) + .delete(options) + .doOnNext(response -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + response, null)) + .block(); } private T toDomainObject(@NonNull Class domainClass, CosmosItemProperties cosmosItemProperties) { diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplate.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplate.java index 17fa5ad..4d5e8fb 100644 --- a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplate.java +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplate.java @@ -37,6 +37,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static com.microsoft.azure.spring.data.cosmosdb.common.CosmosdbUtils.fillAndProcessResponseDiagnostics; + @Slf4j public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, ApplicationContextAware { private static final String COUNT_VALUE_KEY = "_aggregate"; @@ -45,6 +47,8 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica private final String databaseName; private final CosmosClient cosmosClient; + private final ResponseDiagnosticsProcessor responseDiagnosticsProcessor; + private final boolean isPopulateQueryMetrics; private final List collectionCache; @@ -66,6 +70,8 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica this.collectionCache = new ArrayList<>(); this.cosmosClient = cosmosDbFactory.getCosmosClient(); + this.responseDiagnosticsProcessor = cosmosDbFactory.getConfig().getResponseDiagnosticsProcessor(); + this.isPopulateQueryMetrics = cosmosDbFactory.getConfig().isPopulateQueryMetrics(); } /** @@ -87,14 +93,20 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica return cosmosClient .createDatabaseIfNotExists(this.databaseName) - .flatMap(cosmosDatabaseResponse -> cosmosDatabaseResponse - .database() - .createContainerIfNotExists(information.getCollectionName(), - "/" + information.getPartitionKeyFieldName(), information.getRequestUnit()) - .map(cosmosContainerResponse -> { - this.collectionCache.add(information.getCollectionName()); - return cosmosContainerResponse; - })); + .flatMap(cosmosDatabaseResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosDatabaseResponse, null); + return cosmosDatabaseResponse + .database() + .createContainerIfNotExists(information.getCollectionName(), + "/" + information.getPartitionKeyFieldName(), information.getRequestUnit()) + .map(cosmosContainerResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosContainerResponse, null); + this.collectionCache.add(information.getCollectionName()); + return cosmosContainerResponse; + }); + }); } @@ -153,16 +165,22 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica final String query = String.format("select * from root where root.id = '%s'", id.toString()); final FeedOptions options = new FeedOptions(); options.enableCrossPartitionQuery(true); + options.populateQueryMetrics(isPopulateQueryMetrics); + return cosmosClient.getDatabase(databaseName) - .getContainer(containerName) - .queryItems(query, options) - .flatMap(cosmosItemFeedResponse -> Mono.justOrEmpty(cosmosItemFeedResponse - .results() - .stream() - .map(cosmosItem -> toDomainObject(entityClass, cosmosItem)) - .findFirst())) - .onErrorResume(this::databaseAccessExceptionHandler) - .next(); + .getContainer(containerName) + .queryItems(query, options) + .flatMap(cosmosItemFeedResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, cosmosItemFeedResponse); + return Mono.justOrEmpty(cosmosItemFeedResponse + .results() + .stream() + .map(cosmosItem -> toDomainObject(entityClass, cosmosItem)) + .findFirst()); + }) + .onErrorResume(this::databaseAccessExceptionHandler) + .next(); } /** @@ -183,8 +201,12 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica .getContainer(containerName) .getItem(id.toString(), partitionKey) .read() - .flatMap(cosmosItemResponse -> Mono.justOrEmpty(toDomainObject(entityClass, - cosmosItemResponse.properties()))) + .flatMap(cosmosItemResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null); + return Mono.justOrEmpty(toDomainObject(entityClass, + cosmosItemResponse.properties())); + }) .onErrorResume(this::databaseAccessExceptionHandler); } @@ -215,7 +237,11 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica .getContainer(getContainerName(objectToSave.getClass())) .createItem(objectToSave, new CosmosItemRequestOptions()) .onErrorResume(this::databaseAccessExceptionHandler) - .flatMap(cosmosItemResponse -> Mono.just(toDomainObject(domainClass, cosmosItemResponse.properties()))); + .flatMap(cosmosItemResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null); + return Mono.just(toDomainObject(domainClass, cosmosItemResponse.properties())); + }); } /** @@ -240,7 +266,11 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica .getContainer(containerName) .createItem(objectToSave, options) .onErrorResume(this::databaseAccessExceptionHandler) - .flatMap(cosmosItemResponse -> Mono.just(toDomainObject(domainClass, cosmosItemResponse.properties()))); + .flatMap(cosmosItemResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null); + return Mono.just(toDomainObject(domainClass, cosmosItemResponse.properties())); + }); } /** @@ -274,7 +304,11 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica return cosmosClient.getDatabase(this.databaseName) .getContainer(containerName) .upsertItem(object, options) - .flatMap(cosmosItemResponse -> Mono.just(toDomainObject(domainClass, cosmosItemResponse.properties()))) + .flatMap(cosmosItemResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null); + return Mono.just(toDomainObject(domainClass, cosmosItemResponse.properties())); + }) .onErrorResume(this::databaseAccessExceptionHandler); } @@ -298,6 +332,9 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica .getContainer(containerName) .getItem(id.toString(), partitionKey) .delete(options) + .doOnNext(cosmosItemResponse -> + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null)) .onErrorResume(this::databaseAccessExceptionHandler) .then(); } @@ -320,15 +357,22 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica final FeedOptions options = new FeedOptions(); final boolean isCrossPartitionQuery = query.isCrossPartitionQuery(Collections.singletonList(partitionKeyName)); options.enableCrossPartitionQuery(isCrossPartitionQuery); + options.populateQueryMetrics(isPopulateQueryMetrics); return cosmosClient.getDatabase(this.databaseName) .getContainer(containerName) .queryItems(sqlQuerySpec, options) - .flatMap(cosmosItemFeedResponse -> Flux.fromIterable(cosmosItemFeedResponse.results())) + .flatMap(cosmosItemFeedResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, cosmosItemFeedResponse); + return Flux.fromIterable(cosmosItemFeedResponse.results()); + }) .flatMap(cosmosItemProperties -> cosmosClient .getDatabase(this.databaseName) .getContainer(containerName) .getItem(cosmosItemProperties.id(), cosmosItemProperties.get(partitionKeyName)) - .delete()) + .delete() + .doOnNext(cosmosItemResponse -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null))) .onErrorResume(this::databaseAccessExceptionHandler) .then(); } @@ -431,8 +475,11 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica final FeedOptions options = new FeedOptions(); options.enableCrossPartitionQuery(isCrossPartitionQuery); + options.populateQueryMetrics(isPopulateQueryMetrics); return executeQuery(querySpec, containerName, options) + .doOnNext(feedResponse -> fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, feedResponse)) .onErrorResume(this::databaseAccessExceptionHandler) .next() .map(r -> r.results().get(0).getLong(COUNT_VALUE_KEY)); @@ -459,7 +506,13 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica public void deleteContainer(@NonNull String containerName) { Assert.hasText(containerName, "containerName should have text."); try { - cosmosClient.getDatabase(this.databaseName).getContainer(containerName).delete().block(); + cosmosClient.getDatabase(this.databaseName) + .getContainer(containerName) + .delete() + .doOnNext(cosmosContainerResponse -> + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosContainerResponse, null)) + .block(); this.collectionCache.remove(containerName); } catch (Exception e) { throw new CosmosDBAccessException("failed to delete collection: " + containerName, e); @@ -482,11 +535,17 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica final boolean isCrossPartitionQuery = query.isCrossPartitionQuery(getPartitionKeyNames(domainClass)); final FeedOptions feedOptions = new FeedOptions(); feedOptions.enableCrossPartitionQuery(isCrossPartitionQuery); + feedOptions.populateQueryMetrics(isPopulateQueryMetrics); + return cosmosClient .getDatabase(this.databaseName) .getContainer(containerName) .queryItems(sqlQuerySpec, feedOptions) - .flatMap(cosmosItemFeedResponse -> Flux.fromIterable(cosmosItemFeedResponse.results())); + .flatMap(cosmosItemFeedResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + null, cosmosItemFeedResponse); + return Flux.fromIterable(cosmosItemFeedResponse.results()); + }); } private void assertValidId(Object id) { @@ -524,7 +583,11 @@ public class ReactiveCosmosTemplate implements ReactiveCosmosOperations, Applica .getContainer(containerName) .getItem(cosmosItemProperties.id(), partitionKey) .delete(options) - .map(cosmosItemResponse -> cosmosItemProperties); + .map(cosmosItemResponse -> { + fillAndProcessResponseDiagnostics(responseDiagnosticsProcessor, + cosmosItemResponse, null); + return cosmosItemProperties; + }); } private T toDomainObject(@NonNull Class domainClass, CosmosItemProperties cosmosItemProperties) { diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnostics.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnostics.java new file mode 100644 index 0000000..0c4481c --- /dev/null +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnostics.java @@ -0,0 +1,38 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for + * license information. + */ + +package com.microsoft.azure.spring.data.cosmosdb.core; + +import com.azure.data.cosmos.CosmosResponseDiagnostics; +import com.azure.data.cosmos.FeedResponseDiagnostics; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +public class ResponseDiagnostics { + + private CosmosResponseDiagnostics cosmosResponseDiagnostics; + private FeedResponseDiagnostics feedResponseDiagnostics; + + @Override + public String toString() { + final StringBuilder diagnostics = new StringBuilder(); + if (cosmosResponseDiagnostics != null) { + diagnostics.append("cosmosResponseDiagnostics={") + .append(cosmosResponseDiagnostics) + .append("}"); + } + if (feedResponseDiagnostics != null) { + diagnostics.append("feedResponseDiagnostics={") + .append(feedResponseDiagnostics) + .append("}"); + } + return diagnostics.toString(); + } +} diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnosticsProcessor.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnosticsProcessor.java new file mode 100644 index 0000000..1c41ff4 --- /dev/null +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/ResponseDiagnosticsProcessor.java @@ -0,0 +1,24 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for + * license information. + */ + +package com.microsoft.azure.spring.data.cosmosdb.core; + +import javax.annotation.Nullable; + +public interface ResponseDiagnosticsProcessor { + + /** + * Gets called after receiving response from CosmosDb. + * Response Diagnostics are collected from API responses and + * then set in {@link ResponseDiagnostics} object. + *

+ * In case of missing diagnostics from CosmosDb, responseDiagnostics will be null. + * + * @param responseDiagnostics responseDiagnostics object containing CosmosDb response + * diagnostics information + */ + void processResponseDiagnostics(@Nullable ResponseDiagnostics responseDiagnostics); +} diff --git a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/convert/MappingCosmosConverter.java b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/convert/MappingCosmosConverter.java index 62f9938..d1e9105 100644 --- a/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/convert/MappingCosmosConverter.java +++ b/src/main/java/com/microsoft/azure/spring/data/cosmosdb/core/convert/MappingCosmosConverter.java @@ -6,6 +6,9 @@ package com.microsoft.azure.spring.data.cosmosdb.core.convert; import com.azure.data.cosmos.CosmosItemProperties; +import com.azure.data.cosmos.internal.Utils; +import com.azure.data.cosmos.internal.query.QueryItem; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.azure.spring.data.cosmosdb.Constants; @@ -13,6 +16,7 @@ import com.microsoft.azure.spring.data.cosmosdb.core.mapping.CosmosPersistentEnt import com.microsoft.azure.spring.data.cosmosdb.core.mapping.CosmosPersistentProperty; import com.microsoft.azure.spring.data.cosmosdb.exception.CosmosDBAccessException; import org.json.JSONObject; +import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -50,6 +54,11 @@ public class MappingCosmosConverter this.conversionService = new GenericConversionService(); this.objectMapper = objectMapper == null ? ObjectMapperFactory.getObjectMapper() : objectMapper; + + // CosmosDB SDK serializes and deserializes logger, which causes this issue: + // https://github.com/microsoft/spring-data-cosmosdb/issues/423 + // This is a temporary fix while CosmosDB fixes this problem. + Utils.getSimpleObjectMapper().addMixIn(QueryItem.class, QueryItemMixIn.class); } @Override @@ -91,37 +100,6 @@ public class MappingCosmosConverter throw new UnsupportedOperationException("The feature is not implemented yet"); } - public CosmosItemProperties writeDoc(Object sourceEntity) { - if (sourceEntity == null) { - return null; - } - - final CosmosPersistentEntity persistentEntity = - mappingContext.getPersistentEntity(sourceEntity.getClass()); - - if (persistentEntity == null) { - throw new MappingException("no mapping metadata for entity type: " + sourceEntity.getClass().getName()); - } - - final ConvertingPropertyAccessor accessor = getPropertyAccessor(sourceEntity); - final CosmosPersistentProperty idProperty = persistentEntity.getIdProperty(); - final CosmosItemProperties document; - - try { - document = new CosmosItemProperties(objectMapper.writeValueAsString(sourceEntity)); - } catch (JsonProcessingException e) { - throw new CosmosDBAccessException("Failed to map document value.", e); - } - - if (idProperty != null) { - final Object value = accessor.getProperty(idProperty); - final String id = value == null ? null : value.toString(); - document.id(id); - } - - return document; - } - public CosmosItemProperties writeCosmosItemProperties(Object sourceEntity) { if (sourceEntity == null) { return null; @@ -207,4 +185,9 @@ public class MappingCosmosConverter return fromPropertyValue; } + + interface QueryItemMixIn { + @JsonIgnore + Logger getLogger(); + } } diff --git a/src/test/java/com/microsoft/azure/spring/data/cosmosdb/common/ResponseDiagnosticsTestUtils.java b/src/test/java/com/microsoft/azure/spring/data/cosmosdb/common/ResponseDiagnosticsTestUtils.java new file mode 100644 index 0000000..b836b6f --- /dev/null +++ b/src/test/java/com/microsoft/azure/spring/data/cosmosdb/common/ResponseDiagnosticsTestUtils.java @@ -0,0 +1,36 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for + * license information. + */ + +package com.microsoft.azure.spring.data.cosmosdb.common; + +import com.azure.data.cosmos.CosmosResponseDiagnostics; +import com.azure.data.cosmos.FeedResponseDiagnostics; +import com.microsoft.azure.spring.data.cosmosdb.core.ResponseDiagnostics; +import com.microsoft.azure.spring.data.cosmosdb.core.ResponseDiagnosticsProcessor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Getter +@Slf4j +public class ResponseDiagnosticsTestUtils { + + private final ResponseDiagnosticsProcessor responseDiagnosticsProcessor; + private ResponseDiagnostics diagnostics; + + public ResponseDiagnosticsTestUtils() { + responseDiagnosticsProcessor = responseDiagnostics -> { + diagnostics = responseDiagnostics; + }; + } + + public CosmosResponseDiagnostics getCosmosResponseDiagnostics() { + return diagnostics == null ? null : diagnostics.getCosmosResponseDiagnostics(); + } + + public FeedResponseDiagnostics getFeedResponseDiagnostics() { + return diagnostics == null ? null : diagnostics.getFeedResponseDiagnostics(); + } +} diff --git a/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplateIT.java b/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplateIT.java index 6baa424..7e6fd82 100644 --- a/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplateIT.java +++ b/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/CosmosTemplateIT.java @@ -9,6 +9,7 @@ package com.microsoft.azure.spring.data.cosmosdb.core; import com.azure.data.cosmos.CosmosClientException; import com.azure.data.cosmos.PartitionKey; import com.microsoft.azure.spring.data.cosmosdb.CosmosDbFactory; +import com.microsoft.azure.spring.data.cosmosdb.common.ResponseDiagnosticsTestUtils; import com.microsoft.azure.spring.data.cosmosdb.config.CosmosDBConfig; import com.microsoft.azure.spring.data.cosmosdb.core.convert.MappingCosmosConverter; import com.microsoft.azure.spring.data.cosmosdb.core.mapping.CosmosMappingContext; @@ -24,7 +25,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScanner; @@ -36,6 +36,10 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + import static com.microsoft.azure.spring.data.cosmosdb.common.PageTestUtils.validateLastPage; import static com.microsoft.azure.spring.data.cosmosdb.common.PageTestUtils.validateNonLastPage; import static com.microsoft.azure.spring.data.cosmosdb.common.TestConstants.ADDRESSES; @@ -57,10 +61,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import java.util.Collections; -import java.util.List; -import java.util.UUID; - @RunWith(SpringJUnit4ClassRunner.class) @PropertySource(value = { "classpath:application.properties" }) public class CosmosTemplateIT { @@ -83,11 +83,14 @@ public class CosmosTemplateIT { private String cosmosDbUri; @Value("${cosmosdb.key}") private String cosmosDbKey; + @Value("${cosmosdb.populateQueryMetrics}") + private boolean populateQueryMetrics; private static CosmosTemplate cosmosTemplate; private static CosmosEntityInformation personInfo; private static String collectionName; private static boolean initialized; + private static ResponseDiagnosticsTestUtils responseDiagnosticsTestUtils; private Person insertedPerson; @@ -97,8 +100,11 @@ public class CosmosTemplateIT { @Before public void setup() throws ClassNotFoundException { if (!initialized) { + responseDiagnosticsTestUtils = new ResponseDiagnosticsTestUtils(); final CosmosDBConfig dbConfig = CosmosDBConfig.builder(cosmosDbUri, cosmosDbKey, DB_NAME).build(); + dbConfig.setResponseDiagnosticsProcessor(responseDiagnosticsTestUtils.getResponseDiagnosticsProcessor()); + dbConfig.setPopulateQueryMetrics(populateQueryMetrics); final CosmosDbFactory cosmosDbFactory = new CosmosDbFactory(dbConfig); final CosmosMappingContext mappingContext = new CosmosMappingContext(); @@ -133,6 +139,7 @@ public class CosmosTemplateIT { final List result = cosmosTemplate.findAll(Person.class.getSimpleName(), Person.class); assertThat(result.size()).isEqualTo(1); assertThat(result.get(0)).isEqualTo(TEST_PERSON); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -140,6 +147,7 @@ public class CosmosTemplateIT { final Person result = cosmosTemplate.findById(Person.class.getSimpleName(), TEST_PERSON.getId(), Person.class); assertEquals(result, TEST_PERSON); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); final Person nullResult = cosmosTemplate.findById(Person.class.getSimpleName(), NOT_EXIST_ID, Person.class); @@ -156,6 +164,8 @@ public class CosmosTemplateIT { final List ids = Lists.newArrayList(ID_1, ID_2, ID_3); final List result = cosmosTemplate.findByIds(ids, Person.class, collectionName); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + final List expected = Lists.newArrayList(TEST_PERSON, TEST_PERSON_2, TEST_PERSON_3); assertThat(result.size()).isEqualTo(expected.size()); assertThat(result).containsAll(expected); @@ -174,8 +184,13 @@ public class CosmosTemplateIT { cosmosTemplate.upsert(Person.class.getSimpleName(), newPerson, new PartitionKey(personInfo.getPartitionKeyFieldValue(newPerson))); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + final List result = cosmosTemplate.findAll(Person.class); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + assertThat(result.size()).isEqualTo(1); assertEquals(result.get(0).getFirstName(), firstName); } @@ -188,9 +203,14 @@ public class CosmosTemplateIT { cosmosTemplate.upsert(Person.class.getSimpleName(), updated, null); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + final Person result = cosmosTemplate.findById(Person.class.getSimpleName(), updated.getId(), Person.class); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + assertEquals(result, updated); } @@ -225,7 +245,12 @@ public class CosmosTemplateIT { cosmosTemplate.deleteById(Person.class.getSimpleName(), TEST_PERSON.getId(), new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON))); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + final List result = cosmosTemplate.findAll(Person.class); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); assertThat(result.size()).isEqualTo(1); assertEquals(result.get(0), TEST_PERSON_2); } @@ -235,11 +260,18 @@ public class CosmosTemplateIT { final long prevCount = cosmosTemplate.count(collectionName); assertThat(prevCount).isEqualTo(1); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + cosmosTemplate.insert(TEST_PERSON_2, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_2))); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + final long newCount = cosmosTemplate.count(collectionName); assertThat(newCount).isEqualTo(2); + + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -247,12 +279,17 @@ public class CosmosTemplateIT { cosmosTemplate.insert(TEST_PERSON_2, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_2))); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + final Criteria criteria = Criteria.getInstance(CriteriaType.IS_EQUAL, "firstName", Collections.singletonList(TEST_PERSON_2.getFirstName())); final DocumentQuery query = new DocumentQuery(criteria); final long count = cosmosTemplate.count(query, Person.class, collectionName); assertThat(count).isEqualTo(1); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -260,16 +297,25 @@ public class CosmosTemplateIT { cosmosTemplate.insert(TEST_PERSON_2, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_2))); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + final CosmosPageRequest pageRequest = new CosmosPageRequest(0, PAGE_SIZE_1, null); final Page page1 = cosmosTemplate.findAll(pageRequest, Person.class, collectionName); assertThat(page1.getContent().size()).isEqualTo(PAGE_SIZE_1); validateNonLastPage(page1, PAGE_SIZE_1); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + final Page page2 = cosmosTemplate.findAll(page1.getPageable(), Person.class, collectionName); assertThat(page2.getContent().size()).isEqualTo(1); validateLastPage(page2, PAGE_SIZE_1); + + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -277,6 +323,9 @@ public class CosmosTemplateIT { cosmosTemplate.insert(TEST_PERSON_2, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_2))); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + final Criteria criteria = Criteria.getInstance(CriteriaType.IS_EQUAL, "firstName", Collections.singletonList(FIRST_NAME)); final PageRequest pageRequest = new CosmosPageRequest(0, PAGE_SIZE_2, null); @@ -285,6 +334,9 @@ public class CosmosTemplateIT { final Page page = cosmosTemplate.paginationQuery(query, Person.class, collectionName); assertThat(page.getContent().size()).isEqualTo(1); validateLastPage(page, PAGE_SIZE_2); + + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -294,6 +346,9 @@ public class CosmosTemplateIT { cosmosTemplate.insert(TEST_PERSON_3, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_3))); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + final Sort sort = Sort.by(Sort.Direction.DESC, "firstName"); final PageRequest pageRequest = CosmosPageRequest.of(0, PAGE_SIZE_3, null, sort); @@ -306,6 +361,9 @@ public class CosmosTemplateIT { assertThat(result.get(1).getFirstName()).isEqualTo(NEW_FIRST_NAME); assertThat(result.get(2).getFirstName()).isEqualTo(FIRST_NAME); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + } @Test diff --git a/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplateIT.java b/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplateIT.java index db8cd8b..f383806 100644 --- a/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplateIT.java +++ b/src/test/java/com/microsoft/azure/spring/data/cosmosdb/core/ReactiveCosmosTemplateIT.java @@ -5,10 +5,10 @@ */ package com.microsoft.azure.spring.data.cosmosdb.core; -import com.azure.data.cosmos.CosmosClientException; import com.azure.data.cosmos.CosmosKeyCredential; import com.azure.data.cosmos.PartitionKey; import com.microsoft.azure.spring.data.cosmosdb.CosmosDbFactory; +import com.microsoft.azure.spring.data.cosmosdb.common.ResponseDiagnosticsTestUtils; import com.microsoft.azure.spring.data.cosmosdb.common.TestConstants; import com.microsoft.azure.spring.data.cosmosdb.config.CosmosDBConfig; import com.microsoft.azure.spring.data.cosmosdb.core.convert.MappingCosmosConverter; @@ -70,6 +70,8 @@ public class ReactiveCosmosTemplateIT { private String cosmosDbKey; @Value("${cosmosdb.secondaryKey}") private String cosmosDbSecondaryKey; + @Value("${cosmosdb.populateQueryMetrics}") + private boolean populateQueryMetrics; private static ReactiveCosmosTemplate cosmosTemplate; private static String containerName; @@ -77,6 +79,7 @@ public class ReactiveCosmosTemplateIT { private static CosmosKeyCredential cosmosKeyCredential; private static boolean initialized; + private static ResponseDiagnosticsTestUtils responseDiagnosticsTestUtils; @Autowired private ApplicationContext applicationContext; @@ -84,9 +87,12 @@ public class ReactiveCosmosTemplateIT { @Before public void setUp() throws ClassNotFoundException { if (!initialized) { + responseDiagnosticsTestUtils = new ResponseDiagnosticsTestUtils(); cosmosKeyCredential = new CosmosKeyCredential(cosmosDbKey); final CosmosDBConfig dbConfig = CosmosDBConfig.builder(cosmosDbUri, cosmosKeyCredential, DB_NAME).build(); + dbConfig.setResponseDiagnosticsProcessor(responseDiagnosticsTestUtils.getResponseDiagnosticsProcessor()); + dbConfig.setPopulateQueryMetrics(populateQueryMetrics); final CosmosDbFactory dbFactory = new CosmosDbFactory(dbConfig); final CosmosMappingContext mappingContext = new CosmosMappingContext(); @@ -101,6 +107,7 @@ public class ReactiveCosmosTemplateIT { cosmosTemplate.createCollectionIfNotExists(personInfo).block().container(); initialized = true; } + cosmosTemplate.insert(TEST_PERSON, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON))).block(); } @@ -135,6 +142,7 @@ public class ReactiveCosmosTemplateIT { StepVerifier.create(findById) .consumeNextWith(actual -> Assert.assertEquals(actual, TEST_PERSON)) .verifyComplete(); + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -147,6 +155,8 @@ public class ReactiveCosmosTemplateIT { Assert.assertThat(actual.getFirstName(), is(equalTo(TEST_PERSON.getFirstName()))); Assert.assertThat(actual.getLastName(), is(equalTo(TEST_PERSON.getLastName()))); }).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -154,6 +164,8 @@ public class ReactiveCosmosTemplateIT { final Flux flux = cosmosTemplate.findAll(Person.class.getSimpleName(), Person.class); StepVerifier.create(flux).expectNextCount(1).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -162,6 +174,8 @@ public class ReactiveCosmosTemplateIT { TEST_PERSON.getId(), Person.class)) .consumeNextWith(actual -> Assert.assertEquals(actual, TEST_PERSON)) .verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); } @Test @@ -169,6 +183,9 @@ public class ReactiveCosmosTemplateIT { StepVerifier.create(cosmosTemplate.insert(TEST_PERSON_3, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_3)))) .expectNext(TEST_PERSON_3).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); } @Test @@ -177,6 +194,9 @@ public class ReactiveCosmosTemplateIT { StepVerifier.create(cosmosTemplate.insert(TEST_PERSON_3, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_3)))) .expectNext(TEST_PERSON_3).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); } @Test @@ -184,6 +204,9 @@ public class ReactiveCosmosTemplateIT { StepVerifier.create(cosmosTemplate.insert(Person.class.getSimpleName(), TEST_PERSON_2, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_2)))) .expectNext(TEST_PERSON_2).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); } @Test @@ -195,6 +218,9 @@ public class ReactiveCosmosTemplateIT { final Mono upsert = cosmosTemplate.upsert(p, new PartitionKey(personInfo.getPartitionKeyFieldValue(p))); StepVerifier.create(upsert).expectNextCount(1).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); } @Test @@ -207,6 +233,9 @@ public class ReactiveCosmosTemplateIT { final Mono upsert = cosmosTemplate.upsert(p, new PartitionKey(personInfo.getPartitionKeyFieldValue(p))); StepVerifier.create(upsert).expectNextCount(1).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); } @Test @@ -218,20 +247,38 @@ public class ReactiveCosmosTemplateIT { final Mono upsert = cosmosTemplate.upsert(Person.class.getSimpleName(), p, new PartitionKey(personInfo.getPartitionKeyFieldValue(p))); StepVerifier.create(upsert).expectNextCount(1).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); } @Test public void testDeleteById() { cosmosTemplate.insert(TEST_PERSON_4, new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_4))).block(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + Flux flux = cosmosTemplate.findAll(Person.class.getSimpleName(), Person.class); StepVerifier.create(flux).expectNextCount(2).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); + final Mono voidMono = cosmosTemplate.deleteById(Person.class.getSimpleName(), TEST_PERSON_4.getId(), new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON_4))); StepVerifier.create(voidMono).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNotNull(); + flux = cosmosTemplate.findAll(Person.class.getSimpleName(), Person.class); StepVerifier.create(flux).expectNextCount(1).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); } @Test @@ -257,6 +304,9 @@ public class ReactiveCosmosTemplateIT { final Flux personFlux = cosmosTemplate.find(query, Person.class, Person.class.getSimpleName()); StepVerifier.create(personFlux).expectNextCount(1).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); } @Test @@ -266,12 +316,18 @@ public class ReactiveCosmosTemplateIT { final DocumentQuery query = new DocumentQuery(criteria); final Mono exists = cosmosTemplate.exists(query, Person.class, containerName); StepVerifier.create(exists).expectNext(true).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); } @Test public void testCount() { final Mono count = cosmosTemplate.count(containerName); StepVerifier.create(count).expectNext((long) 1).verifyComplete(); + + assertThat(responseDiagnosticsTestUtils.getFeedResponseDiagnostics()).isNotNull(); + assertThat(responseDiagnosticsTestUtils.getCosmosResponseDiagnostics()).isNull(); } @Test diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 1a70c6b..85ff3dd 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -11,3 +11,5 @@ perf.recursive.times=10 perf.batch.size=3 perf.acceptance.percentage=10 +# Populate query metrics +cosmosdb.populateQueryMetrics=true diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml new file mode 100644 index 0000000..41cebed --- /dev/null +++ b/src/test/resources/logback-test.xml @@ -0,0 +1,15 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file