From 68408891a9a0141474e1a3ab1008160d59c8a3ec Mon Sep 17 00:00:00 2001 From: Ujjwal Soni Date: Tue, 19 Nov 2024 17:49:18 +0530 Subject: [PATCH] Samples for fts (#31829) --- sdk/cosmosdb/cosmos/CHANGELOG.md | 6 -- sdk/cosmosdb/cosmos/package.json | 3 +- .../cosmos/samples-dev/ContainerManagement.ts | 27 ++++++ .../samples-dev/Query/FullTextSearch.ts | 86 +++++++++++++++++++ .../v4/javascript/ContainerManagement.js | 27 ++++++ .../v4/javascript/Query/FullTextSearch.js | 85 ++++++++++++++++++ .../v4/typescript/src/ContainerManagement.ts | 27 ++++++ .../v4/typescript/src/Query/FullTextSearch.ts | 86 +++++++++++++++++++ 8 files changed, 340 insertions(+), 7 deletions(-) create mode 100644 sdk/cosmosdb/cosmos/samples-dev/Query/FullTextSearch.ts create mode 100644 sdk/cosmosdb/cosmos/samples/v4/javascript/Query/FullTextSearch.js create mode 100644 sdk/cosmosdb/cosmos/samples/v4/typescript/src/Query/FullTextSearch.ts diff --git a/sdk/cosmosdb/cosmos/CHANGELOG.md b/sdk/cosmosdb/cosmos/CHANGELOG.md index f31d42d7fc8..bdb9945c158 100644 --- a/sdk/cosmosdb/cosmos/CHANGELOG.md +++ b/sdk/cosmosdb/cosmos/CHANGELOG.md @@ -6,12 +6,6 @@ - Full Text and Hybrid Search Support: Implemented full text and indexing policies, and added support for full text and hybrid search queries. -### Breaking Changes - -### Bugs Fixed - -### Other Changes - ## 4.1.1 (2024-08-30) ### Bugs Fixed diff --git a/sdk/cosmosdb/cosmos/package.json b/sdk/cosmosdb/cosmos/package.json index c45316d9277..ec0ebce66b0 100644 --- a/sdk/cosmosdb/cosmos/package.json +++ b/sdk/cosmosdb/cosmos/package.json @@ -141,7 +141,8 @@ "ChangeFeedIteratorLatestVersion.ts", "ChangeFeedIteratorAllVersionsAndDeletes.ts", "Diagnostics.ts", - "HierarchicalPartitioning.ts" + "HierarchicalPartitioning.ts", + "FullTextSearch.ts" ], "productName": "Azure Cosmos DB", "productSlugs": [ diff --git a/sdk/cosmosdb/cosmos/samples-dev/ContainerManagement.ts b/sdk/cosmosdb/cosmos/samples-dev/ContainerManagement.ts index 0cfff509c51..775e7812829 100644 --- a/sdk/cosmosdb/cosmos/samples-dev/ContainerManagement.ts +++ b/sdk/cosmosdb/cosmos/samples-dev/ContainerManagement.ts @@ -174,6 +174,33 @@ async function run(): Promise { }; await database.containers.createIfNotExists(containerDefinition); console.log("Container with vector embedding and indexing policies created"); + + logStep("Create container with full text search container policy"); + + // Create a container with full text policy and full text indexes + const indexingPolicyFTS: IndexingPolicy = { + automatic: true, + includedPaths: [{ path: "/*" }], + excludedPaths: [{ path: '/"_etag"/?' }], + fullTextIndexes: [{ path: "/text1" }, { path: "/text2" }], + }; + + const fullTextPolicy = { + defaultLanguage: "en-US", + fullTextPaths: [ + { path: "/text1", language: "1033" }, + { path: "/text2", language: "en-US" }, + ], + }; + + await database.containers.createIfNotExists({ + id: "ContainerWithFTSPolicy", + partitionKey: { paths: ["/id"] }, + fullTextPolicy: fullTextPolicy, + indexingPolicy: indexingPolicyFTS, + }); + console.log("Container with full text search policy created"); + await finish(); } diff --git a/sdk/cosmosdb/cosmos/samples-dev/Query/FullTextSearch.ts b/sdk/cosmosdb/cosmos/samples-dev/Query/FullTextSearch.ts new file mode 100644 index 00000000000..5a917a5380c --- /dev/null +++ b/sdk/cosmosdb/cosmos/samples-dev/Query/FullTextSearch.ts @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * @summary Demonstrates full text search queries. + */ + +import * as dotenv from "dotenv"; +dotenv.config(); + +import { finish, handleError, logSampleHeader } from "./../Shared/handleError"; +import { CosmosClient, IndexingPolicy } from "@azure/cosmos"; +const key = process.env.COSMOS_KEY || ""; +const endpoint = process.env.COSMOS_ENDPOINT || ""; +const databaseId = process.env.COSMOS_DATABASE || ""; +const containerId = process.env.COSMOS_CONTAINER || ""; +logSampleHeader("Full Text Search Queries"); + +// Establish a new instance of the CosmosClient to be used throughout this demo +const client = new CosmosClient({ endpoint, key }); + +async function run(): Promise { + // create a database + const { database } = await client.databases.createIfNotExists({ id: databaseId }); + + // Create a container with full text policy and full text indexes + const indexingPolicy: IndexingPolicy = { + automatic: true, + includedPaths: [{ path: "/*" }], + excludedPaths: [{ path: '/"_etag"/?' }], + fullTextIndexes: [{ path: "/text1" }], + }; + + const fullTextPolicy = { + defaultLanguage: "en-US", + fullTextPaths: [{ path: "/text1", language: "en-US" }], + }; + + const { container } = await database.containers.createIfNotExists({ + id: containerId, + partitionKey: { paths: ["/id"] }, + fullTextPolicy: fullTextPolicy, + indexingPolicy: indexingPolicy, + }); + + const sample_texts: string[] = [ + "Common popular pop music artists include Taylor Swift and The Weekend.", + "The weekend is coming up soon, do you have any plans?", + "Depending on the artist, their music can be very different.", + "Mozart and Beethoven are some of the most recognizable names in classical music.", + "Taylor acts in many movies, and is considered a great artist.", + ]; + + // Create some items to use with full text search + for (let i = 0; i < 10; i++) { + await container.items.create({ id: "full_text_item" + i, text1: "some-text" }); + } + for (let i = 10; i < 15; i++) { + await container.items.create({ + id: "full_text_item" + i, + text1: sample_texts[i - 10], + vector: [1, 2, 3], + }); + } + + // Run full text search queries using full text score ranking + let query = "SELECT TOP 3 c.text1 FROM c ORDER BY RANK FullTextScore(c.text1, ['artist'])"; + let response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + // Run full text search queries with full text contains + query = + "SELECT TOP 3 c.text1 FROM c WHERE FullTextContains(c.text1, 'artist') ORDER BY RANK RRF (FullTextScore(c.text1, ['movies']),FullTextScore(c.text1, ['music']))"; + response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + // Run hybrid search queries using RRF ranking wth vector distances + query = + "SELECT TOP 3 c.text1 FROM c ORDER BY RANK RRF(FullTextScore(c.text1, ['music']), VectorDistance(c.vector, [1, 2, 3]))"; + response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + await finish(); +} + +run().catch(handleError); diff --git a/sdk/cosmosdb/cosmos/samples/v4/javascript/ContainerManagement.js b/sdk/cosmosdb/cosmos/samples/v4/javascript/ContainerManagement.js index 8289634b41b..53ec2c42d36 100644 --- a/sdk/cosmosdb/cosmos/samples/v4/javascript/ContainerManagement.js +++ b/sdk/cosmosdb/cosmos/samples/v4/javascript/ContainerManagement.js @@ -170,6 +170,33 @@ async function run() { }; await database.containers.createIfNotExists(containerDefinition); console.log("Container with vector embedding and indexing policies created"); + + logStep("Create container with full text search container policy"); + + // Create a container with full text policy and full text indexes + const indexingPolicyFTS = { + automatic: true, + includedPaths: [{ path: "/*" }], + excludedPaths: [{ path: '/"_etag"/?' }], + fullTextIndexes: [{ path: "/text1" }, { path: "/text2" }], + }; + + const fullTextPolicy = { + defaultLanguage: "en-US", + fullTextPaths: [ + { path: "/text1", language: "1033" }, + { path: "/text2", language: "en-US" }, + ], + }; + + await database.containers.createIfNotExists({ + id: "ContainerWithFTSPolicy", + partitionKey: { paths: ["/id"] }, + fullTextPolicy: fullTextPolicy, + indexingPolicy: indexingPolicyFTS, + }); + console.log("Container with full text search policy created"); + await finish(); } diff --git a/sdk/cosmosdb/cosmos/samples/v4/javascript/Query/FullTextSearch.js b/sdk/cosmosdb/cosmos/samples/v4/javascript/Query/FullTextSearch.js new file mode 100644 index 00000000000..fe71dc3c2cd --- /dev/null +++ b/sdk/cosmosdb/cosmos/samples/v4/javascript/Query/FullTextSearch.js @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * @summary Demonstrates full text search queries. + */ + +require("dotenv").config(); + +const { finish, handleError, logSampleHeader } = require("./../Shared/handleError"); +const { CosmosClient } = require("@azure/cosmos"); +const key = process.env.COSMOS_KEY || ""; +const endpoint = process.env.COSMOS_ENDPOINT || ""; +const databaseId = process.env.COSMOS_DATABASE || ""; +const containerId = process.env.COSMOS_CONTAINER || ""; +logSampleHeader("Full Text Search Queries"); + +// Establish a new instance of the CosmosClient to be used throughout this demo +const client = new CosmosClient({ endpoint, key }); + +async function run() { + // create a database + const { database } = await client.databases.createIfNotExists({ id: databaseId }); + + // Create a container with full text policy and full text indexes + const indexingPolicy = { + automatic: true, + includedPaths: [{ path: "/*" }], + excludedPaths: [{ path: '/"_etag"/?' }], + fullTextIndexes: [{ path: "/text1" }], + }; + + const fullTextPolicy = { + defaultLanguage: "en-US", + fullTextPaths: [{ path: "/text1", language: "en-US" }], + }; + + const { container } = await database.containers.createIfNotExists({ + id: containerId, + partitionKey: { paths: ["/id"] }, + fullTextPolicy: fullTextPolicy, + indexingPolicy: indexingPolicy, + }); + + const sample_texts = [ + "Common popular pop music artists include Taylor Swift and The Weekend.", + "The weekend is coming up soon, do you have any plans?", + "Depending on the artist, their music can be very different.", + "Mozart and Beethoven are some of the most recognizable names in classical music.", + "Taylor acts in many movies, and is considered a great artist.", + ]; + + // Create some items to use with full text search + for (let i = 0; i < 10; i++) { + await container.items.create({ id: "full_text_item" + i, text1: "some-text" }); + } + for (let i = 10; i < 15; i++) { + await container.items.create({ + id: "full_text_item" + i, + text1: sample_texts[i - 10], + vector: [1, 2, 3], + }); + } + + // Run full text search queries using full text score ranking + let query = "SELECT TOP 3 c.text1 FROM c ORDER BY RANK FullTextScore(c.text1, ['artist'])"; + let response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + // Run full text search queries with full text contains + query = + "SELECT TOP 3 c.text1 FROM c WHERE FullTextContains(c.text1, 'artist') ORDER BY RANK RRF (FullTextScore(c.text1, ['movies']),FullTextScore(c.text1, ['music']))"; + response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + // Run hybrid search queries using RRF ranking wth vector distances + query = + "SELECT TOP 3 c.text1 FROM c ORDER BY RANK RRF(FullTextScore(c.text1, ['music']), VectorDistance(c.vector, [1, 2, 3]))"; + response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + await finish(); +} + +run().catch(handleError); diff --git a/sdk/cosmosdb/cosmos/samples/v4/typescript/src/ContainerManagement.ts b/sdk/cosmosdb/cosmos/samples/v4/typescript/src/ContainerManagement.ts index 0cfff509c51..775e7812829 100644 --- a/sdk/cosmosdb/cosmos/samples/v4/typescript/src/ContainerManagement.ts +++ b/sdk/cosmosdb/cosmos/samples/v4/typescript/src/ContainerManagement.ts @@ -174,6 +174,33 @@ async function run(): Promise { }; await database.containers.createIfNotExists(containerDefinition); console.log("Container with vector embedding and indexing policies created"); + + logStep("Create container with full text search container policy"); + + // Create a container with full text policy and full text indexes + const indexingPolicyFTS: IndexingPolicy = { + automatic: true, + includedPaths: [{ path: "/*" }], + excludedPaths: [{ path: '/"_etag"/?' }], + fullTextIndexes: [{ path: "/text1" }, { path: "/text2" }], + }; + + const fullTextPolicy = { + defaultLanguage: "en-US", + fullTextPaths: [ + { path: "/text1", language: "1033" }, + { path: "/text2", language: "en-US" }, + ], + }; + + await database.containers.createIfNotExists({ + id: "ContainerWithFTSPolicy", + partitionKey: { paths: ["/id"] }, + fullTextPolicy: fullTextPolicy, + indexingPolicy: indexingPolicyFTS, + }); + console.log("Container with full text search policy created"); + await finish(); } diff --git a/sdk/cosmosdb/cosmos/samples/v4/typescript/src/Query/FullTextSearch.ts b/sdk/cosmosdb/cosmos/samples/v4/typescript/src/Query/FullTextSearch.ts new file mode 100644 index 00000000000..5a917a5380c --- /dev/null +++ b/sdk/cosmosdb/cosmos/samples/v4/typescript/src/Query/FullTextSearch.ts @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * @summary Demonstrates full text search queries. + */ + +import * as dotenv from "dotenv"; +dotenv.config(); + +import { finish, handleError, logSampleHeader } from "./../Shared/handleError"; +import { CosmosClient, IndexingPolicy } from "@azure/cosmos"; +const key = process.env.COSMOS_KEY || ""; +const endpoint = process.env.COSMOS_ENDPOINT || ""; +const databaseId = process.env.COSMOS_DATABASE || ""; +const containerId = process.env.COSMOS_CONTAINER || ""; +logSampleHeader("Full Text Search Queries"); + +// Establish a new instance of the CosmosClient to be used throughout this demo +const client = new CosmosClient({ endpoint, key }); + +async function run(): Promise { + // create a database + const { database } = await client.databases.createIfNotExists({ id: databaseId }); + + // Create a container with full text policy and full text indexes + const indexingPolicy: IndexingPolicy = { + automatic: true, + includedPaths: [{ path: "/*" }], + excludedPaths: [{ path: '/"_etag"/?' }], + fullTextIndexes: [{ path: "/text1" }], + }; + + const fullTextPolicy = { + defaultLanguage: "en-US", + fullTextPaths: [{ path: "/text1", language: "en-US" }], + }; + + const { container } = await database.containers.createIfNotExists({ + id: containerId, + partitionKey: { paths: ["/id"] }, + fullTextPolicy: fullTextPolicy, + indexingPolicy: indexingPolicy, + }); + + const sample_texts: string[] = [ + "Common popular pop music artists include Taylor Swift and The Weekend.", + "The weekend is coming up soon, do you have any plans?", + "Depending on the artist, their music can be very different.", + "Mozart and Beethoven are some of the most recognizable names in classical music.", + "Taylor acts in many movies, and is considered a great artist.", + ]; + + // Create some items to use with full text search + for (let i = 0; i < 10; i++) { + await container.items.create({ id: "full_text_item" + i, text1: "some-text" }); + } + for (let i = 10; i < 15; i++) { + await container.items.create({ + id: "full_text_item" + i, + text1: sample_texts[i - 10], + vector: [1, 2, 3], + }); + } + + // Run full text search queries using full text score ranking + let query = "SELECT TOP 3 c.text1 FROM c ORDER BY RANK FullTextScore(c.text1, ['artist'])"; + let response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + // Run full text search queries with full text contains + query = + "SELECT TOP 3 c.text1 FROM c WHERE FullTextContains(c.text1, 'artist') ORDER BY RANK RRF (FullTextScore(c.text1, ['movies']),FullTextScore(c.text1, ['music']))"; + response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + // Run hybrid search queries using RRF ranking wth vector distances + query = + "SELECT TOP 3 c.text1 FROM c ORDER BY RANK RRF(FullTextScore(c.text1, ['music']), VectorDistance(c.vector, [1, 2, 3]))"; + response = await container.items.query(query, { forceQueryPlan: true }).fetchAll(); + console.log("Response: ", response.resources); + + await finish(); +} + +run().catch(handleError);