Azure
/
azure-databricks-client
зеркало из https://github.com/Azure/azure-databricks-client.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

feat: list operations now support AsyncPageable if next_page_token is returned (#196) 

* feat: jobs/list and jobs/runs/list now returns AsyncPageable

* More AsyncPageable support

* fixing style issue
This commit is contained in:
Jason Wang 2024-06-19 23:01:48 -07:00 коммит произвёл GitHub
Родитель d5c1b2039c
Коммит 5be4182381
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
19 изменённых файлов: 427 добавлений и 24 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj
Просмотреть файл

@ -10,6 +10,7 @@
<ItemGroup>
<PackageReference Include="Azure.Identity" Version="1.12.0" />
<PackageReference Include="Polly" Version="8.4.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>

148
csharp/Microsoft.Azure.Databricks.Client.Test/JobApiClientTest.cs
Просмотреть файл

@ -598,10 +598,11 @@ public class JobApiClientTest : ApiClientTest
}
[TestMethod]
[Obsolete]
public async Task TestList()
{
var apiUri = new Uri(JobsApiUri, "list");
var requestUri = new Uri(apiUri, "?limit=20&offset=0&expand_tasks=false");
var requestUri = new Uri(apiUri, "?limit=20&expand_tasks=false&offset=0");
var expectedResponse = new JobList
{
@ -644,6 +645,78 @@ public class JobApiClientTest : ApiClientTest
);
}
[TestMethod]
public async Task TestListPageable()
{
var apiUri = new Uri(JobsApiUri, "list");
var expectedRequestUrl1 = new Uri(apiUri, "?limit=2&expand_tasks=false");
var expectedRequestUrl2 = new Uri(apiUri, "?limit=2&expand_tasks=false&page_token=second");
var expectedRequestUrl3 = new Uri(apiUri, "?limit=2&expand_tasks=false&page_token=third");
const string response1 = @"
{
""jobs"":[{""job_id"": 1}, {""job_id"": 2}],
""has_more"": true,
""next_page_token"": ""second"",
""prev_page_token"": """"
}
";
const string response2 = @"
{
""jobs"":[{""job_id"": 3}, {""job_id"": 4}],
""has_more"": true,
""next_page_token"": ""third"",
""prev_page_token"": ""first""
}
";
const string response3 = @"
{
""jobs"":[{""job_id"": 5}, {""job_id"": 6}],
""has_more"": false,
""next_page_token"": """",
""prev_page_token"": ""second""
}
";
var handler = CreateMockHandler();
handler
.SetupRequest(HttpMethod.Get, expectedRequestUrl1)
.ReturnsResponse(HttpStatusCode.OK, response1, "application/json")
.Verifiable();
handler.SetupRequest(HttpMethod.Get, expectedRequestUrl2)
.ReturnsResponse(HttpStatusCode.OK, response2, "application/json")
.Verifiable();
handler.SetupRequest(HttpMethod.Get, expectedRequestUrl3)
.ReturnsResponse(HttpStatusCode.OK, response3, "application/json")
.Verifiable();
var hc = handler.CreateClient();
hc.BaseAddress = BaseApiUri;
using var client = new JobsApiClient(hc);
var jobs = client.ListPageable(pageSize: 2);
Assert.AreEqual(6, await jobs.CountAsync());
handler.VerifyRequest(
HttpMethod.Get,
expectedRequestUrl1,
Times.Once()
);
handler.VerifyRequest(
HttpMethod.Get,
expectedRequestUrl2,
Times.Once()
);
handler.VerifyRequest(
HttpMethod.Get,
expectedRequestUrl3,
Times.Once()
);
}
[TestMethod]
public async Task TestRunNow()
{
@ -1073,6 +1146,79 @@ public class JobApiClientTest : ApiClientTest
);
}
[TestMethod]
public async Task TestRunsListPageable()
{
var apiUri = new Uri(JobsApiUri, "runs/list");
var expectedRequestUrl1 = new Uri(apiUri, "?limit=2&job_id=11223344");
var expectedRequestUrl2 = new Uri(apiUri, "?limit=2&job_id=11223344&page_token=second");
var expectedRequestUrl3 = new Uri(apiUri, "?limit=2&job_id=11223344&page_token=third");
const string response1 = @"
{
""runs"":[{""job_id"": 11223344,""run_id"": 1}, {""job_id"": 11223344,""run_id"": 2}],
""has_more"": true,
""next_page_token"": ""second"",
""prev_page_token"": """"
}
";
const string response2 = @"
{
""runs"":[{""job_id"": 11223344,""run_id"": 3}, {""job_id"": 11223344,""run_id"": 4}],
""has_more"": true,
""next_page_token"": ""third"",
""prev_page_token"": ""first""
}
";
const string response3 = @"
{
""runs"":[{""job_id"": 11223344,""run_id"": 5}, {""job_id"": 11223344,""run_id"": 6}],
""has_more"": false,
""next_page_token"": """",
""prev_page_token"": ""second""
}
";
var handler = CreateMockHandler();
handler
.SetupRequest(HttpMethod.Get, expectedRequestUrl1)
.ReturnsResponse(HttpStatusCode.OK, response1, "application/json")
.Verifiable();
handler.SetupRequest(HttpMethod.Get, expectedRequestUrl2)
.ReturnsResponse(HttpStatusCode.OK, response2, "application/json")
.Verifiable();
handler.SetupRequest(HttpMethod.Get, expectedRequestUrl3)
.ReturnsResponse(HttpStatusCode.OK, response3, "application/json")
.Verifiable();
var hc = handler.CreateClient();
hc.BaseAddress = BaseApiUri;
using var client = new JobsApiClient(hc);
var runs = client.RunsListPageable(jobId: 11223344, pageSize: 2);
Assert.AreEqual(6, await runs.CountAsync());
handler.VerifyRequest(
HttpMethod.Get,
expectedRequestUrl1,
Times.Once()
);
handler.VerifyRequest(
HttpMethod.Get,
expectedRequestUrl2,
Times.Once()
);
handler.VerifyRequest(
HttpMethod.Get,
expectedRequestUrl3,
Times.Once()
);
}
[TestMethod]
public async Task TestRunsExport()
{

3
csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj
Просмотреть файл

@ -16,7 +16,8 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Polly" Version="8.4.0" />
<PackageReference Include="Polly" Version="8.4.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>

16
csharp/Microsoft.Azure.Databricks.Client/ClusterPoliciesApiClient.cs
Просмотреть файл

@ -2,6 +2,7 @@
// Licensed under the MIT License.
using Microsoft.Azure.Databricks.Client.Models;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
@ -96,11 +97,7 @@ public class ClusterPoliciesApiClient : ApiClient, IClusterPoliciesApi
public async Task<(IEnumerable<PolicyFamily>, string)> ListPolicyFamily(int maxResults = 20, string pageToken = default, CancellationToken cancellationToken = default)
{
var requestUri = $"{ApiVersion}/policy-families?max_results={maxResults}";
if (!string.IsNullOrEmpty(pageToken))
{
requestUri += $"&page_token={pageToken}";
}
requestUri += string.IsNullOrEmpty(pageToken) ? string.Empty : $"&page_token={pageToken}";
var response = await HttpGet<JsonObject>(this.HttpClient, requestUri, cancellationToken).ConfigureAwait(false);
@ -112,4 +109,13 @@ public class ClusterPoliciesApiClient : ApiClient, IClusterPoliciesApi
return (families, nextPageToken);
}
public global::Azure.AsyncPageable<PolicyFamily> ListPolicyFamilyPageable(int pageSize, CancellationToken cancellationToken = default)
{
return new AsyncPageable<PolicyFamily>(async (pageToken) =>
{
var (policyFamilyList, nextPageToken) = await ListPolicyFamily(pageSize, pageToken, cancellationToken).ConfigureAwait(false);
return (policyFamilyList.ToList(), !string.IsNullOrEmpty(nextPageToken), nextPageToken);
});
}
}

2
csharp/Microsoft.Azure.Databricks.Client/IClusterPoliciesApi.cs
Просмотреть файл

@ -53,4 +53,6 @@ public interface IClusterPoliciesApi : IDisposable
Task<PolicyFamily> GetPolicyFamily(string id, CancellationToken cancellationToken = default);
Task<(IEnumerable<PolicyFamily>, string)> ListPolicyFamily(int maxResults, string pageToken = default, CancellationToken cancellationToken = default);
global::Azure.AsyncPageable<PolicyFamily> ListPolicyFamilyPageable(int pageSize, CancellationToken cancellationToken = default);
}

13
csharp/Microsoft.Azure.Databricks.Client/IJobsApi.cs
Просмотреть файл

@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using Azure;
using Microsoft.Azure.Databricks.Client.Models;
using System;
using System.Collections.Generic;
@ -24,9 +25,16 @@ public interface IJobsApi : IDisposable
/// <param name="offset">The offset of the first job to return, relative to the most recently created job.</param>
/// <param name="name">A filter on the list based on the exact (case insensitive) job name.</param>
/// <param name="expandTasks">Whether to include task and cluster details in the response.</param>
[Obsolete("The offset parameter is deprecated. Use method with pageToken to iterate through the pages.")]
Task<JobList> List(int limit = 20, int offset = 0, string name = default, bool expandTasks = false,
CancellationToken cancellationToken = default);
/// <summary>
/// List all jobs.
/// </summary>
global::Azure.AsyncPageable<Job> ListPageable(int pageSize = 20, string name = default, bool expandTasks = false,
CancellationToken cancellationToken = default);
/// <summary>
/// Deletes the job and sends an email to the addresses specified in JobSettings.email_notifications. No action will occur if the job has already been removed. After the job is removed, neither its details or its run history will be visible via the Jobs UI or API. The job is guaranteed to be removed upon completion of this request. However, runs that were active before the receipt of this request may still be active. They will be terminated asynchronously.
/// </summary>
@ -133,6 +141,11 @@ public interface IJobsApi : IDisposable
bool expandTasks = default, DateTimeOffset? startTimeFrom = default,
DateTimeOffset? startTimeTo = default, CancellationToken cancellationToken = default);
global::Azure.AsyncPageable<Run> RunsListPageable(long? jobId = default, int pageSize = 25,
bool activeOnly = default, bool completedOnly = default, RunType? runType = default,
bool expandTasks = default, DateTimeOffset? startTimeFrom = default,
DateTimeOffset? startTimeTo = default, CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves the metadata of a run.
/// </summary>

25
csharp/Microsoft.Azure.Databricks.Client/IPipelinesApi.cs
Просмотреть файл

@ -1,6 +1,5 @@
using Microsoft.Azure.Databricks.Client.Models;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
@ -14,6 +13,8 @@ public interface IPipelinesApi : IDisposable
/// </summary>
Task<PipelinesList> List(int maxResults = 25, string pageToken = default, CancellationToken cancellationToken = default);
global::Azure.AsyncPageable<Pipeline> ListPageable(int pageSize = 25, CancellationToken cancellationToken = default);
/// <summary>
/// Creates a new data processing pipeline based on the requested configuration.
/// If successful, this method returns the ID of the new pipeline.
@ -70,6 +71,12 @@ public interface IPipelinesApi : IDisposable
string untilUpdateId = null,
CancellationToken cancellationToken = default);
global::Azure.AsyncPageable<PipelineUpdate> ListUpdatesPageable(
string pipelineId,
int pageSize = 25,
string untilUpdateId = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Starts or queues a pipeline update.
/// </summary>
@ -87,8 +94,18 @@ public interface IPipelinesApi : IDisposable
Task<PipelineEventsList> ListEvents(
string pipelineId,
int maxResults = 25,
string orderBy = null,
string filter = null,
string pageToken = null,
string orderBy = default,
string filter = default,
string pageToken = default,
CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves events for a pipeline.
/// </summary>
global::Azure.AsyncPageable<PipelineEvent> ListEventsPageable(
string pipelineId,
int pageSize = 25,
string orderBy = default,
string filter = default,
CancellationToken cancellationToken = default);
}

5
csharp/Microsoft.Azure.Databricks.Client/IReposApi.cs
Просмотреть файл

@ -18,6 +18,11 @@ public interface IReposApi : IDisposable
/// <param name="pageToken">Token used to get the next page of results. If not specified, returns the first page of results as well as a next page token if there are more results.</param>
Task<(IEnumerable<Repo>, string)> List(string pathPrefix = default, string pageToken = default, CancellationToken cancellationToken = default);
/// <summary>
/// Returns repos that the calling user has Manage permissions on. Results are paginated with each page containing twenty repos.
/// </summary>
/// <param name="pathPrefix">Filters repos that have paths starting with the given path prefix.</param>
global::Azure.AsyncPageable<Repo> ListPageable(string pathPrefix = default, CancellationToken cancellationToken = default);
/// <summary>
/// Creates a repo in the workspace and links it to the remote Git repo specified. Note that repos created programmatically must be linked to a remote Git repo, unlike repos created in the browser.

59
csharp/Microsoft.Azure.Databricks.Client/JobsApiClient.cs
Просмотреть файл

@ -13,6 +13,7 @@ using System.Threading;
using System.Threading.Tasks;
namespace Microsoft.Azure.Databricks.Client;
public class JobsApiClient : ApiClient, IJobsApi
{
protected override string ApiVersion => "2.1";
@ -39,6 +40,20 @@ public class JobsApiClient : ApiClient, IJobsApi
return jobIdentifier["job_id"]!.GetValue<long>();
}
private string BuildJobsListUrl(int limit, string name, bool expandTasks)
{
StringBuilder url = new($"{ApiVersion}/jobs/list?limit={limit}");
if (name is not null)
{
url.Append($"&name={name}");
}
url.Append($"&expand_tasks={expandTasks.ToString().ToLowerInvariant()}");
return url.ToString();
}
[Obsolete("The offset parameter is deprecated. Use method with pageToken to iterate through the pages.")]
public async Task<JobList> List(int limit = 20, int offset = 0, string name = default, bool expandTasks = false,
CancellationToken cancellationToken = default)
{
@ -52,14 +67,10 @@ public class JobsApiClient : ApiClient, IJobsApi
throw new ArgumentOutOfRangeException(nameof(offset), "offset must be greater than or equal to 0");
}
var requestUri = $"{ApiVersion}/jobs/list?limit={limit}&offset={offset}&expand_tasks={expandTasks.ToString().ToLowerInvariant()}";
var url = BuildJobsListUrl(limit, name, expandTasks);
url += $"&offset={offset}";
if (name is not null)
{
requestUri += $"&name={name}";
}
var response = await HttpGet<JsonObject>(this.HttpClient, requestUri, cancellationToken)
var response = await HttpGet<JsonObject>(this.HttpClient, url, cancellationToken)
.ConfigureAwait(false);
response.TryGetPropertyValue("jobs", out var jobsNode);
@ -68,6 +79,23 @@ public class JobsApiClient : ApiClient, IJobsApi
return new JobList { Jobs = jobs, HasMore = hasMore };
}
public global::Azure.AsyncPageable<Job> ListPageable(int pageSize = 20, string name = null, bool expandTasks = false, CancellationToken cancellationToken = default)
{
if (pageSize < 1 || pageSize > 25)
{
throw new ArgumentOutOfRangeException(nameof(pageSize), "pageSize must be between 1 and 25");
}
return new AsyncPageable<Job>(async (nextPageToken) =>
{
var url = BuildJobsListUrl(pageSize, name, expandTasks);
url += string.IsNullOrEmpty(nextPageToken) ? string.Empty : $"&page_token={nextPageToken}";
var jobList = await HttpGet<JobList>(this.HttpClient, url, cancellationToken).ConfigureAwait(false);
return (jobList.Jobs.ToList(), jobList.HasMore, jobList.NextPageToken);
});
}
public async Task Delete(long jobId, CancellationToken cancellationToken = default)
{
await HttpPost(this.HttpClient, $"{ApiVersion}/jobs/delete", new { job_id = jobId }, cancellationToken).ConfigureAwait(false);
@ -193,9 +221,26 @@ public class JobsApiClient : ApiClient, IJobsApi
{
string url = BuildRunsListUrl(jobId, limit, activeOnly, completedOnly, runType, expandTasks, startTimeFrom, startTimeTo);
url += string.IsNullOrEmpty(pageToken) ? string.Empty : $"&page_token={pageToken}";
await Console.Out.WriteLineAsync("Request: " + url);
return await HttpGet<RunList>(this.HttpClient, url, cancellationToken).ConfigureAwait(false);
}
public global::Azure.AsyncPageable<Run> RunsListPageable(long? jobId = null, int pageSize = 25,
bool activeOnly = false, bool completedOnly = false, RunType? runType = null, bool expandTasks = false,
DateTimeOffset? startTimeFrom = null, DateTimeOffset? startTimeTo = null, CancellationToken cancellationToken = default)
{
return new AsyncPageable<Run>(async (nextPageToken) =>
{
var response = await RunsList(nextPageToken, jobId, pageSize, activeOnly, completedOnly, runType, expandTasks,
startTimeFrom, startTimeTo, cancellationToken).ConfigureAwait(false);
return (response.Runs.ToList(), response.HasMore, response.NextPageToken);
});
}
public async Task<(Run, RepairHistory)> RunsGet(long runId, bool includeHistory = default,
CancellationToken cancellationToken = default)
{

17
csharp/Microsoft.Azure.Databricks.Client/Models/JobList.cs
Просмотреть файл

@ -8,6 +8,11 @@ namespace Microsoft.Azure.Databricks.Client.Models;
public record JobList
{
public JobList()
{
this.Jobs = new List<Job>();
}
/// <summary>
/// The list of jobs.
/// </summary>
@ -20,4 +25,16 @@ public record JobList
[JsonPropertyName("has_more")]
[JsonIgnore(Condition = JsonIgnoreCondition.Never)]
public bool HasMore { get; set; }
/// <summary>
/// A token that can be used to list the next page of runs.
/// </summary>
[JsonPropertyName("next_page_token")]
public string NextPageToken { get; set; }
/// <summary>
/// A token that can be used to list the previous page of runs.
/// </summary>
[JsonPropertyName("prev_page_token")]
public string PrevPageToken { get; set; }
}

3
csharp/Microsoft.Azure.Databricks.Client/Models/PipelineEvent.cs
Просмотреть файл

@ -11,6 +11,9 @@ public record PipelineEventsList
[JsonPropertyName("next_page_token")]
public string NextPageToken { get; set; }
[JsonIgnore]
public bool HasMore => !string.IsNullOrEmpty(NextPageToken);
/// <summary>
/// If present, a token to fetch the previous page of events.
/// </summary>

3
csharp/Microsoft.Azure.Databricks.Client/Models/PipelineUpdatesList.cs
Просмотреть файл

@ -17,6 +17,9 @@ public record PipelineUpdatesList
[JsonPropertyName("next_page_token")]
public string NextPageToken { get; set; }
[JsonIgnore]
public bool HasMore => !string.IsNullOrEmpty(this.NextPageToken);
/// <summary>
/// If present, then this token can be used in a subsequent request to fetch the previous page.
/// </summary>

3
csharp/Microsoft.Azure.Databricks.Client/Models/PipelinesList.cs
Просмотреть файл

@ -16,4 +16,7 @@ public class PipelinesList
/// </summary>
[JsonPropertyName("next_page_token")]
public string NextPageToken { get; set; }
[JsonIgnore]
public bool HasMore => !string.IsNullOrEmpty(this.NextPageToken);
}

7
csharp/Microsoft.Azure.Databricks.Client/Models/RunList.cs
Просмотреть файл

@ -8,6 +8,11 @@ namespace Microsoft.Azure.Databricks.Client.Models;
public record RunList
{
public RunList()
{
this.Runs = new List<Run>();
}
/// <summary>
/// A list of runs, from most recently started to least.
/// </summary>
@ -31,4 +36,4 @@ public record RunList
/// </summary>
[JsonPropertyName("prev_page_token")]
public string PrevPageToken { get; set; }
}
}

31
csharp/Microsoft.Azure.Databricks.Client/Pageable.cs Normal file
Просмотреть файл

@ -0,0 +1,31 @@
using Azure;
using System.Collections.Generic;
using System;
using System.Threading.Tasks;
namespace Microsoft.Azure.Databricks.Client
{
internal class AsyncPageable<T> : global::Azure.AsyncPageable<T>
{
private readonly Func<string, Task<(List<T>, bool, string)>> _getNextPage;
public AsyncPageable(Func<string, Task<(List<T>, bool, string)>> getNextPage)
{
this._getNextPage = getNextPage;
}
public override async IAsyncEnumerable<Page<T>> AsPages(string continuationToken = null, int? pageSizeHint = null)
{
var nextPageToken = continuationToken;
var hasNextPage = true;
do
{
var response = await _getNextPage(nextPageToken);
hasNextPage = response.Item2;
nextPageToken = response.Item3;
yield return Page<T>.FromValues(response.Item1, nextPageToken, null);
} while (hasNextPage);
}
}
}

48
csharp/Microsoft.Azure.Databricks.Client/PipelinesApiClient.cs
Просмотреть файл

@ -35,6 +35,17 @@ public class PipelinesApiClient : ApiClient, IPipelinesApi
return await HttpGet<PipelinesList>(this.HttpClient, requestUri, cancellationToken).ConfigureAwait(false);
}
public global::Azure.AsyncPageable<Pipeline> ListPageable(int pageSize = 25, CancellationToken cancellationToken = default)
{
return new AsyncPageable<Pipeline>(
async (string pageToken) =>
{
var response = await List(pageSize, pageToken, cancellationToken).ConfigureAwait(false);
return (response.Pipelines.ToList(), response.HasMore, response.NextPageToken);
}
);
}
public async Task<(string, PipelineSpecification)> Create(
PipelineSpecification pipelineSpecification,
bool dryRun = true,
@ -129,6 +140,21 @@ public class PipelinesApiClient : ApiClient, IPipelinesApi
return await HttpGet<PipelineUpdatesList>(this.HttpClient, requestUri, cancellationToken).ConfigureAwait(false);
}
public global::Azure.AsyncPageable<PipelineUpdate> ListUpdatesPageable(
string pipelineId,
int pageSize = 25,
string untilUpdateId = null,
CancellationToken cancellationToken = default)
{
return new AsyncPageable<PipelineUpdate>(
async (string pageToken) =>
{
var response = await ListUpdates(pipelineId, pageSize, pageToken, untilUpdateId, cancellationToken).ConfigureAwait(false);
return (response.Updates.ToList(), response.HasMore, response.NextPageToken);
}
);
}
public async Task<string> Start(
string pipelineId,
bool fullRefresh = false,
@ -166,9 +192,9 @@ public class PipelinesApiClient : ApiClient, IPipelinesApi
public async Task<PipelineEventsList> ListEvents(
string pipelineId,
int maxResults = 25,
string orderBy = null,
string filter = null,
string pageToken = null,
string orderBy = default,
string filter = default,
string pageToken = default,
CancellationToken cancellationToken = default)
{
var requestUriSb = new StringBuilder($"{ApiVersion}/pipelines/{pipelineId}/events?max_results={maxResults}");
@ -192,4 +218,20 @@ public class PipelinesApiClient : ApiClient, IPipelinesApi
return await HttpGet<PipelineEventsList>(this.HttpClient, requestUri, cancellationToken).ConfigureAwait(false);
}
public global::Azure.AsyncPageable<PipelineEvent> ListEventsPageable(
string pipelineId,
int pageSize = 25,
string orderBy = default,
string filter = default,
CancellationToken cancellationToken = default)
{
return new AsyncPageable<PipelineEvent>(
async (string pageToken) =>
{
var response = await ListEvents(pipelineId, pageSize, orderBy, filter, pageToken, cancellationToken).ConfigureAwait(false);
return (response.Events.ToList(), response.HasMore, response.NextPageToken);
}
);
}
}

9
csharp/Microsoft.Azure.Databricks.Client/ReposApiClient.cs
Просмотреть файл

@ -64,6 +64,15 @@ public class ReposApiClient : ApiClient, IReposApi
return (repos, nextPageToken);
}
public global::Azure.AsyncPageable<Repo> ListPageable(string pathPrefix = null, CancellationToken cancellationToken = default)
{
return new AsyncPageable<Repo>(async (pageToken) =>
{
var (repos, nextPageToken) = await List(pathPrefix, pageToken, cancellationToken).ConfigureAwait(false);
return (repos.ToList(), !string.IsNullOrEmpty(nextPageToken), nextPageToken);
});
}
public async Task Update(long repoId, string branch = null, string tag = null, RepoSparseCheckout sparseCheckout = null, CancellationToken cancellationToken = default)
{
var requestUri = $"{_apiBaseUrl}/{repoId}";

15
csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/Interfaces/ITablesApi.cs
Просмотреть файл

@ -11,12 +11,19 @@ public interface ITablesApi : IDisposable
Task<(IEnumerable<TableSummary>, string)> ListSummaries(
string catalogName,
int maxResults = 10000,
int? maxResults = 10000,
string schemaNamePattern = default,
string tableNamePattern = default,
string pageToken = default,
CancellationToken cancellationToken = default);
global::Azure.AsyncPageable<TableSummary> ListSummariesPageable(
string catalogName,
int? pageSize = default,
string schemaNamePattern = default,
string tableNamePattern = default,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets an array of all tables for the current metastore under the parent catalog and schema.
/// </summary>
@ -28,6 +35,12 @@ public interface ITablesApi : IDisposable
bool? includeDeltaMetadata = default,
CancellationToken cancellationToken = default);
global::Azure.AsyncPageable<Table> ListPageable(
string catalogName,
string schemaName,
int? pageSize = default,
bool? includeDeltaMetadata = default);
/// <summary>
/// Gets a table from the metastore for a specific catalog and schema.
/// </summary>

43
csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/TablesApiClient.cs
Просмотреть файл

@ -18,7 +18,7 @@ public class TablesApiClient : ApiClient, ITablesApi
public async Task<(IEnumerable<TableSummary>, string)> ListSummaries(
string catalogName,
int maxResults = 10000,
int? maxResults = 10000,
string schemaNamePattern = default,
string tableNamePattern = default,
string pageToken = default,
@ -55,6 +55,27 @@ public class TablesApiClient : ApiClient, ITablesApi
return (tables, nextPageToken);
}
public global::Azure.AsyncPageable<TableSummary> ListSummariesPageable(
string catalogName,
int? pageSize = default,
string schemaNamePattern = default,
string tableNamePattern = default,
CancellationToken cancellationToken = default)
{
return new AsyncPageable<TableSummary>(async (pageToken) =>
{
var (tableSummaries, nextPageToken) = await ListSummaries(
catalogName,
pageSize,
schemaNamePattern,
tableNamePattern,
pageToken,
cancellationToken).ConfigureAwait(false);
return (tableSummaries.ToList(), !string.IsNullOrEmpty(nextPageToken), nextPageToken);
});
}
public async Task<(IEnumerable<Table>, string)> List(
string catalogName,
string schemaName,
@ -94,6 +115,26 @@ public class TablesApiClient : ApiClient, ITablesApi
return (tables, nextPageToken);
}
public global::Azure.AsyncPageable<Table> ListPageable(
string catalogName,
string schemaName,
int? pageSize = default,
bool? includeDeltaMetadata = default)
{
return new AsyncPageable<Table>(async (pageToken) =>
{
var (tables, nextPageToken) = await List(
catalogName,
schemaName,
pageSize,
pageToken,
includeDeltaMetadata,
CancellationToken.None).ConfigureAwait(false);
return (tables.ToList(), !string.IsNullOrEmpty(nextPageToken), nextPageToken);
});
}
public async Task<Table> Get(
string fullTableName,
bool? includeDeltaMetadata = default,