From 268757f51ec983e355a8826b0bdf30190e1f308a Mon Sep 17 00:00:00 2001 From: Connor Peet Date: Sat, 21 Oct 2017 09:08:25 -0700 Subject: [PATCH] fix(codegen): call stack errors when regenerating proto fildes --- bin/generate-methods.js | 11 ++-- proto/rpc.proto | 70 ++++++++++++++++++++++--- src/builder.ts | 1 + src/rpc.ts | 111 ++++++++++++++++++++++++++++++++-------- 4 files changed, 162 insertions(+), 31 deletions(-) diff --git a/bin/generate-methods.js b/bin/generate-methods.js index cf0f8bd..59f8ae5 100644 --- a/bin/generate-methods.js +++ b/bin/generate-methods.js @@ -225,14 +225,19 @@ function walk(ast, iterator, path = []) { }); } -function markResponsesFor(message) { +function markResponsesFor(message, seen = []) { + const name = message.node.name; + if (seen.includes(name)) { + return; + } + message.response = true; _(message.node.fields) .values() .map(f => messages.find(f.type)) .filter(Boolean) - .forEach(markResponsesFor); + .forEach(m => markResponsesFor(m, seen.concat(name))); } function prepareForGeneration(ast) { @@ -256,7 +261,7 @@ function prepareForGeneration(ast) { .values() .map(m => messages.find(m.responseType)) .filter(Boolean) - .forEach(markResponsesFor); + .forEach(m => markResponsesFor(m)); } }); } diff --git a/proto/rpc.proto b/proto/rpc.proto index e69776c..a8652f4 100644 --- a/proto/rpc.proto +++ b/proto/rpc.proto @@ -95,7 +95,13 @@ service Lease { body: "*" }; } - // TODO(xiangli) List all existing Leases? + // LeaseLeases lists all existing leases. + rpc LeaseLeases(LeaseLeasesRequest) returns (LeaseLeasesResponse) { + option (google.api.http) = { + post: "/v3alpha/kv/lease/leases" + body: "*" + }; + } } service Cluster { // MemberAdd adds a member into the cluster. @@ -149,7 +155,7 @@ service Maintenance { body: "*" }; } - // Hash returns the hash of the local KV state for consistency checking purpose. + // Hash computes the hash of the KV's backend. // This is designed for testing; do not use this in production when there // are ongoing transactions. rpc Hash(HashRequest) returns (HashResponse) { @@ -158,6 +164,13 @@ service Maintenance { body: "*" }; } + // HashKV computes the hash of all MVCC keys up to a given revision. + rpc HashKV(HashKVRequest) returns (HashKVResponse) { + option (google.api.http) = { + post: "/v3alpha/maintenance/hash" + body: "*" + }; + } // Snapshot sends a snapshot of the entire backend from a member over a stream to a client. rpc Snapshot(SnapshotRequest) returns (stream SnapshotResponse) { option (google.api.http) = { @@ -165,6 +178,13 @@ service Maintenance { body: "*" }; } + // MoveLeader requests current leader node to transfer its leadership to transferee. + rpc MoveLeader(MoveLeaderRequest) returns (MoveLeaderResponse) { + option (google.api.http) = { + post: "/v3alpha/maintenance/transfer-leadership" + body: "*" + }; + } } service Auth { // AuthEnable enables authentication. @@ -331,7 +351,6 @@ message RangeRequest { bool serializable = 7; // keys_only when set returns only the keys and not the values. bool keys_only = 8; - // count_only when set returns only the count of the keys in the range. bool count_only = 9; // min_mod_revision is the lower bound for returned key mod revisions; all keys with @@ -406,6 +425,7 @@ message RequestOp { RangeRequest request_range = 1; PutRequest request_put = 2; DeleteRangeRequest request_delete_range = 3; + TxnRequest request_txn = 4; } } message ResponseOp { @@ -414,6 +434,7 @@ message ResponseOp { RangeResponse response_range = 1; PutResponse response_put = 2; DeleteRangeResponse response_delete_range = 3; + TxnResponse response_txn = 4; } } message Compare { @@ -428,6 +449,7 @@ message Compare { Create = 1; Mod = 2; Value= 3; + Lease = 4; } // result is logical comparison operation for this comparison. CompareResult result = 1; @@ -444,7 +466,14 @@ message Compare { int64 mod_revision = 6; // value is the value of the given key, in bytes. bytes value = 7; + // lease is the lease id of the given key. + int64 lease = 8; + // leave room for more target_union field tags, jump to 64 } + // range_end compares the given target to all keys in the range [key, range_end). + // See RangeRequest for more details on key ranges. + bytes range_end = 64; + // TODO: fill out with most of the rest of RangeRequest fields when needed. } // From google paxosdb paper: // Our implementation hinges around a powerful primitive which we call MultiOp. All other database @@ -484,7 +513,7 @@ message TxnResponse { // CompactionRequest compacts the key-value store up to a given revision. All superseded keys // with a revision less than the compaction revision will be removed. message CompactionRequest { - // revision is the key-value store revision for the compaction operation. + // revision is the key-value store revision for the compaction operation. int64 revision = 1; // physical is set so the RPC will wait until the compaction is physically // applied to the local database such that compacted entries are totally @@ -496,9 +525,20 @@ message CompactionResponse { } message HashRequest { } +message HashKVRequest { + // revision is the key-value store revision for the hash operation. + int64 revision = 1; +} +message HashKVResponse { + ResponseHeader header = 1; + // hash is the hash value computed from the responding member's MVCC keys up to a given revision. + uint32 hash = 2; + // compact_revision is the compacted revision of key-value store when hash begins. + int64 compact_revision = 3; +} message HashResponse { ResponseHeader header = 1; - // hash is the hash value computed from the responding member's key-value store. + // hash is the hash value computed from the responding member's KV's backend. uint32 hash = 2; } message SnapshotRequest { @@ -567,7 +607,7 @@ message WatchResponse { // at a compacted index. // // This happens when creating a watcher at a compacted revision or the watcher cannot - // catch up with the progress of the key-value store. + // catch up with the progress of the key-value store. // // The client should treat the watcher as canceled and should not try to create any // watcher with the same start_revision again. @@ -625,6 +665,16 @@ message LeaseTimeToLiveResponse { // Keys is the list of keys attached to this lease. repeated bytes keys = 5; } +message LeaseLeasesRequest { +} +message LeaseStatus { + int64 ID = 1; + // TODO: int64 TTL = 2; +} +message LeaseLeasesResponse { + ResponseHeader header = 1; + repeated LeaseStatus leases = 2; +} message Member { // ID is the member ID for this member. uint64 ID = 1; @@ -678,9 +728,17 @@ message DefragmentRequest { message DefragmentResponse { ResponseHeader header = 1; } +message MoveLeaderRequest { + // targetID is the node ID for the new leader. + uint64 targetID = 1; +} +message MoveLeaderResponse { + ResponseHeader header = 1; +} enum AlarmType { None = 0; // default, used to query if any alarm is active Nospace = 1; // space quota is exhausted + Corrupt = 2; // kv store corruption detected } message AlarmRequest { enum AlarmAction { diff --git a/src/builder.ts b/src/builder.ts index 0152143..759cb64 100644 --- a/src/builder.ts +++ b/src/builder.ts @@ -33,6 +33,7 @@ export const compareTarget: { [key in keyof typeof RPC.CompareTarget]: keyof RPC Version: 'version', Create: 'create_revision', Mod: 'mod_revision', + Lease: 'lease', }; /** diff --git a/src/rpc.ts b/src/rpc.ts index 4b665e3..f4ce5c9 100644 --- a/src/rpc.ts +++ b/src/rpc.ts @@ -107,6 +107,12 @@ export class LeaseClient { public leaseTimeToLive(req: ILeaseTimeToLiveRequest): Promise { return this.client.exec('Lease', 'leaseTimeToLive', req); } + /** + * LeaseLeases lists all existing leases. + */ + public leaseLeases(): Promise { + return this.client.exec('Lease', 'leaseLeases', {}); + } } export class ClusterClient { @@ -158,19 +164,31 @@ export class MaintenanceClient { return this.client.exec('Maintenance', 'defragment', {}); } /** - * Hash returns the hash of the local KV state for consistency checking purpose. + * Hash computes the hash of the KV's backend. * This is designed for testing; do not use this in production when there * are ongoing transactions. */ public hash(): Promise { return this.client.exec('Maintenance', 'hash', {}); } + /** + * HashKV computes the hash of all MVCC keys up to a given revision. + */ + public hashKV(req: IHashKVRequest): Promise { + return this.client.exec('Maintenance', 'hashKV', req); + } /** * Snapshot sends a snapshot of the entire backend from a member over a stream to a client. */ public snapshot(): Promise> { return this.client.getConnection('Maintenance').then(cnx => (cnx.client).snapshot({})); } + /** + * MoveLeader requests current leader node to transfer its leadership to transferee. + */ + public moveLeader(req: IMoveLeaderRequest): Promise { + return this.client.exec('Maintenance', 'moveLeader', req); + } } export class AuthClient { @@ -475,11 +493,13 @@ export interface IRequestOp { request_range?: IRangeRequest; request_put?: IPutRequest; request_delete_range?: IDeleteRangeRequest; + request_txn?: ITxnRequest; } export interface IResponseOp { response_range: IRangeResponse; response_put: IPutResponse; response_delete_range: IDeleteRangeResponse; + response_txn: ITxnResponse; } export enum CompareResult { Equal = 0, @@ -492,6 +512,7 @@ export enum CompareTarget { Create = 1, Mod = 2, Value = 3, + Lease = 4, } export interface ICompare { /** @@ -522,6 +543,15 @@ export interface ICompare { * value is the value of the given key, in bytes. */ value?: Buffer; + /** + * lease is the lease id of the given key. + */ + lease?: string | number; + /** + * range_end compares the given target to all keys in the range [key, range_end). + * See RangeRequest for more details on key ranges. + */ + range_end?: Buffer; } export interface ITxnRequest { /** @@ -555,7 +585,7 @@ export interface ITxnResponse { } export interface ICompactionRequest { /** - * revision is the key-value store revision for the compaction operation. + * revision is the key-value store revision for the compaction operation. */ revision?: string | number; /** @@ -568,10 +598,27 @@ export interface ICompactionRequest { export interface ICompactionResponse { header: IResponseHeader; } +export interface IHashKVRequest { + /** + * revision is the key-value store revision for the hash operation. + */ + revision?: string | number; +} +export interface IHashKVResponse { + header: IResponseHeader; + /** + * hash is the hash value computed from the responding member's MVCC keys up to a given revision. + */ + hash: string; + /** + * compact_revision is the compacted revision of key-value store when hash begins. + */ + compact_revision: string; +} export interface IHashResponse { header: IResponseHeader; /** - * hash is the hash value computed from the responding member's key-value store. + * hash is the hash value computed from the responding member's KV's backend. */ hash: string; } @@ -746,6 +793,13 @@ export interface ILeaseTimeToLiveResponse { */ keys: Buffer[]; } +export interface ILeaseStatus { + ID: string; +} +export interface ILeaseLeasesResponse { + header: IResponseHeader; + leases: ILeaseStatus[]; +} export interface IMember { /** * ID is the member ID for this member. @@ -821,6 +875,15 @@ export interface IMemberListResponse { export interface IDefragmentResponse { header: IResponseHeader; } +export interface IMoveLeaderRequest { + /** + * targetID is the node ID for the new leader. + */ + targetID?: string | number; +} +export interface IMoveLeaderResponse { + header: IResponseHeader; +} export enum AlarmType { /** * default, used to query if any alarm is active @@ -830,6 +893,10 @@ export enum AlarmType { * space quota is exhausted */ Nospace = 1, + /** + * kv store corruption detected + */ + Corrupt = 2, } export enum AlarmAction { Get = 0, @@ -1017,6 +1084,25 @@ export interface IAuthRoleGrantPermissionResponse { export interface IAuthRoleRevokePermissionResponse { header: IResponseHeader; } +export interface IUser { + name?: Buffer; + password?: Buffer; + roles?: string[]; +} +export enum Permission { + Read = 0, + Write = 1, + Readwrite = 2, +} +export interface IPermission { + permType: keyof typeof Permission; + key: Buffer; + range_end: Buffer; +} +export interface IRole { + name?: Buffer; + keyPermission?: IPermission[]; +} export interface IKeyValue { /** * key is the first key for the range. If range_end is not given, the request only looks up key. @@ -1055,25 +1141,6 @@ export interface IEvent { */ prev_kv: IKeyValue; } -export interface IUser { - name?: Buffer; - password?: Buffer; - roles?: string[]; -} -export enum Permission { - Read = 0, - Write = 1, - Readwrite = 2, -} -export interface IPermission { - permType: keyof typeof Permission; - key: Buffer; - range_end: Buffer; -} -export interface IRole { - name?: Buffer; - keyPermission?: IPermission[]; -} export const Services = { KV: KVClient, Watch: WatchClient,