github
/
vitess-gh
зеркало из https://github.com/github/vitess-gh.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge branch 'master' into suguwork

This commit is contained in:
Sugu Sougoumarane 2015-10-17 21:07:05 -07:00
Родитель c17675130b 2aa089fc97
Коммит 29c7d9fdf0
85 изменённых файлов: 1187 добавлений и 6782 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
Makefile
Просмотреть файл

@ -117,6 +117,9 @@ docker_lite:
docker_guestbook:
cd examples/kubernetes/guestbook && ./build.sh
docker_etcd:
cd docker/etcd-lite && ./build.sh
# This rule loads the working copy of the code into a bootstrap image,
# and then runs the tests inside Docker.
# Example: $ make docker_test flavor=mariadb

11
docker/etcd-lite/build.sh
Просмотреть файл

@ -3,12 +3,19 @@
# This is the script to build the vitess/etcd-lite Docker image by extracting
# the pre-built binaries from a vitess/etcde image.
version="v2.0.13"
set -e
# Build a fresh base vitess/etcd image
(cd ../etcd ; sudo docker build -t vitess/etcd:$version .)
# Extract files from vitess/etcd image
mkdir base
sudo docker run -ti --rm -v $PWD/base:/base -u root vitess/etcd:v2.0.13 bash -c 'cp -R /go/bin/etcd /go/bin/etcdctl /base/'
sudo docker run -ti --rm -v $PWD/base:/base -u root vitess/etcd:$version bash -c 'cp -R /go/bin/etcd /go/bin/etcdctl /base/'
# Build vitess/etcd-lite image
sudo docker build -t vitess/etcd:v2.0.13-lite .
sudo docker build -t vitess/etcd:$version-lite .
# Clean up temporary files
sudo rm -rf base

11
go/cmd/vtcombo/plugin_bsonp3vtgateservice.go
Просмотреть файл

@ -1,11 +0,0 @@
// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
// Imports and register the bsonp3 vtgateservice server
import (
_ "github.com/youtube/vitess/go/vt/vtgate/bsonp3vtgateservice"
)

11
go/cmd/vtgate/plugin_bsonp3vtgateservice.go
Просмотреть файл

@ -1,11 +0,0 @@
// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
// Imports and register the bsonp3 vtgateservice server
import (
_ "github.com/youtube/vitess/go/vt/vtgate/bsonp3vtgateservice"
)

43
go/cmd/vtgateclienttest/bsonp3clienttest/gorpc_goclient_test.go
Просмотреть файл

@ -1,43 +0,0 @@
// Copyright 2015 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bsonp3clienttest
import (
"net"
"net/http"
"testing"
"github.com/youtube/vitess/go/cmd/vtgateclienttest/goclienttest"
"github.com/youtube/vitess/go/cmd/vtgateclienttest/services"
"github.com/youtube/vitess/go/rpcplus"
"github.com/youtube/vitess/go/rpcwrap/bsonrpc"
"github.com/youtube/vitess/go/vt/vtgate/bsonp3vtgateservice"
)
func TestBSONRPCP3GoClient(t *testing.T) {
service := services.CreateServices()
// listen on a random port
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("Cannot listen: %v", err)
}
defer listener.Close()
// Create a Go Rpc server and listen on the port
server := rpcplus.NewServer()
server.Register(bsonp3vtgateservice.New(service))
// create the HTTP server, serve the server from it
handler := http.NewServeMux()
bsonrpc.ServeCustomRPC(handler, server)
httpServer := http.Server{
Handler: handler,
}
go httpServer.Serve(listener)
// and run the test suite
goclienttest.TestGoClient(t, "bsonp3", listener.Addr().String())
}

10
go/cmd/vtgateclienttest/bsonp3clienttest/import.go
Просмотреть файл

@ -1,10 +0,0 @@
// Copyright 2015 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bsonp3clienttest
import (
// import the bsonp3 client, it will register itself
_ "github.com/youtube/vitess/go/vt/vtgate/bsonp3vtgateconn"
)

11
go/cmd/vtgateclienttest/plugin_bsonp3vtgateservice.go
Просмотреть файл

@ -1,11 +0,0 @@
// Copyright 2013, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
// Imports and register the bsonp3 vtgateservice server
import (
_ "github.com/youtube/vitess/go/vt/vtgate/bsonp3vtgateservice"
)

6
go/vt/binlog/updatestreamctl.go
Просмотреть файл

@ -127,12 +127,18 @@ func logError() {
// EnableUpdateStreamService enables the RPC service for UpdateStream
func EnableUpdateStreamService(dbname string, mysqld mysqlctl.MysqlDaemon) {
defer logError()
if UpdateStreamRpcService == nil {
return
}
UpdateStreamRpcService.enable(dbname, mysqld)
}
// DisableUpdateStreamService disables the RPC service for UpdateStream
func DisableUpdateStreamService() {
defer logError()
if UpdateStreamRpcService == nil {
return
}
UpdateStreamRpcService.disable()
}

2
go/vt/tabletmanager/tmclient/rpc_client_api.go
Просмотреть файл

@ -23,7 +23,7 @@ import (
// TabletManagerProtocol is the implementation to use for tablet
// manager protocol. It is exported for tests only.
var TabletManagerProtocol = flag.String("tablet_manager_protocol", "bson", "the protocol to use to talk to vttablet")
var TabletManagerProtocol = flag.String("tablet_manager_protocol", "grpc", "the protocol to use to talk to vttablet")
// ErrFunc is used by streaming RPCs that don't return a specific result
type ErrFunc func() error

401
go/vt/vtgate/bsonp3vtgateconn/conn.go
Просмотреть файл

@ -1,401 +0,0 @@
// Copyright 2015, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package bsonp3vtgateconn provides go rpc connectivity for VTGate,
// with BSON-encoded proto3 structs.
package bsonp3vtgateconn
import (
"strings"
"time"
mproto "github.com/youtube/vitess/go/mysql/proto"
"github.com/youtube/vitess/go/rpcplus"
"github.com/youtube/vitess/go/rpcwrap/bsonrpc"
"github.com/youtube/vitess/go/vt/callerid"
tproto "github.com/youtube/vitess/go/vt/tabletserver/proto"
"github.com/youtube/vitess/go/vt/vterrors"
"github.com/youtube/vitess/go/vt/vtgate/proto"
"github.com/youtube/vitess/go/vt/vtgate/vtgateconn"
"golang.org/x/net/context"
qpb "github.com/youtube/vitess/go/vt/proto/query"
topopb "github.com/youtube/vitess/go/vt/proto/topodata"
pb "github.com/youtube/vitess/go/vt/proto/vtgate"
)
func init() {
vtgateconn.RegisterDialer("bsonp3", dial)
}
type vtgateConn struct {
rpcConn *rpcplus.Client
}
func dial(ctx context.Context, address string, timeout time.Duration) (vtgateconn.Impl, error) {
network := "tcp"
if strings.Contains(address, "/") {
network = "unix"
}
rpcConn, err := bsonrpc.DialHTTP(network, address, timeout)
if err != nil {
return nil, err
}
return &vtgateConn{rpcConn: rpcConn}, nil
}
func (conn *vtgateConn) Execute(ctx context.Context, query string, bindVars map[string]interface{}, tabletType topopb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
var s *pb.Session
if session != nil {
s = session.(*pb.Session)
}
request := &pb.ExecuteRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
Query: tproto.BoundQueryToProto3(query, bindVars),
TabletType: tabletType,
NotInTransaction: notInTransaction,
}
response := &pb.ExecuteResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.Execute", request, response); err != nil {
return nil, session, vterrors.FromJSONError(err)
}
if response.Error != nil {
return nil, response.Session, vterrors.FromVtRPCError(response.Error)
}
return mproto.Proto3ToQueryResult(response.Result), response.Session, nil
}
func (conn *vtgateConn) ExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]interface{}, tabletType topopb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
var s *pb.Session
if session != nil {
s = session.(*pb.Session)
}
request := &pb.ExecuteShardsRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
Query: tproto.BoundQueryToProto3(query, bindVars),
Keyspace: keyspace,
Shards: shards,
TabletType: tabletType,
NotInTransaction: notInTransaction,
}
response := &pb.ExecuteShardsResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.ExecuteShards", request, response); err != nil {
return nil, session, vterrors.FromJSONError(err)
}
if response.Error != nil {
return nil, response.Session, vterrors.FromVtRPCError(response.Error)
}
return mproto.Proto3ToQueryResult(response.Result), response.Session, nil
}
func (conn *vtgateConn) ExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]interface{}, tabletType topopb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
var s *pb.Session
if session != nil {
s = session.(*pb.Session)
}
request := &pb.ExecuteKeyspaceIdsRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
Query: tproto.BoundQueryToProto3(query, bindVars),
Keyspace: keyspace,
KeyspaceIds: keyspaceIds,
TabletType: tabletType,
NotInTransaction: notInTransaction,
}
response := &pb.ExecuteKeyspaceIdsResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.ExecuteKeyspaceIds", request, response); err != nil {
return nil, session, vterrors.FromJSONError(err)
}
if response.Error != nil {
return nil, response.Session, vterrors.FromVtRPCError(response.Error)
}
return mproto.Proto3ToQueryResult(response.Result), response.Session, nil
}
func (conn *vtgateConn) ExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topopb.KeyRange, bindVars map[string]interface{}, tabletType topopb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
var s *pb.Session
if session != nil {
s = session.(*pb.Session)
}
request := &pb.ExecuteKeyRangesRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
Query: tproto.BoundQueryToProto3(query, bindVars),
Keyspace: keyspace,
KeyRanges: keyRanges,
TabletType: tabletType,
NotInTransaction: notInTransaction,
}
response := &pb.ExecuteKeyRangesResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.ExecuteKeyRanges", request, response); err != nil {
return nil, session, vterrors.FromJSONError(err)
}
if response.Error != nil {
return nil, response.Session, vterrors.FromVtRPCError(response.Error)
}
return mproto.Proto3ToQueryResult(response.Result), response.Session, nil
}
func (conn *vtgateConn) ExecuteEntityIds(ctx context.Context, query string, keyspace string, entityColumnName string, entityKeyspaceIDs []*pb.ExecuteEntityIdsRequest_EntityId, bindVars map[string]interface{}, tabletType topopb.TabletType, notInTransaction bool, session interface{}) (*mproto.QueryResult, interface{}, error) {
var s *pb.Session
if session != nil {
s = session.(*pb.Session)
}
request := &pb.ExecuteEntityIdsRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
Query: tproto.BoundQueryToProto3(query, bindVars),
Keyspace: keyspace,
EntityColumnName: entityColumnName,
EntityKeyspaceIds: entityKeyspaceIDs,
TabletType: tabletType,
NotInTransaction: notInTransaction,
}
response := &pb.ExecuteEntityIdsResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.ExecuteEntityIds", request, response); err != nil {
return nil, session, vterrors.FromJSONError(err)
}
if response.Error != nil {
return nil, response.Session, vterrors.FromVtRPCError(response.Error)
}
return mproto.Proto3ToQueryResult(response.Result), response.Session, nil
}
func (conn *vtgateConn) ExecuteBatchShards(ctx context.Context, queries []proto.BoundShardQuery, tabletType topopb.TabletType, asTransaction bool, session interface{}) ([]mproto.QueryResult, interface{}, error) {
var s *pb.Session
if session != nil {
s = session.(*pb.Session)
}
request := &pb.ExecuteBatchShardsRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
Queries: proto.BoundShardQueriesToProto(queries),
TabletType: tabletType,
AsTransaction: asTransaction,
}
response := &pb.ExecuteBatchShardsResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.ExecuteBatchShards", request, response); err != nil {
return nil, session, vterrors.FromJSONError(err)
}
if response.Error != nil {
return nil, response.Session, vterrors.FromVtRPCError(response.Error)
}
return mproto.Proto3ToQueryResults(response.Results), response.Session, nil
}
func (conn *vtgateConn) ExecuteBatchKeyspaceIds(ctx context.Context, queries []proto.BoundKeyspaceIdQuery, tabletType topopb.TabletType, asTransaction bool, session interface{}) ([]mproto.QueryResult, interface{}, error) {
var s *pb.Session
if session != nil {
s = session.(*pb.Session)
}
request := &pb.ExecuteBatchKeyspaceIdsRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
Queries: proto.BoundKeyspaceIdQueriesToProto(queries),
TabletType: tabletType,
AsTransaction: asTransaction,
}
response := &pb.ExecuteBatchKeyspaceIdsResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.ExecuteBatchKeyspaceIds", request, response); err != nil {
return nil, session, vterrors.FromJSONError(err)
}
if response.Error != nil {
return nil, response.Session, vterrors.FromVtRPCError(response.Error)
}
return mproto.Proto3ToQueryResults(response.Results), response.Session, nil
}
func (conn *vtgateConn) StreamExecute(ctx context.Context, query string, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
return conn.StreamExecute2(ctx, query, bindVars, tabletType)
}
func (conn *vtgateConn) StreamExecute2(ctx context.Context, query string, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
req := &pb.StreamExecuteRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Query: tproto.BoundQueryToProto3(query, bindVars),
TabletType: tabletType,
}
sr := make(chan *pb.StreamExecuteResponse, 10)
c := conn.rpcConn.StreamGo("VTGateP3.StreamExecute", req, sr)
srr := make(chan *qpb.QueryResult)
go func() {
for v := range sr {
srr <- v.Result
}
close(srr)
}()
return sendStreamResults(c, srr)
}
func (conn *vtgateConn) StreamExecuteShards(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
return conn.StreamExecuteShards2(ctx, query, keyspace, shards, bindVars, tabletType)
}
func (conn *vtgateConn) StreamExecuteShards2(ctx context.Context, query string, keyspace string, shards []string, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
req := &pb.StreamExecuteShardsRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Query: tproto.BoundQueryToProto3(query, bindVars),
Keyspace: keyspace,
Shards: shards,
TabletType: tabletType,
}
sr := make(chan *pb.StreamExecuteShardsResponse, 10)
c := conn.rpcConn.StreamGo("VTGateP3.StreamExecuteShards", req, sr)
srr := make(chan *qpb.QueryResult)
go func() {
for v := range sr {
srr <- v.Result
}
close(srr)
}()
return sendStreamResults(c, srr)
}
func (conn *vtgateConn) StreamExecuteKeyRanges(ctx context.Context, query string, keyspace string, keyRanges []*topopb.KeyRange, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
return conn.StreamExecuteKeyRanges2(ctx, query, keyspace, keyRanges, bindVars, tabletType)
}
func (conn *vtgateConn) StreamExecuteKeyRanges2(ctx context.Context, query string, keyspace string, keyRanges []*topopb.KeyRange, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
req := &pb.StreamExecuteKeyRangesRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Query: tproto.BoundQueryToProto3(query, bindVars),
Keyspace: keyspace,
KeyRanges: keyRanges,
TabletType: tabletType,
}
sr := make(chan *pb.StreamExecuteKeyRangesResponse, 10)
c := conn.rpcConn.StreamGo("VTGateP3.StreamExecuteKeyRanges", req, sr)
srr := make(chan *qpb.QueryResult)
go func() {
for v := range sr {
srr <- v.Result
}
close(srr)
}()
return sendStreamResults(c, srr)
}
func (conn *vtgateConn) StreamExecuteKeyspaceIds(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
return conn.StreamExecuteKeyspaceIds2(ctx, query, keyspace, keyspaceIds, bindVars, tabletType)
}
func (conn *vtgateConn) StreamExecuteKeyspaceIds2(ctx context.Context, query string, keyspace string, keyspaceIds [][]byte, bindVars map[string]interface{}, tabletType topopb.TabletType) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
req := &pb.StreamExecuteKeyspaceIdsRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Query: tproto.BoundQueryToProto3(query, bindVars),
Keyspace: keyspace,
KeyspaceIds: keyspaceIds,
TabletType: tabletType,
}
sr := make(chan *pb.StreamExecuteKeyspaceIdsResponse, 10)
c := conn.rpcConn.StreamGo("VTGateP3.StreamExecuteKeyspaceIds", req, sr)
srr := make(chan *qpb.QueryResult)
go func() {
for v := range sr {
srr <- v.Result
}
close(srr)
}()
return sendStreamResults(c, srr)
}
func sendStreamResults(c *rpcplus.Call, sr chan *qpb.QueryResult) (<-chan *mproto.QueryResult, vtgateconn.ErrFunc, error) {
srout := make(chan *mproto.QueryResult, 1)
go func() {
defer close(srout)
for r := range sr {
srout <- mproto.Proto3ToQueryResult(r)
}
}()
errFunc := func() error {
return vterrors.FromJSONError(c.Error)
}
return srout, errFunc, nil
}
func (conn *vtgateConn) Begin(ctx context.Context) (interface{}, error) {
return conn.Begin2(ctx)
}
func (conn *vtgateConn) Commit(ctx context.Context, session interface{}) error {
return conn.Commit2(ctx, session)
}
func (conn *vtgateConn) Rollback(ctx context.Context, session interface{}) error {
return conn.Rollback2(ctx, session)
}
func (conn *vtgateConn) Begin2(ctx context.Context) (interface{}, error) {
request := &pb.BeginRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
}
response := &pb.BeginResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.Begin", request, response); err != nil {
return nil, vterrors.FromJSONError(err)
}
// Return a non-nil pointer
session := &pb.Session{}
if response.Session != nil {
session = response.Session
}
return session, nil
}
func (conn *vtgateConn) Commit2(ctx context.Context, session interface{}) error {
s := session.(*pb.Session)
request := &pb.CommitRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
}
response := &pb.CommitResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.Commit", request, response); err != nil {
return vterrors.FromJSONError(err)
}
return nil
}
func (conn *vtgateConn) Rollback2(ctx context.Context, session interface{}) error {
s := session.(*pb.Session)
request := &pb.RollbackRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Session: s,
}
response := &pb.RollbackResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.Rollback", request, response); err != nil {
return vterrors.FromJSONError(err)
}
return nil
}
func (conn *vtgateConn) SplitQuery(ctx context.Context, keyspace string, query string, bindVars map[string]interface{}, splitColumn string, splitCount int) ([]*pb.SplitQueryResponse_Part, error) {
request := &pb.SplitQueryRequest{
CallerId: callerid.EffectiveCallerIDFromContext(ctx),
Keyspace: keyspace,
Query: tproto.BoundQueryToProto3(query, bindVars),
SplitColumn: splitColumn,
SplitCount: int64(splitCount),
}
response := &pb.SplitQueryResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.SplitQuery", request, response); err != nil {
return nil, vterrors.FromJSONError(err)
}
return response.Splits, nil
}
func (conn *vtgateConn) GetSrvKeyspace(ctx context.Context, keyspace string) (*topopb.SrvKeyspace, error) {
request := &pb.GetSrvKeyspaceRequest{
Keyspace: keyspace,
}
response := &pb.GetSrvKeyspaceResponse{}
if err := conn.rpcConn.Call(ctx, "VTGateP3.GetSrvKeyspace", request, response); err != nil {
return nil, vterrors.FromJSONError(err)
}
return response.SrvKeyspace, nil
}
func (conn *vtgateConn) Close() {
conn.rpcConn.Close()
}

57
go/vt/vtgate/bsonp3vtgateconn/conn_rpc_test.go
Просмотреть файл

@ -1,57 +0,0 @@
// Copyright 2015, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bsonp3vtgateconn
import (
"net"
"net/http"
"testing"
"time"
"github.com/youtube/vitess/go/rpcplus"
"github.com/youtube/vitess/go/rpcwrap/bsonrpc"
"github.com/youtube/vitess/go/vt/vtgate/bsonp3vtgateservice"
"github.com/youtube/vitess/go/vt/vtgate/vtgateconntest"
"golang.org/x/net/context"
)
// TestBsonP3VTGateConn makes sure the BsonP3RPC service works
func TestBsonP3VTGateConn(t *testing.T) {
// fake service
service := vtgateconntest.CreateFakeServer(t)
// listen on a random port
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("Cannot listen: %v", err)
}
// Create a Go Rpc server and listen on the port
server := rpcplus.NewServer()
server.Register(bsonp3vtgateservice.New(service))
// create the HTTP server, serve the server from it
handler := http.NewServeMux()
bsonrpc.ServeCustomRPC(handler, server)
httpServer := http.Server{
Handler: handler,
}
go httpServer.Serve(listener)
// Create a Go RPC client connecting to the server
ctx := context.Background()
client, err := dial(ctx, listener.Addr().String(), 30*time.Second)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
vtgateconntest.RegisterTestDialProtocol(client)
// run the test suite
vtgateconntest.TestSuite(t, client, service)
vtgateconntest.TestErrorSuite(t, service)
// and clean up
client.Close()
}

370
go/vt/vtgate/bsonp3vtgateservice/server.go
Просмотреть файл

@ -1,370 +0,0 @@
// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package bsonp3vtgateservice provides to go rpc glue for vtgate, with
// BSON-encoded proto3 structs.
package bsonp3vtgateservice
import (
"flag"
"time"
"github.com/youtube/vitess/go/vt/callerid"
"github.com/youtube/vitess/go/vt/servenv"
"github.com/youtube/vitess/go/vt/vterrors"
"github.com/youtube/vitess/go/vt/vtgate"
"github.com/youtube/vitess/go/vt/vtgate/proto"
"github.com/youtube/vitess/go/vt/vtgate/vtgateservice"
"golang.org/x/net/context"
mproto "github.com/youtube/vitess/go/mysql/proto"
pb "github.com/youtube/vitess/go/vt/proto/vtgate"
tproto "github.com/youtube/vitess/go/vt/tabletserver/proto"
)
var (
rpcTimeout = flag.Duration("bsonp3_timeout", 20*time.Second, "rpc timeout")
)
// VTGateP3 is the public structure that is exported via BSON RPC
type VTGateP3 struct {
server vtgateservice.VTGateService
}
// Execute is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) Execute(ctx context.Context, request *pb.ExecuteRequest, response *pb.ExecuteResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
reply := new(proto.QueryResult)
executeErr := vtg.server.Execute(ctx, string(request.Query.Sql), tproto.Proto3ToBindVariables(request.Query.BindVariables), request.TabletType, proto.ProtoToSession(request.Session), request.NotInTransaction, reply)
if executeErr == nil {
response.Error = vtgate.RPCErrorToVtRPCError(reply.Err)
response.Result = mproto.QueryResultToProto3(reply.Result)
response.Session = proto.SessionToProto(reply.Session)
}
return vterrors.ToJSONError(executeErr)
}
// ExecuteShards is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) ExecuteShards(ctx context.Context, request *pb.ExecuteShardsRequest, response *pb.ExecuteShardsResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
reply := new(proto.QueryResult)
executeErr := vtg.server.ExecuteShards(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.Keyspace,
request.Shards,
request.TabletType,
proto.ProtoToSession(request.Session),
request.NotInTransaction,
reply)
if executeErr == nil {
response.Error = vtgate.RPCErrorToVtRPCError(reply.Err)
response.Result = mproto.QueryResultToProto3(reply.Result)
response.Session = proto.SessionToProto(reply.Session)
}
return vterrors.ToJSONError(executeErr)
}
// ExecuteKeyspaceIds is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) ExecuteKeyspaceIds(ctx context.Context, request *pb.ExecuteKeyspaceIdsRequest, response *pb.ExecuteKeyspaceIdsResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
reply := new(proto.QueryResult)
executeErr := vtg.server.ExecuteKeyspaceIds(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.Keyspace,
request.KeyspaceIds,
request.TabletType,
proto.ProtoToSession(request.Session),
request.NotInTransaction,
reply)
if executeErr == nil {
response.Error = vtgate.RPCErrorToVtRPCError(reply.Err)
response.Result = mproto.QueryResultToProto3(reply.Result)
response.Session = proto.SessionToProto(reply.Session)
}
return vterrors.ToJSONError(executeErr)
}
// ExecuteKeyRanges is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) ExecuteKeyRanges(ctx context.Context, request *pb.ExecuteKeyRangesRequest, response *pb.ExecuteKeyRangesResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
reply := new(proto.QueryResult)
executeErr := vtg.server.ExecuteKeyRanges(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.Keyspace,
request.KeyRanges,
request.TabletType,
proto.ProtoToSession(request.Session),
request.NotInTransaction,
reply)
if executeErr == nil {
response.Error = vtgate.RPCErrorToVtRPCError(reply.Err)
response.Result = mproto.QueryResultToProto3(reply.Result)
response.Session = proto.SessionToProto(reply.Session)
}
return vterrors.ToJSONError(executeErr)
}
// ExecuteEntityIds is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) ExecuteEntityIds(ctx context.Context, request *pb.ExecuteEntityIdsRequest, response *pb.ExecuteEntityIdsResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
reply := new(proto.QueryResult)
executeErr := vtg.server.ExecuteEntityIds(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.Keyspace,
request.EntityColumnName,
request.EntityKeyspaceIds,
request.TabletType,
proto.ProtoToSession(request.Session),
request.NotInTransaction,
reply)
if executeErr == nil {
response.Error = vtgate.RPCErrorToVtRPCError(reply.Err)
response.Result = mproto.QueryResultToProto3(reply.Result)
response.Session = proto.SessionToProto(reply.Session)
}
return vterrors.ToJSONError(executeErr)
}
// ExecuteBatchShards is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) ExecuteBatchShards(ctx context.Context, request *pb.ExecuteBatchShardsRequest, response *pb.ExecuteBatchShardsResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
reply := new(proto.QueryResultList)
executeErr := vtg.server.ExecuteBatchShards(ctx,
proto.ProtoToBoundShardQueries(request.Queries),
request.TabletType,
request.AsTransaction,
proto.ProtoToSession(request.Session),
reply)
if executeErr == nil {
response.Error = vtgate.RPCErrorToVtRPCError(reply.Err)
response.Results = tproto.QueryResultListToProto3(reply.List)
response.Session = proto.SessionToProto(reply.Session)
}
return vterrors.ToJSONError(executeErr)
}
// ExecuteBatchKeyspaceIds is the RPC version of
// vtgateservice.VTGateService method
func (vtg *VTGateP3) ExecuteBatchKeyspaceIds(ctx context.Context, request *pb.ExecuteBatchKeyspaceIdsRequest, response *pb.ExecuteBatchKeyspaceIdsResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
reply := new(proto.QueryResultList)
executeErr := vtg.server.ExecuteBatchKeyspaceIds(ctx,
proto.ProtoToBoundKeyspaceIdQueries(request.Queries),
request.TabletType,
request.AsTransaction,
proto.ProtoToSession(request.Session),
reply)
if executeErr == nil {
response.Error = vtgate.RPCErrorToVtRPCError(reply.Err)
response.Results = tproto.QueryResultListToProto3(reply.List)
response.Session = proto.SessionToProto(reply.Session)
}
return vterrors.ToJSONError(executeErr)
}
// StreamExecute is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) StreamExecute(ctx context.Context, request *pb.StreamExecuteRequest, sendReply func(interface{}) error) (err error) {
defer vtg.server.HandlePanic(&err)
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
vtgErr := vtg.server.StreamExecute(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.TabletType,
func(reply *proto.QueryResult) error {
return sendReply(&pb.StreamExecuteResponse{
Result: mproto.QueryResultToProto3(reply.Result),
})
})
return vterrors.ToJSONError(vtgErr)
}
// StreamExecuteShards is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) StreamExecuteShards(ctx context.Context, request *pb.StreamExecuteShardsRequest, sendReply func(interface{}) error) (err error) {
defer vtg.server.HandlePanic(&err)
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
vtgErr := vtg.server.StreamExecuteShards(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.Keyspace,
request.Shards,
request.TabletType,
func(value *proto.QueryResult) error {
return sendReply(&pb.StreamExecuteShardsResponse{
Result: mproto.QueryResultToProto3(value.Result),
})
})
return vterrors.ToJSONError(vtgErr)
}
// StreamExecuteKeyspaceIds is the RPC version of
// vtgateservice.VTGateService method
func (vtg *VTGateP3) StreamExecuteKeyspaceIds(ctx context.Context, request *pb.StreamExecuteKeyspaceIdsRequest, sendReply func(interface{}) error) (err error) {
defer vtg.server.HandlePanic(&err)
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
vtgErr := vtg.server.StreamExecuteKeyspaceIds(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.Keyspace,
request.KeyspaceIds,
request.TabletType,
func(value *proto.QueryResult) error {
return sendReply(&pb.StreamExecuteKeyspaceIdsResponse{
Result: mproto.QueryResultToProto3(value.Result),
})
})
return vterrors.ToJSONError(vtgErr)
}
// StreamExecuteKeyRanges is the RPC version of
// vtgateservice.VTGateService method
func (vtg *VTGateP3) StreamExecuteKeyRanges(ctx context.Context, request *pb.StreamExecuteKeyRangesRequest, sendReply func(interface{}) error) (err error) {
defer vtg.server.HandlePanic(&err)
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
vtgErr := vtg.server.StreamExecuteKeyRanges(ctx,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.Keyspace,
request.KeyRanges,
request.TabletType,
func(value *proto.QueryResult) error {
return sendReply(&pb.StreamExecuteKeyRangesResponse{
Result: mproto.QueryResultToProto3(value.Result),
})
})
return vterrors.ToJSONError(vtgErr)
}
// Begin is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) Begin(ctx context.Context, request *pb.BeginRequest, response *pb.BeginResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
outSession := new(proto.Session)
vtgErr := vtg.server.Begin(ctx, outSession)
if vtgErr == nil {
response.Session = proto.SessionToProto(outSession)
return nil
}
return vterrors.ToJSONError(vtgErr)
}
// Commit is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) Commit(ctx context.Context, request *pb.CommitRequest, response *pb.CommitResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
vtgErr := vtg.server.Commit(ctx, proto.ProtoToSession(request.Session))
return vterrors.ToJSONError(vtgErr)
}
// Rollback is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) Rollback(ctx context.Context, request *pb.RollbackRequest, response *pb.RollbackResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
vtgErr := vtg.server.Rollback(ctx, proto.ProtoToSession(request.Session))
return vterrors.ToJSONError(vtgErr)
}
// SplitQuery is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) SplitQuery(ctx context.Context, request *pb.SplitQueryRequest, response *pb.SplitQueryResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
ctx = callerid.NewContext(ctx,
request.CallerId,
callerid.NewImmediateCallerID("bsonp3 client"))
splits, vtgErr := vtg.server.SplitQuery(ctx,
request.Keyspace,
string(request.Query.Sql),
tproto.Proto3ToBindVariables(request.Query.BindVariables),
request.SplitColumn,
int(request.SplitCount))
if vtgErr != nil {
return vterrors.ToJSONError(vtgErr)
}
response.Splits = splits
return nil
}
// GetSrvKeyspace is the RPC version of vtgateservice.VTGateService method
func (vtg *VTGateP3) GetSrvKeyspace(ctx context.Context, request *pb.GetSrvKeyspaceRequest, response *pb.GetSrvKeyspaceResponse) (err error) {
defer vtg.server.HandlePanic(&err)
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(*rpcTimeout))
defer cancel()
sk, vtgErr := vtg.server.GetSrvKeyspace(ctx, request.Keyspace)
if vtgErr != nil {
return vterrors.ToJSONError(vtgErr)
}
response.SrvKeyspace = sk
return nil
}
// New returns a new VTGateP3 service
func New(vtGate vtgateservice.VTGateService) *VTGateP3 {
return &VTGateP3{vtGate}
}
func init() {
vtgate.RegisterVTGates = append(vtgate.RegisterVTGates, func(vtGate vtgateservice.VTGateService) {
// Using same name as regular gorpc, so they're enabled/disabled together.
servenv.Register("vtgateservice", New(vtGate))
})
}

3
go/vt/vtgate/gorpcvtgateconn/conn_rpc_test.go
Просмотреть файл

@ -12,7 +12,6 @@ import (
"github.com/youtube/vitess/go/rpcplus"
"github.com/youtube/vitess/go/rpcwrap/bsonrpc"
"github.com/youtube/vitess/go/vt/vtgate"
"github.com/youtube/vitess/go/vt/vtgate/gorpcvtgateservice"
"github.com/youtube/vitess/go/vt/vtgate/vtgateconntest"
"golang.org/x/net/context"
@ -32,8 +31,6 @@ func TestGoRPCVTGateConn(t *testing.T) {
// Create a Go Rpc server and listen on the port
server := rpcplus.NewServer()
server.Register(gorpcvtgateservice.New(service))
// TODO(aaijazi): remove this flag once all VtGate Gorpc clients properly support the new behavior
*vtgate.RPCErrorOnlyInReply = true
// create the HTTP server, serve the server from it
handler := http.NewServeMux()

119
go/vt/vtgate/gorpcvtgateservice/server.go
Просмотреть файл

@ -48,10 +48,7 @@ func (vtg *VTGate) Execute(ctx context.Context, request *proto.Query, reply *pro
request.NotInTransaction,
reply)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// ExecuteShard is the RPC version of vtgateservice.VTGateService method
@ -72,10 +69,7 @@ func (vtg *VTGate) ExecuteShard(ctx context.Context, request *proto.QueryShard,
request.NotInTransaction,
reply)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// ExecuteKeyspaceIds is the RPC version of vtgateservice.VTGateService method
@ -96,10 +90,7 @@ func (vtg *VTGate) ExecuteKeyspaceIds(ctx context.Context, request *proto.Keyspa
request.NotInTransaction,
reply)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// ExecuteKeyRanges is the RPC version of vtgateservice.VTGateService method
@ -120,10 +111,7 @@ func (vtg *VTGate) ExecuteKeyRanges(ctx context.Context, request *proto.KeyRange
request.NotInTransaction,
reply)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// ExecuteEntityIds is the RPC version of vtgateservice.VTGateService method
@ -145,10 +133,7 @@ func (vtg *VTGate) ExecuteEntityIds(ctx context.Context, request *proto.EntityId
request.NotInTransaction,
reply)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// ExecuteBatchShard is the RPC version of vtgateservice.VTGateService method
@ -166,10 +151,7 @@ func (vtg *VTGate) ExecuteBatchShard(ctx context.Context, request *proto.BatchQu
request.Session,
reply)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// ExecuteBatchKeyspaceIds is the RPC version of
@ -188,10 +170,7 @@ func (vtg *VTGate) ExecuteBatchKeyspaceIds(ctx context.Context, request *proto.K
request.Session,
reply)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// StreamExecute is the RPC version of vtgateservice.VTGateService method
@ -225,18 +204,10 @@ func (vtg *VTGate) StreamExecute2(ctx context.Context, request *proto.Query, sen
if vtgErr == nil {
return nil
}
if *vtgate.RPCErrorOnlyInReply {
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
// Sending back errors this way is not backwards compatible. If a (new) server sends an additional
// QueryResult with an error, and the (old) client doesn't know how to read it, it will cause
// problems where the client will get out of sync with the number of QueryResults sent.
// That's why this the error is only sent this way when the --rpc_errors_only_in_reply flag is set
// (signalling that all clients are able to handle new-style errors).
return sendReply(qr)
}
return vtgErr
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
return sendReply(qr)
}
// StreamExecuteShard is the RPC version of vtgateservice.VTGateService method
@ -274,18 +245,10 @@ func (vtg *VTGate) StreamExecuteShard2(ctx context.Context, request *proto.Query
if vtgErr == nil {
return nil
}
if *vtgate.RPCErrorOnlyInReply {
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
// Sending back errors this way is not backwards compatible. If a (new) server sends an additional
// QueryResult with an error, and the (old) client doesn't know how to read it, it will cause
// problems where the client will get out of sync with the number of QueryResults sent.
// That's why this the error is only sent this way when the --rpc_errors_only_in_reply flag is set
// (signalling that all clients are able to handle new-style errors).
return sendReply(qr)
}
return vtgErr
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
return sendReply(qr)
}
// StreamExecuteKeyspaceIds is the RPC version of
@ -325,18 +288,10 @@ func (vtg *VTGate) StreamExecuteKeyspaceIds2(ctx context.Context, request *proto
if vtgErr == nil {
return nil
}
if *vtgate.RPCErrorOnlyInReply {
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
// Sending back errors this way is not backwards compatible. If a (new) server sends an additional
// QueryResult with an error, and the (old) client doesn't know how to read it, it will cause
// problems where the client will get out of sync with the number of QueryResults sent.
// That's why this the error is only sent this way when the --rpc_errors_only_in_reply flag is set
// (signalling that all clients are able to handle new-style errors).
return sendReply(qr)
}
return vtgErr
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
return sendReply(qr)
}
// StreamExecuteKeyRanges is the RPC version of
@ -376,18 +331,10 @@ func (vtg *VTGate) StreamExecuteKeyRanges2(ctx context.Context, request *proto.K
if vtgErr == nil {
return nil
}
if *vtgate.RPCErrorOnlyInReply {
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
// Sending back errors this way is not backwards compatible. If a (new) server sends an additional
// QueryResult with an error, and the (old) client doesn't know how to read it, it will cause
// problems where the client will get out of sync with the number of QueryResults sent.
// That's why this the error is only sent this way when the --rpc_errors_only_in_reply flag is set
// (signalling that all clients are able to handle new-style errors).
return sendReply(qr)
}
return vtgErr
// If there was an app error, send a QueryResult back with it.
qr := new(proto.QueryResult)
vtgate.AddVtGateError(vtgErr, &qr.Err)
return sendReply(qr)
}
// Begin is the RPC version of vtgateservice.VTGateService method
@ -426,10 +373,7 @@ func (vtg *VTGate) Begin2(ctx context.Context, request *proto.BeginRequest, repl
reply.Session = &proto.Session{}
vtgErr := vtg.server.Begin(ctx, reply.Session)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// Commit2 is the RPC version of vtgateservice.VTGateService method
@ -442,10 +386,7 @@ func (vtg *VTGate) Commit2(ctx context.Context, request *proto.CommitRequest, re
callerid.NewImmediateCallerID("gorpc client"))
vtgErr := vtg.server.Commit(ctx, request.Session)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// Rollback2 is the RPC version of vtgateservice.VTGateService method
@ -458,10 +399,7 @@ func (vtg *VTGate) Rollback2(ctx context.Context, request *proto.RollbackRequest
callerid.NewImmediateCallerID("gorpc client"))
vtgErr := vtg.server.Rollback(ctx, request.Session)
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// SplitQuery is the RPC version of vtgateservice.VTGateService method
@ -480,10 +418,7 @@ func (vtg *VTGate) SplitQuery(ctx context.Context, request *proto.SplitQueryRequ
request.SplitCount)
reply.Splits = splits
vtgate.AddVtGateError(vtgErr, &reply.Err)
if *vtgate.RPCErrorOnlyInReply {
return nil
}
return vtgErr
return nil
}
// GetSrvKeyspace is the RPC version of vtgateservice.VTGateService method

6
go/vt/vtgate/vtgate.go
Просмотреть файл

@ -8,7 +8,6 @@ package vtgate
import (
"errors"
"flag"
"fmt"
"math"
"strings"
@ -97,11 +96,6 @@ type RegisterVTGate func(vtgateservice.VTGateService)
// RegisterVTGates stores register funcs for VTGate server.
var RegisterVTGates []RegisterVTGate
var (
// RPCErrorOnlyInReply informs vtgateservice(s) about how to return errors.
RPCErrorOnlyInReply = flag.Bool("rpc-error-only-in-reply", false, "if true, supported RPC calls from vtgateservice(s) will only return errors as part of the RPC server response")
)
// Init initializes VTGate server.
func Init(hc discovery.HealthCheck, topoServer topo.Server, serv SrvTopoServer, schema *planbuilder.Schema, cell string, retryDelay time.Duration, retryCount int, connTimeoutTotal, connTimeoutPerConn, connLife time.Duration, maxInFlight int, testGateway string) {
if rpcVTGate != nil {

4
java/client/src/main/java/com/youtube/vitess/client/RpcClientFactory.java
Просмотреть файл

@ -1,10 +1,10 @@
package com.youtube.vitess.client;
import java.net.SocketAddress;
import java.net.InetSocketAddress;
/**
* RpcClientFactory creates a concrete RpcClient.
*/
public interface RpcClientFactory {
RpcClient create(Context ctx, SocketAddress address);
RpcClient create(Context ctx, InetSocketAddress address);
}

4
java/grpc-client/src/main/java/com/youtube/vitess/client/grpc/GrpcClientFactory.java
Просмотреть файл

@ -7,14 +7,14 @@ import com.youtube.vitess.client.RpcClientFactory;
import io.grpc.netty.NegotiationType;
import io.grpc.netty.NettyChannelBuilder;
import java.net.SocketAddress;
import java.net.InetSocketAddress;
/**
* GrpcClientFactory creates RpcClients with the gRPC implemenation.
*/
public class GrpcClientFactory implements RpcClientFactory {
@Override
public RpcClient create(Context ctx, SocketAddress address) {
public RpcClient create(Context ctx, InetSocketAddress address) {
return new GrpcClient(
NettyChannelBuilder.forAddress(address).negotiationType(NegotiationType.PLAINTEXT).build());
}

1
java/pom.xml
Просмотреть файл

@ -15,7 +15,6 @@
<module>gorpc</module>
<module>grpc-client</module>
<module>hadoop</module>
<module>vtgate-client</module>
</modules>
<organization>

21
java/vtgate-client/README.md
Просмотреть файл

@ -1,21 +0,0 @@
VtGate Java Driver
==================
Add the following repository and dependency to your pom.xml to use this client library.
```
<repositories>
<repository>
<id>youtube-snapshots</id>
<url>https://github.com/youtube/mvn-repo/raw/master/snapshots</url>
</repository>
</repositories>
```
```
<dependency>
<groupId>com.youtube.vitess</groupId>
<artifactId>vtgate-client</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
```

150
java/vtgate-client/pom.xml
Просмотреть файл

@ -1,150 +0,0 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.youtube.vitess</groupId>
<artifactId>vitess-parent</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>vtgate-client</artifactId>
<dependencies>
<dependency>
<groupId>com.youtube.vitess</groupId>
<artifactId>gorpc-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.5.1</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>2.5.1</version>
<!-- Include the test jar to reuse Hadoop testing utils -->
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<pluginRepositories>
<pluginRepository>
<releases>
<updatePolicy>never</updatePolicy>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
<id>central</id>
<name>Central Repository</name>
<url>https://repo.maven.apache.org/maven2</url>
</pluginRepository>
<pluginRepository>
<id>protoc-plugin</id>
<url>https://dl.bintray.com/sergei-ivanov/maven/</url>
</pluginRepository>
</pluginRepositories>
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.2.3.Final</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
<argLine>${surefireArgLine}</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.13</version>
<configuration>
<argLine>${failsafeArgLine}</argLine>
<systemPropertyVariables>
<vtgate.test.env>com.youtube.vitess.vtgate.TestEnv</vtgate.test.env>
<vtgate.rpcclient.factory>com.youtube.vitess.vtgate.rpcclient.gorpc.BsonRpcClientFactory</vtgate.rpcclient.factory>
</systemPropertyVariables>
</configuration>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.google.protobuf.tools</groupId>
<artifactId>maven-protoc-plugin</artifactId>
<version>0.4.2</version>
<configuration>
<protocArtifact>com.google.protobuf:protoc:${protobuf.java.version}:exe:${os.detected.classifier}</protocArtifact>
<pluginId>grpc-java</pluginId>
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
<protoSourceRoot>../../proto</protoSourceRoot>
<includes>
<include>query.proto</include>
<include>topodata.proto</include>
<include>vtgate.proto</include>
<include>vtrpc.proto</include>
</includes>
</configuration>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>compile-custom</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

90
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/BatchQuery.java
Просмотреть файл

@ -1,90 +0,0 @@
package com.youtube.vitess.vtgate;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
public class BatchQuery {
private List<String> sqls;
private List<List<BindVariable>> bindVarsList;
private String keyspace;
private String tabletType;
private List<byte[]> keyspaceIds;
private Object session;
private BatchQuery(String keyspace, String tabletType) {
this.keyspace = keyspace;
this.tabletType = tabletType;
this.sqls = new LinkedList<>();
this.bindVarsList = new LinkedList<>();
}
public List<String> getSqls() {
return sqls;
}
public List<List<BindVariable>> getBindVarsList() {
return bindVarsList;
}
public String getTabletType() {
return tabletType;
}
public String getKeyspace() {
return keyspace;
}
public List<byte[]> getKeyspaceIds() {
return keyspaceIds;
}
public Object getSession() {
return session;
}
public void setSession(Object session) {
this.session = session;
}
public static class BatchQueryBuilder {
private BatchQuery query;
public BatchQueryBuilder(String keyspace, String tabletType) {
query = new BatchQuery(keyspace, tabletType);
}
public BatchQuery build() {
if (query.sqls.size() == 0) {
throw new IllegalStateException("query must have at least one sql");
}
if (query.keyspaceIds == null) {
throw new IllegalStateException("query must have keyspaceIds set");
}
return query;
}
public BatchQueryBuilder addSqlAndBindVars(String sql, List<BindVariable> bindVars) {
query.sqls.add(sql);
query.bindVarsList.add(bindVars);
return this;
}
public BatchQueryBuilder withKeyspaceIds(List<KeyspaceId> keyspaceIds) {
List<byte[]> kidsBytes = new ArrayList<>();
for (KeyspaceId kid : keyspaceIds) {
kidsBytes.add(kid.getBytes());
}
query.keyspaceIds = kidsBytes;
return this;
}
public BatchQueryBuilder withAddedKeyspaceId(KeyspaceId keyspaceId) {
if (query.getKeyspaceIds() == null) {
query.keyspaceIds = new ArrayList<byte[]>();
}
query.getKeyspaceIds().add(keyspaceId.getBytes());
return this;
}
}
}

27
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/BatchQueryResponse.java
Просмотреть файл

@ -1,27 +0,0 @@
package com.youtube.vitess.vtgate;
import java.util.List;
public class BatchQueryResponse {
private List<QueryResult> results;
private Object session;
private String error;
public BatchQueryResponse(List<QueryResult> results, Object session, String error) {
this.results = results;
this.session = session;
this.error = error;
}
public List<QueryResult> getResults() {
return results;
}
public Object getSession() {
return session;
}
public String getError() {
return error;
}
}

121
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/BindVariable.java
Просмотреть файл

@ -1,121 +0,0 @@
package com.youtube.vitess.vtgate;
import com.google.common.primitives.UnsignedLong;
import org.apache.commons.lang.CharEncoding;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.ISODateTimeFormat;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
public class BindVariable {
private String name;
private Type type;
private long longVal;
private UnsignedLong uLongVal;
private double doubleVal;
private byte[] byteArrayVal;
private BindVariable(String name, Type type) {
this.name = name;
this.type = type;
}
public static BindVariable forNull(String name) {
return new BindVariable(name, Type.NULL);
}
public static BindVariable forInt(String name, Integer value) {
return forLong(name, value.longValue());
}
public static BindVariable forLong(String name, Long value) {
BindVariable bv = new BindVariable(name, Type.LONG);
bv.longVal = value;
return bv;
}
public static BindVariable forFloat(String name, Float value) {
return forDouble(name, value.doubleValue());
}
public static BindVariable forDouble(String name, Double value) {
BindVariable bv = new BindVariable(name, Type.DOUBLE);
bv.doubleVal = value;
return bv;
}
public static BindVariable forULong(String name, UnsignedLong value) {
BindVariable bv = new BindVariable(name, Type.UNSIGNED_LONG);
bv.uLongVal = value;
return bv;
}
public static BindVariable forDateTime(String name, DateTime value) {
String dateTimeStr =
value.toString(ISODateTimeFormat.dateHourMinuteSecondMillis()).replace('T', ' ');
return forString(name, dateTimeStr);
}
public static BindVariable forDate(String name, DateTime value) {
String date = value.toString(ISODateTimeFormat.date());
return forString(name, date);
}
public static BindVariable forTime(String name, DateTime value) {
String time = value.toString(DateTimeFormat.forPattern("HH:mm:ss"));
return forString(name, time);
}
public static BindVariable forString(String name, String value) {
try {
return forBytes(name, value.getBytes(CharEncoding.ISO_8859_1));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
public static BindVariable forBytes(String name, byte[] value) {
BindVariable bv = new BindVariable(name, Type.BYTE_ARRAY);
bv.byteArrayVal = value;
return bv;
}
public static BindVariable forShort(String name, Short value) {
return forLong(name, value.longValue());
}
public static BindVariable forBigDecimal(String name, BigDecimal value) {
return forDouble(name, value.doubleValue());
}
public String getName() {
return name;
}
public Type getType() {
return type;
}
public long getLongVal() {
return longVal;
}
public UnsignedLong getULongVal() {
return uLongVal;
}
public double getDoubleVal() {
return doubleVal;
}
public byte[] getByteArrayVal() {
return byteArrayVal;
}
public static enum Type {
NULL, LONG, UNSIGNED_LONG, DOUBLE, BYTE_ARRAY;
}
}

44
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/Exceptions.java
Просмотреть файл

@ -1,44 +0,0 @@
package com.youtube.vitess.vtgate;
public class Exceptions {
/**
* Exception raised at the tablet or MySQL layer due to issues such as invalid syntax, etc.
*/
@SuppressWarnings("serial")
public static class DatabaseException extends Exception {
public DatabaseException(String message) {
super(message);
}
}
/**
* Exception raised by MySQL due to violation of unique key constraint
*/
@SuppressWarnings("serial")
public static class IntegrityException extends DatabaseException {
public IntegrityException(String message) {
super(message);
}
}
/**
* Exception caused due to irrecoverable connection failures or other low level exceptions
*/
@SuppressWarnings("serial")
public static class ConnectionException extends Exception {
public ConnectionException(String message) {
super(message);
}
}
/**
* Exception raised due to fetching a non-existent field or with the wrong type
*/
@SuppressWarnings("serial")
public static class InvalidFieldException extends RuntimeException {
public InvalidFieldException(String message) {
super(message);
}
}
}

158
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/Field.java
Просмотреть файл

@ -1,158 +0,0 @@
package com.youtube.vitess.vtgate;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.Row.Cell;
import org.apache.commons.lang.CharEncoding;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.ISODateTimeFormat;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
public class Field {
/**
* MySQL field flags bitset values e.g. to distinguish between signed and unsigned integer.
* Comments are taken from the original source code.
* These numbers should exactly match values defined in dist/mysql-5.1.52/include/mysql_com.h
*/
public enum Flag {
// VT_ZEROVALUE_FLAG is not part of the MySQL specification and only used in unit tests.
VT_ZEROVALUE_FLAG(0),
VT_NOT_NULL_FLAG (1), /* Field can't be NULL */
VT_PRI_KEY_FLAG(2), /* Field is part of a primary key */
VT_UNIQUE_KEY_FLAG(4), /* Field is part of a unique key */
VT_MULTIPLE_KEY_FLAG(8), /* Field is part of a key */
VT_BLOB_FLAG(16), /* Field is a blob */
VT_UNSIGNED_FLAG(32), /* Field is unsigned */
VT_ZEROFILL_FLAG(64), /* Field is zerofill */
VT_BINARY_FLAG(128), /* Field is binary */
/* The following are only sent to new clients */
VT_ENUM_FLAG(256), /* field is an enum */
VT_AUTO_INCREMENT_FLAG(512), /* field is a autoincrement field */
VT_TIMESTAMP_FLAG(1024), /* Field is a timestamp */
VT_SET_FLAG(2048), /* field is a set */
VT_NO_DEFAULT_VALUE_FLAG(4096), /* Field doesn't have default value */
VT_ON_UPDATE_NOW_FLAG(8192), /* Field is set to NOW on UPDATE */
VT_NUM_FLAG(32768); /* Field is num (for clients) */
public long mysqlFlag;
Flag(long mysqlFlag) {
this.mysqlFlag = mysqlFlag;
}
}
private String name;
private FieldType type;
private long mysqlFlags;
public static Field newFieldFromMysql(String name, int mysqlTypeId, long mysqlFlags) {
for (FieldType ft : FieldType.values()) {
if (ft.mysqlType == mysqlTypeId) {
return new Field(name, ft, mysqlFlags);
}
}
throw new RuntimeException("Unknown MySQL type: " + mysqlTypeId);
}
@VisibleForTesting
static Field newFieldForTest(FieldType fieldType, Flag flag) {
return new Field("dummyField", fieldType, flag.mysqlFlag);
}
private Field(String name, FieldType type, long mysqlFlags) {
this.name = name;
this.type = type;
this.mysqlFlags = mysqlFlags;
}
public Cell convertValueToCell(byte[] bytes) {
if ((mysqlFlags & Flag.VT_UNSIGNED_FLAG.mysqlFlag) != 0) {
return new Cell(name, convert(bytes), type.unsignedJavaType);
} else {
return new Cell(name, convert(bytes), type.javaType);
}
}
Object convert(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return null;
}
String s = null;
try {
s = new String(bytes, CharEncoding.ISO_8859_1);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
switch (type) {
case VT_DECIMAL:
return new BigDecimal(s);
case VT_TINY:
return Integer.valueOf(s);
case VT_SHORT:
return Integer.valueOf(s);
case VT_LONG:
return Long.valueOf(s);
case VT_FLOAT:
return Float.valueOf(s);
case VT_DOUBLE:
return Double.valueOf(s);
case VT_NULL:
return null;
case VT_TIMESTAMP:
s = s.replace(' ', 'T');
return DateTime.parse(s);
case VT_LONGLONG:
// This can be an unsigned or a signed long
if ((mysqlFlags & Flag.VT_UNSIGNED_FLAG.mysqlFlag) != 0) {
return UnsignedLong.valueOf(s);
} else {
return Long.valueOf(s);
}
case VT_INT24:
return Integer.valueOf(s);
case VT_DATE:
return DateTime.parse(s, ISODateTimeFormat.date());
case VT_TIME:
DateTime d = DateTime.parse(s, DateTimeFormat.forPattern("HH:mm:ss"));
return d;
case VT_DATETIME:
s = s.replace(' ', 'T');
return DateTime.parse(s);
case VT_YEAR:
return Short.valueOf(s);
case VT_NEWDATE:
return DateTime.parse(s, ISODateTimeFormat.date());
case VT_VARCHAR:
return bytes;
case VT_BIT:
return bytes;
case VT_NEWDECIMAL:
return new BigDecimal(s);
case VT_ENUM:
return s;
case VT_SET:
return s;
case VT_TINY_BLOB:
return bytes;
case VT_MEDIUM_BLOB:
return bytes;
case VT_LONG_BLOB:
return bytes;
case VT_BLOB:
return bytes;
case VT_VAR_STRING:
return bytes;
case VT_STRING:
return bytes;
case VT_GEOMETRY:
return bytes;
default:
throw new RuntimeException("invalid field type " + this);
}
}
}

57
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/FieldType.java
Просмотреть файл

@ -1,57 +0,0 @@
package com.youtube.vitess.vtgate;
import com.google.common.primitives.UnsignedLong;
import org.joda.time.DateTime;
import java.math.BigDecimal;
/**
* Represents all field types supported by Vitess and their corresponding types in Java. mysqlType
* numbers should exactly match values defined in dist/mysql-5.1.52/include/mysql/mysql_com.h
*
*/
public enum FieldType {
VT_DECIMAL(0, BigDecimal.class),
VT_TINY(1, Integer.class),
VT_SHORT(2, Integer.class),
VT_LONG(3, Long.class),
VT_FLOAT(4, Float.class),
VT_DOUBLE(5, Double.class),
VT_NULL(6, null),
VT_TIMESTAMP(7, DateTime.class),
VT_LONGLONG(8, Long.class, UnsignedLong.class),
VT_INT24(9, Integer.class),
VT_DATE(10, DateTime.class),
VT_TIME(11, DateTime.class),
VT_DATETIME(12, DateTime.class),
VT_YEAR(13, Short.class),
VT_NEWDATE(14, DateTime.class),
VT_VARCHAR(15, byte[].class),
VT_BIT(16, byte[].class),
VT_NEWDECIMAL(246, BigDecimal.class),
VT_ENUM(247, String.class),
VT_SET(248, String.class),
VT_TINY_BLOB(249, byte[].class),
VT_MEDIUM_BLOB(250, byte[].class),
VT_LONG_BLOB(251, byte[].class),
VT_BLOB(252, byte[].class),
VT_VAR_STRING(253, byte[].class),
VT_STRING(254, byte[].class),
VT_GEOMETRY(255, byte[].class);
public final int mysqlType;
public final Class javaType;
public final Class unsignedJavaType;
FieldType(int mysqlType, Class javaType) {
this.mysqlType = mysqlType;
this.javaType = javaType;
this.unsignedJavaType = javaType;
}
FieldType(int mysqlType, Class javaType, Class unsignedJavaType) {
this.mysqlType = mysqlType;
this.javaType = javaType;
this.unsignedJavaType = unsignedJavaType;
}
}

29
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/KeyRange.java
Просмотреть файл

@ -1,29 +0,0 @@
package com.youtube.vitess.vtgate;
import java.util.HashMap;
import java.util.Map;
public class KeyRange {
private static final String START = "Start";
private static final String END = "End";
private static final byte[] NULL_BYTES = "".getBytes();
public static final KeyRange ALL = new KeyRange(null, null);
KeyspaceId start;
KeyspaceId end;
public KeyRange(KeyspaceId start, KeyspaceId end) {
this.start = start;
this.end = end;
}
public Map<String, byte[]> toMap() {
byte[] startBytes = start == null ? NULL_BYTES : start.getBytes();
byte[] endBytes = end == null ? NULL_BYTES : end.getBytes();
Map<String, byte[]> map = new HashMap<>();
map.put(START, startBytes);
map.put(END, endBytes);
return map;
}
}

91
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/KeyspaceId.java
Просмотреть файл

@ -1,91 +0,0 @@
package com.youtube.vitess.vtgate;
import com.google.common.primitives.Longs;
import com.google.common.primitives.UnsignedLong;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
/**
* KeyspaceId can be either String or UnsignedLong. Use factory method valueOf to create instances
*/
public class KeyspaceId implements Comparable<KeyspaceId> {
private Object id;
public static final String COL_NAME = "keyspace_id";
public KeyspaceId() {
}
public void setId(Object id) {
if (!(id instanceof String) && !(id instanceof UnsignedLong)) {
throw new IllegalArgumentException(
"invalid id type, must be either String or UnsignedLong " + id.getClass());
}
this.id = id;
}
public Object getId() {
return id;
}
public byte[] getBytes() {
if (id instanceof String) {
try {
return Hex.decodeHex(((String) id).toCharArray());
} catch (DecoderException e) {
throw new IllegalArgumentException("illegal string id", e);
}
} else {
return Longs.toByteArray(((UnsignedLong) id).longValue());
}
}
/**
* Creates a KeyspaceId from id which must be a String or UnsignedLong.
*/
public static KeyspaceId valueOf(Object id) {
KeyspaceId kid = new KeyspaceId();
kid.setId(id);
return kid;
}
@Override
public boolean equals(Object o) {
if (o instanceof KeyspaceId) {
return this.compareTo((KeyspaceId) o) == 0;
}
return false;
}
@Override
public int hashCode() {
if (id instanceof UnsignedLong) {
return ((UnsignedLong) id).hashCode();
}
return ((String) id).hashCode();
}
@Override
public int compareTo(KeyspaceId o) {
if (o == null) {
throw new NullPointerException();
}
if (id instanceof UnsignedLong && o.id instanceof UnsignedLong) {
UnsignedLong thisId = (UnsignedLong) id;
UnsignedLong otherId = (UnsignedLong) o.id;
return thisId.compareTo(otherId);
}
if (id instanceof String && o.id instanceof String) {
String thisId = (String) id;
String otherId = (String) o.id;
return thisId.compareTo(otherId);
}
throw new IllegalArgumentException("unexpected id types");
}
}

132
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/Query.java
Просмотреть файл

@ -1,132 +0,0 @@
package com.youtube.vitess.vtgate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Represents a VtGate query request. Use QueryBuilder to construct instances.
*
*/
public class Query {
private String sql;
private String keyspace;
private List<BindVariable> bindVars;
private String tabletType;
private List<byte[]> keyspaceIds;
private List<Map<String, byte[]>> keyRanges;
private boolean streaming;
private Object session;
private Query(String sql, String keyspace, String tabletType) {
this.sql = sql;
this.keyspace = keyspace;
this.tabletType = tabletType;
}
public String getSql() {
return sql;
}
public String getKeyspace() {
return keyspace;
}
public List<BindVariable> getBindVars() {
return bindVars;
}
public String getTabletType() {
return tabletType;
}
public List<byte[]> getKeyspaceIds() {
return keyspaceIds;
}
public List<Map<String, byte[]>> getKeyRanges() {
return keyRanges;
}
public boolean isStreaming() {
return streaming;
}
public Object getSession() {
return session;
}
public void setSession(Object session) {
this.session = session;
}
public static class QueryBuilder {
private Query query;
public QueryBuilder(String sql, String keyspace, String tabletType) {
query = new Query(sql, keyspace, tabletType);
}
public Query build() {
if (query.keyRanges == null && query.keyspaceIds == null) {
throw new IllegalStateException("query must have either keyspaceIds or keyRanges");
}
if (query.keyRanges != null && query.keyspaceIds != null) {
throw new IllegalStateException("query cannot have both keyspaceIds and keyRanges");
}
return query;
}
public QueryBuilder setBindVars(List<BindVariable> bindVars) {
query.bindVars = bindVars;
return this;
}
public QueryBuilder setKeyspaceIds(List<KeyspaceId> keyspaceIds) {
List<byte[]> kidsBytes = new ArrayList<>();
for (KeyspaceId kid : keyspaceIds) {
kidsBytes.add(kid.getBytes());
}
query.keyspaceIds = kidsBytes;
return this;
}
public QueryBuilder setKeyRanges(List<KeyRange> keyRanges) {
List<Map<String, byte[]>> keyRangeMaps = new ArrayList<>();
for (KeyRange kr : keyRanges) {
keyRangeMaps.add(kr.toMap());
}
query.keyRanges = keyRangeMaps;
return this;
}
public QueryBuilder setStreaming(boolean streaming) {
query.streaming = streaming;
return this;
}
public QueryBuilder addBindVar(BindVariable bindVariable) {
if (query.getBindVars() == null) {
query.bindVars = new ArrayList<BindVariable>();
}
query.getBindVars().add(bindVariable);
return this;
}
public QueryBuilder addKeyspaceId(KeyspaceId keyspaceId) {
if (query.getKeyspaceIds() == null) {
query.keyspaceIds = new ArrayList<byte[]>();
}
query.getKeyspaceIds().add(keyspaceId.getBytes());
return this;
}
public QueryBuilder addKeyRange(KeyRange keyRange) {
if (query.getKeyRanges() == null) {
query.keyRanges = new ArrayList<Map<String, byte[]>>();
}
query.getKeyRanges().add(keyRange.toMap());
return this;
}
}
}

25
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/QueryResponse.java
Просмотреть файл

@ -1,25 +0,0 @@
package com.youtube.vitess.vtgate;
public class QueryResponse {
private QueryResult result;
private Object session;
private String error;
public QueryResponse(QueryResult result, Object session, String error) {
this.result = result;
this.session = session;
this.error = error;
}
public QueryResult getResult() {
return result;
}
public Object getSession() {
return session;
}
public String getError() {
return error;
}
}

39
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/QueryResult.java
Просмотреть файл

@ -1,39 +0,0 @@
package com.youtube.vitess.vtgate;
import com.youtube.vitess.vtgate.cursor.Cursor;
import java.util.List;
/**
* Represents a VtGate query result set. For selects, rows are better accessed through the iterator
* {@link Cursor}.
*/
public class QueryResult {
private List<Row> rows;
private List<Field> fields;
private long rowsAffected;
private long lastRowId;
public QueryResult(List<Row> rows, List<Field> fields, long rowsAffected, long lastRowId) {
this.rows = rows;
this.fields = fields;
this.rowsAffected = rowsAffected;
this.lastRowId = lastRowId;
}
public List<Field> getFields() {
return fields;
}
public List<Row> getRows() {
return rows;
}
public long getRowsAffected() {
return rowsAffected;
}
public long getLastRowId() {
return lastRowId;
}
}

182
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/Row.java
Просмотреть файл

@ -1,182 +0,0 @@
package com.youtube.vitess.vtgate;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.Exceptions.InvalidFieldException;
import com.youtube.vitess.vtgate.Row.Cell;
import org.joda.time.DateTime;
import java.math.BigDecimal;
import java.util.Iterator;
import java.util.List;
public class Row implements Iterator<Cell>, Iterable<Cell> {
private ImmutableMap<String, Cell> contents;
private Iterator<String> iterator;
public Row(List<Cell> cells) {
ImmutableMap.Builder<String, Cell> builder = new ImmutableMap.Builder<>();
for (Cell cell : cells) {
builder.put(cell.getName(), cell);
}
contents = builder.build();
iterator = contents.keySet().iterator();
}
public int size() {
return contents.keySet().size();
}
public Object getObject(int index) throws InvalidFieldException {
if (index >= size()) {
throw new InvalidFieldException("invalid field index " + index);
}
return getObject(contents.keySet().asList().get(index));
}
public Object getObject(String fieldName) throws InvalidFieldException {
if (!contents.containsKey(fieldName)) {
throw new InvalidFieldException("invalid field name " + fieldName);
}
return contents.get(fieldName).getValue();
}
public Integer getInt(String fieldName) throws InvalidFieldException {
return (Integer) getAndCheckType(fieldName, Integer.class);
}
public Integer getInt(int index) throws InvalidFieldException {
return (Integer) getAndCheckType(index, Integer.class);
}
public UnsignedLong getULong(String fieldName) throws InvalidFieldException {
return (UnsignedLong) getAndCheckType(fieldName, UnsignedLong.class);
}
public UnsignedLong getULong(int index) throws InvalidFieldException {
return (UnsignedLong) getAndCheckType(index, UnsignedLong.class);
}
public String getString(String fieldName) throws InvalidFieldException {
return (String) getAndCheckType(fieldName, String.class);
}
public String getString(int index) throws InvalidFieldException {
return (String) getAndCheckType(index, String.class);
}
public Long getLong(String fieldName) throws InvalidFieldException {
return (Long) getAndCheckType(fieldName, Long.class);
}
public Long getLong(int index) throws InvalidFieldException {
return (Long) getAndCheckType(index, Long.class);
}
public Double getDouble(String fieldName) throws InvalidFieldException {
return (Double) getAndCheckType(fieldName, Double.class);
}
public Double getDouble(int index) throws InvalidFieldException {
return (Double) getAndCheckType(index, Double.class);
}
public Float getFloat(String fieldName) throws InvalidFieldException {
return (Float) getAndCheckType(fieldName, Float.class);
}
public Float getFloat(int index) throws InvalidFieldException {
return (Float) getAndCheckType(index, Float.class);
}
public DateTime getDateTime(String fieldName) throws InvalidFieldException {
return (DateTime) getAndCheckType(fieldName, DateTime.class);
}
public DateTime getDateTime(int index) throws InvalidFieldException {
return (DateTime) getAndCheckType(index, DateTime.class);
}
public byte[] getBytes(String fieldName) throws InvalidFieldException {
return (byte[]) getAndCheckType(fieldName, byte[].class);
}
public byte[] getBytes(int index) throws InvalidFieldException {
return (byte[]) getAndCheckType(index, byte[].class);
}
public BigDecimal getBigDecimal(String fieldName) throws InvalidFieldException {
return (BigDecimal) getAndCheckType(fieldName, BigDecimal.class);
}
public BigDecimal getBigDecimal(int index) throws InvalidFieldException {
return (BigDecimal) getAndCheckType(index, BigDecimal.class);
}
public Short getShort(String fieldName) throws InvalidFieldException {
return (Short) getAndCheckType(fieldName, Short.class);
}
public Short getShort(int index) throws InvalidFieldException {
return (Short) getAndCheckType(index, Short.class);
}
private Object getAndCheckType(String fieldName, Class clazz) throws InvalidFieldException {
Object o = getObject(fieldName);
if (o != null && !clazz.isInstance(o)) {
throw new InvalidFieldException(
"type mismatch expected:" + clazz.getName() + " actual: " + o.getClass().getName());
}
return o;
}
private Object getAndCheckType(int index, Class clazz) throws InvalidFieldException {
return getAndCheckType(contents.keySet().asList().get(index), clazz);
}
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public Cell next() {
return contents.get(iterator.next());
}
@Override
public void remove() {
throw new UnsupportedOperationException("can't remove from row");
}
@Override
public Iterator<Cell> iterator() {
return this;
}
public static class Cell {
private String name;
private Object value;
private Class type;
public Cell(String name, Object value, Class type) {
this.name = name;
this.value = value;
this.type = type;
}
public String getName() {
return name;
}
public Object getValue() {
return value;
}
public Class getType() {
return type;
}
}
}

31
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/SplitQueryRequest.java
Просмотреть файл

@ -1,31 +0,0 @@
package com.youtube.vitess.vtgate;
public class SplitQueryRequest {
private String sql;
private String keyspace;
private String splitColumn;
private int splitCount;
public SplitQueryRequest(String sql, String keyspace, int splitCount, String splitColumn) {
this.sql = sql;
this.keyspace = keyspace;
this.splitCount = splitCount;
this.splitColumn = splitColumn;
}
public String getSql() {
return this.sql;
}
public String getKeyspace() {
return this.keyspace;
}
public int getSplitCount() {
return this.splitCount;
}
public String getSplitColumn() {
return this.splitColumn;
}
}

22
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/SplitQueryResponse.java
Просмотреть файл

@ -1,22 +0,0 @@
package com.youtube.vitess.vtgate;
import java.util.Map;
public class SplitQueryResponse {
private Map<Query, Long> queries;
private String error;
public SplitQueryResponse(Map<Query, Long> queries, String error) {
this.queries = queries;
this.error = error;
}
public Map<Query, Long> getQueries() {
return queries;
}
public String getError() {
return error;
}
}

131
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/TestEnv.java
Просмотреть файл

@ -1,131 +0,0 @@
package com.youtube.vitess.vtgate;
import com.google.common.primitives.UnsignedLong;
import com.google.gson.Gson;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.rpcclient.RpcClientFactory;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Helper class to hold the configurations for VtGate setup used in integration tests
*/
public class TestEnv {
public static final String PROPERTY_KEY_RPCCLIENT_FACTORY_CLASS = "vtgate.rpcclient.factory";
private Map<String, List<String>> shardKidMap;
private Map<String, Integer> tablets = new HashMap<>();
private String keyspace;
private Process pythonScriptProcess;
private int port;
private List<KeyspaceId> kids;
public void setKeyspace(String keyspace) {
this.keyspace = keyspace;
}
public String getKeyspace() {
return this.keyspace;
}
public int getPort() {
return this.port;
}
public void setPort(int port) {
this.port = port;
}
public Process getPythonScriptProcess() {
return this.pythonScriptProcess;
}
public void setPythonScriptProcess(Process process) {
this.pythonScriptProcess = process;
}
public void setShardKidMap(Map<String, List<String>> shardKidMap) {
this.shardKidMap = shardKidMap;
}
public Map<String, List<String>> getShardKidMap() {
return this.shardKidMap;
}
public void addTablet(String type, int count) {
tablets.put(type, count);
}
public String getShardNames() {
return StringUtils.join(shardKidMap.keySet(), ",");
}
public String getTabletConfig() {
return new Gson().toJson(tablets);
}
/**
* Return all keyspaceIds in the Keyspace
*/
public List<KeyspaceId> getAllKeyspaceIds() {
if (kids != null) {
return kids;
}
kids = new ArrayList<>();
for (List<String> ids : shardKidMap.values()) {
for (String id : ids) {
kids.add(KeyspaceId.valueOf(UnsignedLong.valueOf(id)));
}
}
return kids;
}
/**
* Return all keyspaceIds in a specific shard
*/
public List<KeyspaceId> getKeyspaceIds(String shardName) {
List<String> kidsStr = shardKidMap.get(shardName);
if (kidsStr != null) {
List<KeyspaceId> kids = new ArrayList<>();
for (String kid : kidsStr) {
kids.add(KeyspaceId.valueOf(UnsignedLong.valueOf(kid)));
}
return kids;
}
return null;
}
/**
* Get setup command to launch a cluster.
*/
public List<String> getSetupCommand() {
String vtTop = System.getenv("VTTOP");
if (vtTop == null) {
throw new RuntimeException("cannot find env variable: VTTOP");
}
List<String> command = new ArrayList<String>();
command.add("python");
command.add(vtTop + "/test/java_vtgate_test_helper.py");
command.add("--shards");
command.add(getShardNames());
command.add("--tablet-config");
command.add(getTabletConfig());
command.add("--keyspace");
command.add(keyspace);
return command;
}
public RpcClientFactory getRpcClientFactory() {
String rpcClientFactoryClass = System.getProperty(PROPERTY_KEY_RPCCLIENT_FACTORY_CLASS);
try {
Class<?> clazz = Class.forName(rpcClientFactoryClass);
return (RpcClientFactory)clazz.newInstance();
} catch (ClassNotFoundException|IllegalAccessException|InstantiationException e) {
throw new RuntimeException(e);
}
}
}

126
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/VtGate.java
Просмотреть файл

@ -1,126 +0,0 @@
package com.youtube.vitess.vtgate;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.Exceptions.DatabaseException;
import com.youtube.vitess.vtgate.Exceptions.IntegrityException;
import com.youtube.vitess.vtgate.cursor.Cursor;
import com.youtube.vitess.vtgate.cursor.CursorImpl;
import com.youtube.vitess.vtgate.cursor.StreamCursor;
import com.youtube.vitess.vtgate.rpcclient.RpcClient;
import com.youtube.vitess.vtgate.rpcclient.RpcClientFactory;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Random;
/**
* A single threaded VtGate client
*
* <p>Usage:
*
* <pre>
*VtGate vtGate = VtGate.connect(addresses);
*Query query = new QueryBuilder()...add params...build();
*Cursor cursor = vtGate.execute(query);
*for(Row row : cursor) {
* processRow(row);
*}
*
*For DMLs
*vtgate.begin();
*Query query = new QueryBuilder()...add params...build();
*vtgate.execute(query);
*vtgate.commit();
*
*vtgate.close();
*</pre>
*
*/
public class VtGate {
private static String INTEGRITY_ERROR_MSG = "(errno 1062)";
private RpcClient client;
private Object session;
/**
* Opens connection to a VtGate server. Connection remains open until close() is called.
*
* @param addresses comma separated list of host:port pairs
* @param timeoutMs connection timeout in milliseconds, 0 for no timeout
* @throws ConnectionException
*/
public static VtGate connect(String addresses, int timeoutMs, RpcClientFactory rpcFactory) throws ConnectionException {
List<String> addressList = Arrays.asList(addresses.split(","));
int index = new Random().nextInt(addressList.size());
RpcClient client = rpcFactory.create(addressList.get(index), timeoutMs);
return new VtGate(client);
}
private VtGate(RpcClient client) {
this.client = client;
}
public void begin() throws ConnectionException {
session = client.begin();
}
public Cursor execute(Query query) throws DatabaseException, ConnectionException {
if (session != null) {
query.setSession(session);
}
QueryResponse response = client.execute(query);
if (response.getSession() != null) {
session = response.getSession();
}
String error = response.getError();
if (error != null) {
if (error.contains(INTEGRITY_ERROR_MSG)) {
throw new IntegrityException(error);
}
throw new DatabaseException(response.getError());
}
if (query.isStreaming()) {
return new StreamCursor(response.getResult(), client);
}
return new CursorImpl(response.getResult());
}
/**
* Split a query into primary key range query parts. Rows corresponding to the sub queries will
* add up to original queries' rows. Sub queries are by default built to run against 'rdonly'
* instances. Batch jobs or MapReduce jobs that needs to scan all rows can use these queries to
* parallelize full table scans.
*/
public Map<Query, Long> splitQuery(String keyspace, String sql, int splitCount, String pkColumn)
throws ConnectionException, DatabaseException {
SplitQueryRequest req = new SplitQueryRequest(sql, keyspace, splitCount, pkColumn);
SplitQueryResponse response = client.splitQuery(req);
if (response.getError() != null) {
throw new DatabaseException(response.getError());
}
return response.getQueries();
}
public void commit() throws ConnectionException {
try {
client.commit(session);
} finally {
session = null;
}
}
public void rollback() throws ConnectionException {
try {
client.rollback(session);
} finally {
session = null;
}
}
public void close() throws ConnectionException {
if (session != null) {
rollback();
}
client.close();
}
}

11
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/cursor/Cursor.java
Просмотреть файл

@ -1,11 +0,0 @@
package com.youtube.vitess.vtgate.cursor;
import com.youtube.vitess.vtgate.Row;
import java.util.Iterator;
public interface Cursor extends Iterator<Row>, Iterable<Row> {
public long getRowsAffected();
public long getLastRowId();
}

43
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/cursor/CursorImpl.java
Просмотреть файл

@ -1,43 +0,0 @@
package com.youtube.vitess.vtgate.cursor;
import com.youtube.vitess.vtgate.QueryResult;
import com.youtube.vitess.vtgate.Row;
import java.util.Iterator;
public class CursorImpl implements Cursor {
private QueryResult result;
private Iterator<Row> iter;
public CursorImpl(QueryResult result) {
this.result = result;
this.iter = result.getRows().iterator();
}
public Row next() {
return this.iter.next();
}
public boolean hasNext() {
return this.iter.hasNext();
}
@Override
public void remove() {
throw new UnsupportedOperationException("cannot remove from results");
}
@Override
public Iterator<Row> iterator() {
return this;
}
public long getRowsAffected() {
return result.getRowsAffected();
}
public long getLastRowId() {
return result.getLastRowId();
}
}

97
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/cursor/StreamCursor.java
Просмотреть файл

@ -1,97 +0,0 @@
package com.youtube.vitess.vtgate.cursor;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.QueryResult;
import com.youtube.vitess.vtgate.Row;
import com.youtube.vitess.vtgate.rpcclient.RpcClient;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import java.util.Iterator;
/**
* StreamCursor is for iterating through stream query results. When the current buffer is completed,
* StreamCursor does an RPC fetch for the next set until the stream ends or an error occurs.
*
*/
public class StreamCursor implements Cursor {
static final Logger logger = LogManager.getLogger(StreamCursor.class.getName());
private QueryResult queryResult;
private Iterator<Row> iterator;
private RpcClient client;
private boolean streamEnded;
public StreamCursor(QueryResult currentResult, RpcClient client) {
this.queryResult = currentResult;
this.iterator = currentResult.getRows().iterator();
this.client = client;
}
@Override
public boolean hasNext() {
fetchMoreIfNeeded();
return iterator.hasNext();
}
@Override
public Row next() {
fetchMoreIfNeeded();
return this.iterator.next();
}
@Override
public void remove() {
throw new UnsupportedOperationException("cannot remove from results");
}
@Override
public Iterator<Row> iterator() {
return this;
}
public long getRowsAffected() {
throw new UnsupportedOperationException("not supported for streaming cursors");
}
public long getLastRowId() {
throw new UnsupportedOperationException("not supported for streaming cursors");
}
/**
* Update the iterator by fetching more rows if current buffer is done
*/
private void fetchMoreIfNeeded() {
// Either current buffer is not empty or we have reached end of stream,
// nothing to do here.
if (this.iterator.hasNext() || streamEnded) {
return;
}
QueryResult qr;
try {
qr = client.streamNext(queryResult.getFields());
} catch (ConnectionException e) {
logger.error("connection exception while streaming", e);
throw new RuntimeException(e);
}
// null reply indicates EndOfStream, mark stream as ended and return
if (qr == null) {
streamEnded = true;
return;
}
// For scatter streaming queries, VtGate sends fields data from each
// shard. Since fields has already been fetched, just ignore these and
// fetch the next batch.
if (qr.getRows().size() == 0) {
fetchMoreIfNeeded();
} else {
// We got more rows, update the current buffer and iterator
queryResult = qr;
iterator = queryResult.getRows().iterator();
}
}
}

96
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/README.md
Просмотреть файл

@ -1,96 +0,0 @@
# Hadoop Integration
This package contains the necessary implementations for providing Hadoop support on Vitess. This allows mapreducing over tables stored in Vitess from Hadoop.
Let's look at an example. Consider a table with the following schema that is stored in Vitess across several shards.
```
create table sample_table (
id bigint auto_increment,
name varchar(64),
keyspace_id bigint(20) unsigned NOT NULL,
primary key (id)) Engine=InnoDB;
```
Let's say we want to write a MapReduce job that imports this table from Vitess to HDFS where each row is turned into a CSV record in HDFS.
We can use [VitessInputFormat](https://github.com/youtube/vitess/blob/090830a29c10ae813a2edae18ad780067fb46c05/java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/VitessInputFormat.java), an implementation of Hadoop's [InputFormat](http://hadoop.apache.org/docs/stable/api/org/apache/hadoop/mapred/InputFormat.html), for that. With VitessInputFormat, rows from the source table are streamed to the mapper task. Each input record has a [NullWritable](https://hadoop.apache.org/docs/r2.2.0/api/org/apache/hadoop/io/NullWritable.html) key (no key, really), and [RowWritable](https://github.com/youtube/vitess/blob/090830a29c10ae813a2edae18ad780067fb46c05/java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/writables/RowWritable.java) as value, which is a writable implementation for the entire row's contents.
Here is an example implementation of our mapper, which transforms each row into a CSV Text.
```java
public class TableCsvMapper extends
Mapper<NullWritable, RowWritable, NullWritable, Text> {
@Override
public void map(NullWritable key, RowWritable value, Context context)
throws IOException, InterruptedException {
Row row = value.getRow();
StringBuilder asCsv = new StringBuilder();
asCsv.append(row.getInt("id"));
asCsv.append(",");
asCsv.append(row.getString("name"));
asCsv.append(",");
asCsv.append(row.getULong("keyspace_id"));
context.write(NullWritable.get(), new Text(asCsv.toString()));
}
}
```
The controller code for this MR job is shown below. Note that we are not specifying any sharding/replication related information here. VtGate figures out the right number of shards and replicas to fetch the rows from. The MR author only needs to worry about which rows to fetch (query), how to process them (mapper) and the extent of parallelism (splitCount).
```java
public static void main(String[] args) {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "map vitess table");
job.setJarByClass(VitessInputFormat.class);
job.setMapperClass(TableCsvMapper.class);
String vtgateAddresses = "localhost:15011,localhost:15012,localhost:15013";
String keyspace = "SAMPLE_KEYSPACE";
String query = "select id, name from sample_table";
int splitCount = 100;
VitessInputFormat.setInput(job, vtgateAddresses, keyspace, query, splitCount);
job.setMapOutputKeyClass(NullWritable.class);
job.setMapOutputValueClass(Text.class);
...// set reducer and outputpath and launch the job
}
```
Refer [this integration test](https://github.com/youtube/vitess/blob/090830a29c10ae813a2edae18ad780067fb46c05/java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/hadoop/MapReduceIT.java) for a working example a MapReduce job on Vitess.
## How it Works
VitessInputFormat relies on VtGate's [SplitQuery](https://github.com/youtube/vitess/blob/c680b39852662739a4f5f539ad3ff48ae83d26d1/go/vt/vtgate/vtgateservice/interface.go#L38) service to obtain the input splits. This service accepts a SplitQueryRequest which consists of an input query and the desired number of splits (splitCount). SplitQuery returns SplitQueryResult, which has a list of SplitQueryParts. SplitQueryPart consists of a KeyRangeQuery and a size estimate of how many rows this sub-query might return. SplitQueryParts return rows that are mutually exclusive and collectively exhaustive - all rows belonging to the original input query will be returned by one and exactly one SplitQueryPart.
VitessInputFormat turns each SplitQueryPart into a mapper task. The number of splits generated may not be exactly equal to the desired split count specified in the input. Specifically, if the desired split count is not a multiple of the number of shards, then VtGate will round it up to the next bigger multiple of number of shards.
In addition to splitting the query, the SplitQuery service also acts as a gatekeeper that rejects queries unsuitable for MapReduce. Queries with potentially expensive operations such as Joins, Group By, inner queries, Distinct, Order By, etc are not allowed. Specifically, only input queries of the following syntax are permitted.
```
select [list of columns] from table where [list of simple column filters];
```
There are additional constraints on the table schema to ensure that the sub queries do not result in full table scans. The table must have a primary key (simple or composite) and the leading primary key must be of one of the following types.
```
VT_TINY, VT_SHORT, VT_LONG, VT_LONGLONG, VT_INT24, VT_FLOAT, VT_DOUBLE
```
#### Split Generation
Here's how SplitQuery works. VtGate forwards the input query to randomly chosen ‘rdonly’ vttablets in each shard with a split count, M = original split count / N, where N is the number of shards. Each vttablet parses the query and rejects it if it does not meet the constraints. If it is a valid query, the tablet fetches the min and max value for the leading primary key column from MySQL. Split the [min, max] range of into M intervals. Construct subqueries by appending where clauses corresponding to PK range intervals to the original query and return it to VtGate. VtGate aggregates the splits received from tablets and constructs KeyRangeQueries by appending KeyRange corresponding to that shard. The following diagram depicts this flow for a sample request of split size 6 on a cluster with two shards.
![Image](https://raw.githubusercontent.com/youtube/vitess/090830a29c10ae813a2edae18ad780067fb46c05/java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/SplitQuery.png)
## Other Considerations
1. Specifying splitCount - Each split is a streaming query executed by a single mapper task. splitCount determines the number of mapper tasks that will be created and thus the extent of parallelism. Having too few, but long-running, splits would limit the throughput of the MR job as a whole. Long-running splits also makes retries of individual tasks failures more expensive as compared to leaner splits. On the other side, having too many splits can lead to extra overhead in task setup and connection overhead with VtGate. So, identifying the ideal split count is a balance between the two.
2. Joining multiple tables - Currently Vitess does not currently mapping over joined tables. However, this can be easily achieved by writing a multi-mapper MapReduce job and performing a reduce-side join in the MR job.
3. Views - Database Views are not great for full-table scans. If you need to map over a View, consider mapping over the underlying tables instead.

Двоичные данные
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/SplitQuery.png
Просмотреть файл

Двоичный файл не отображается.
Режим "рядом"

До

Ширина:  |  Высота:  |  Размер: 33 KiB

78
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/VitessConf.java
Просмотреть файл

@ -1,78 +0,0 @@
package com.youtube.vitess.vtgate.hadoop;
import com.youtube.vitess.vtgate.rpcclient.RpcClientFactory;
import org.apache.hadoop.conf.Configuration;
/**
* Collection of configuration properties used for {@link VitessInputFormat}
*/
public class VitessConf {
public static final String HOSTS = "vitess.vtgate.hosts";
public static final String CONN_TIMEOUT_MS = "vitess.vtgate.conn_timeout_ms";
public static final String INPUT_KEYSPACE = "vitess.vtgate.hadoop.keyspace";
public static final String INPUT_QUERY = "vitess.vtgate.hadoop.input_query";
public static final String SPLITS = "vitess.vtgate.hadoop.splits";
public static final String SPLIT_COLUMN = "vitess.vtgate.hadoop.splitcolumn";
public static final String RPC_FACTORY_CLASS = "vtgate.rpcclient.factory";
public static final String HOSTS_DELIM = ",";
private Configuration conf;
public VitessConf(Configuration conf) {
this.conf = conf;
}
public String getHosts() {
return conf.get(HOSTS);
}
public void setHosts(String hosts) {
conf.set(HOSTS, hosts);
}
public int getTimeoutMs() {
return conf.getInt(CONN_TIMEOUT_MS, 0);
}
public void setTimeoutMs(int timeoutMs) {
conf.setInt(CONN_TIMEOUT_MS, timeoutMs);
}
public String getKeyspace() {
return conf.get(INPUT_KEYSPACE);
}
public void setKeyspace(String keyspace) {
conf.set(INPUT_KEYSPACE, keyspace);
}
public String getInputQuery() {
return conf.get(INPUT_QUERY);
}
public void setInputQuery(String query) {
conf.set(INPUT_QUERY, query);
}
public int getSplits() {
return conf.getInt(SPLITS, 1);
}
public void setSplits(int splits) {
conf.setInt(SPLITS, splits);
}
public String getSplitColumn() {
return conf.get(SPLIT_COLUMN);
}
public void setSplitColumn(String splitColumn) {
conf.set(SPLIT_COLUMN, splitColumn);
}
public String getRpcFactoryClass() { return conf.get(RPC_FACTORY_CLASS); }
public void setRpcFactoryClass(Class<? extends RpcClientFactory> clz) {
conf.set(RPC_FACTORY_CLASS, clz.getName());
}
}

83
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/VitessInputFormat.java
Просмотреть файл

@ -1,83 +0,0 @@
package com.youtube.vitess.vtgate.hadoop;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.Exceptions.DatabaseException;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.hadoop.writables.KeyspaceIdWritable;
import com.youtube.vitess.vtgate.hadoop.writables.RowWritable;
import com.youtube.vitess.vtgate.rpcclient.RpcClientFactory;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* {@link VitessInputFormat} is the {@link InputFormat} for tables in Vitess. Input splits (
* {@link VitessInputSplit}) are fetched from VtGate via an RPC. map() calls are supplied with a
* {@link KeyspaceIdWritable}, {@link RowWritable} pair.
*/
public class VitessInputFormat extends InputFormat<NullWritable, RowWritable> {
@Override
public List<InputSplit> getSplits(JobContext context) {
try {
VitessConf conf = new VitessConf(context.getConfiguration());
Class<? extends RpcClientFactory> rpcFactoryClass = null;
VtGate vtgate;
try {
rpcFactoryClass = (Class<? extends RpcClientFactory>)Class.forName(conf.getRpcFactoryClass());
vtgate = VtGate.connect(conf.getHosts(), conf.getTimeoutMs(), rpcFactoryClass.newInstance());
} catch (ClassNotFoundException|InstantiationException|IllegalAccessException e) {
throw new RuntimeException(e);
}
Map<Query, Long> queries =
vtgate.splitQuery(conf.getKeyspace(), conf.getInputQuery(), conf.getSplits(), conf.getSplitColumn());
List<InputSplit> splits = new LinkedList<>();
for (Query query : queries.keySet()) {
Long size = queries.get(query);
InputSplit split = new VitessInputSplit(query, size);
splits.add(split);
}
for (InputSplit split : splits) {
((VitessInputSplit) split).setLocations(conf.getHosts().split(VitessConf.HOSTS_DELIM));
}
return splits;
} catch (ConnectionException | DatabaseException e) {
throw new RuntimeException(e);
}
}
public RecordReader<NullWritable, RowWritable> createRecordReader(InputSplit split,
TaskAttemptContext context) throws IOException, InterruptedException {
return new VitessRecordReader();
}
/**
* Sets the necessary configurations for Vitess table input source
*/
public static void setInput(Job job,
String hosts,
String keyspace,
String query,
int splits,
Class<? extends RpcClientFactory> rpcFactoryClass) {
job.setInputFormatClass(VitessInputFormat.class);
VitessConf vtConf = new VitessConf(job.getConfiguration());
vtConf.setHosts(hosts);
vtConf.setKeyspace(keyspace);
vtConf.setInputQuery(query);
vtConf.setSplits(splits);
vtConf.setRpcFactoryClass(rpcFactoryClass);
}
}

64
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/VitessInputSplit.java
Просмотреть файл

@ -1,64 +0,0 @@
package com.youtube.vitess.vtgate.hadoop;
import com.google.gson.Gson;
import com.youtube.vitess.vtgate.Query;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
/**
* {@link VitessInputSplit} has a Query corresponding to a set of rows from a Vitess table input
* source. Locations point to the same VtGate hosts used to fetch the splits. Length information is
* only approximate.
*
*/
public class VitessInputSplit extends InputSplit implements Writable {
private String[] locations;
private Query query;
private long length;
private final Gson gson = new Gson();
public VitessInputSplit(Query query, long length) {
this.query = query;
this.length = length;
}
public VitessInputSplit() {
}
public Query getQuery() {
return query;
}
public void setLocations(String[] locations) {
this.locations = locations;
}
@Override
public long getLength() throws IOException, InterruptedException {
return length;
}
@Override
public String[] getLocations() throws IOException, InterruptedException {
return locations;
}
@Override
public void readFields(DataInput input) throws IOException {
query = gson.fromJson(input.readUTF(), Query.class);
length = input.readLong();
}
@Override
public void write(DataOutput output) throws IOException {
output.writeUTF(gson.toJson(query));
output.writeLong(length);
}
}

93
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/VitessRecordReader.java
Просмотреть файл

@ -1,93 +0,0 @@
package com.youtube.vitess.vtgate.hadoop;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.Exceptions.DatabaseException;
import com.youtube.vitess.vtgate.Row;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.cursor.Cursor;
import com.youtube.vitess.vtgate.hadoop.writables.RowWritable;
import com.youtube.vitess.vtgate.rpcclient.RpcClientFactory;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import java.io.IOException;
public class VitessRecordReader extends RecordReader<NullWritable, RowWritable> {
private VitessInputSplit split;
private VtGate vtgate;
private VitessConf conf;
private RowWritable rowWritable;
private long rowsProcessed = 0;
private Cursor cursor;
/**
* Fetch connection parameters from Configuraiton and open VtGate connection.
*/
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
this.split = (VitessInputSplit) split;
conf = new VitessConf(context.getConfiguration());
try {
Class<? extends RpcClientFactory> rpcFactoryClass = (Class<? extends RpcClientFactory>)Class.forName(conf.getRpcFactoryClass());
vtgate = VtGate.connect(conf.getHosts(), conf.getTimeoutMs(), rpcFactoryClass.newInstance());
} catch (ConnectionException|ClassNotFoundException|InstantiationException|IllegalAccessException e) {
throw new RuntimeException(e);
}
}
@Override
public void close() throws IOException {
if (vtgate != null) {
try {
vtgate.close();
vtgate = null;
} catch (ConnectionException e) {
throw new RuntimeException(e);
}
}
}
@Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
@Override
public RowWritable getCurrentValue() throws IOException, InterruptedException {
return rowWritable;
}
@Override
public float getProgress() throws IOException, InterruptedException {
if (rowsProcessed > split.getLength()) {
return 0.9f;
}
return rowsProcessed / split.getLength();
}
/**
* Fetches the next row. If this is the first invocation for the split, execute the streaming
* query. Subsequent calls just advance the iterator.
*/
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (cursor == null) {
try {
cursor = vtgate.execute(split.getQuery());
} catch (DatabaseException | ConnectionException e) {
throw new RuntimeException(e);
}
}
if (!cursor.hasNext()) {
return false;
}
Row row = cursor.next();
rowWritable = new RowWritable(row);
rowsProcessed++;
return true;
}
}

79
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/utils/GsonAdapters.java
Просмотреть файл

@ -1,79 +0,0 @@
package com.youtube.vitess.vtgate.hadoop.utils;
import com.google.common.primitives.UnsignedLong;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import com.google.gson.JsonSyntaxException;
import com.google.gson.TypeAdapter;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import com.google.gson.stream.JsonWriter;
import org.apache.commons.codec.binary.Base64;
import java.io.IOException;
import java.lang.reflect.Type;
/**
* Custom GSON adapters for {@link UnsignedLong} and {@link Class} types
*/
public class GsonAdapters {
public static final TypeAdapter<UnsignedLong> UNSIGNED_LONG = new TypeAdapter<UnsignedLong>() {
@Override
public UnsignedLong read(JsonReader in) throws IOException {
if (in.peek() == JsonToken.NULL) {
in.nextNull();
return null;
}
try {
return UnsignedLong.valueOf(in.nextString());
} catch (NumberFormatException e) {
throw new JsonSyntaxException(e);
}
}
@Override
public void write(JsonWriter out, UnsignedLong value) throws IOException {
out.value(value.toString());
}
};
public static final TypeAdapter<Class> CLASS = new TypeAdapter<Class>() {
@Override
public Class read(JsonReader in) throws IOException {
if (in.peek() == JsonToken.NULL) {
in.nextNull();
return null;
}
try {
return Class.forName(in.nextString());
} catch (NumberFormatException | ClassNotFoundException e) {
throw new JsonSyntaxException(e);
}
}
@Override
public void write(JsonWriter out, Class value) throws IOException {
out.value(value.getName());
}
};
public static final Object BYTE_ARRAY = new ByteArrayAdapter();
private static class ByteArrayAdapter implements JsonSerializer<byte[]>, JsonDeserializer<byte[]> {
@Override
public byte[] deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
return Base64.decodeBase64(json.getAsString());
}
@Override
public JsonElement serialize(byte[] src, Type typeOfSrc, JsonSerializationContext context) {
return new JsonPrimitive(Base64.encodeBase64String(src));
}
}
}

89
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/writables/KeyspaceIdWritable.java
Просмотреть файл

@ -1,89 +0,0 @@
package com.youtube.vitess.vtgate.hadoop.writables;
import com.google.common.primitives.UnsignedLong;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.hadoop.utils.GsonAdapters;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
/**
* Serializable version of {@link KeyspaceId}
*/
public class KeyspaceIdWritable implements WritableComparable<KeyspaceIdWritable> {
private KeyspaceId keyspaceId;
// KeyspaceId can be UnsignedLong which needs a custom adapter
private Gson gson = new GsonBuilder().registerTypeAdapter(UnsignedLong.class,
GsonAdapters.UNSIGNED_LONG).create();
public KeyspaceIdWritable() {}
public KeyspaceIdWritable(KeyspaceId keyspaceId) {
this.keyspaceId = keyspaceId;
}
@Override
public void write(DataOutput out) throws IOException {
if (keyspaceId.getId() instanceof UnsignedLong) {
out.writeUTF("UnsignedLong");
out.writeUTF(((UnsignedLong) (keyspaceId.getId())).toString());
} else {
out.writeUTF("String");
out.writeUTF((String) (keyspaceId.getId()));
}
}
public KeyspaceId getKeyspaceId() {
return keyspaceId;
}
@Override
public void readFields(DataInput in) throws IOException {
String type = in.readUTF();
Object id;
if ("UnsignedLong".equals(type)) {
id = UnsignedLong.valueOf(in.readUTF());
} else {
id = in.readUTF();
}
keyspaceId = KeyspaceId.valueOf(id);
}
@Override
public String toString() {
return toJson();
}
public String toJson() {
return gson.toJson(keyspaceId, KeyspaceId.class);
}
@Override
public int compareTo(KeyspaceIdWritable o) {
if (o == null) {
throw new NullPointerException();
}
return this.keyspaceId.compareTo(o.keyspaceId);
}
@Override
public boolean equals(Object o) {
if (o instanceof KeyspaceIdWritable) {
return this.compareTo((KeyspaceIdWritable) o) == 0;
}
return false;
}
@Override
public int hashCode() {
return keyspaceId.hashCode();
}
}

125
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/hadoop/writables/RowWritable.java
Просмотреть файл

@ -1,125 +0,0 @@
package com.youtube.vitess.vtgate.hadoop.writables;
import com.google.common.primitives.UnsignedLong;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.youtube.vitess.vtgate.Row;
import com.youtube.vitess.vtgate.Row.Cell;
import com.youtube.vitess.vtgate.hadoop.utils.GsonAdapters;
import org.apache.commons.net.util.Base64;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Date;
import java.util.LinkedList;
/**
* Serializable version of {@link Row}. Only implements {@link Writable} and not
* {@link WritableComparable} since this is not meant to be used as a key.
*
*/
public class RowWritable implements Writable {
private Row row;
// Row contains UnsignedLong and Class objects which need custom adapters
private Gson gson = new GsonBuilder()
.registerTypeHierarchyAdapter(byte[].class, GsonAdapters.BYTE_ARRAY)
.registerTypeAdapter(UnsignedLong.class, GsonAdapters.UNSIGNED_LONG)
.registerTypeAdapter(Class.class, GsonAdapters.CLASS)
.create();
public RowWritable() {
}
public RowWritable(Row row) {
this.row = row;
}
public Row getRow() {
return row;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(row.size());
for (Cell cell : row) {
writeCell(out, cell);
}
}
public void writeCell(DataOutput out, Cell cell) throws IOException {
out.writeUTF(cell.getName());
out.writeUTF(cell.getType().getName());
if (cell.getType().equals(byte[].class)){
out.writeUTF(Base64.encodeBase64String((byte[])cell.getValue()));
} else{
out.writeUTF(cell.getValue().toString());
}
}
@Override
public void readFields(DataInput in) throws IOException {
int size = in.readInt();
LinkedList<Cell> cells = new LinkedList<>();
for (int i = 0; i < size; i++) {
cells.add(readCell(in));
}
row = new Row(cells);
}
public Cell readCell(DataInput in) throws IOException {
String name = in.readUTF();
String type = in.readUTF();
String value = in.readUTF();
Object val = null;
Class clazz;
try {
clazz = Class.forName(type);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
if (clazz.equals(Double.class)) {
val = Double.valueOf(value);
}
if (clazz.equals(Integer.class)) {
val = Integer.valueOf(value);
}
if (clazz.equals(Long.class)) {
val = Long.valueOf(value);
}
if (clazz.equals(Float.class)) {
val = Float.valueOf(value);
}
if (clazz.equals(UnsignedLong.class)) {
val = UnsignedLong.valueOf(value);
}
if (clazz.equals(Date.class)) {
val = Date.parse(value);
}
if (clazz.equals(String.class)) {
val = value;
}
if (clazz.equals(byte[].class)) {
val = Base64.decodeBase64(value);
}
if (val == null) {
throw new RuntimeException("unknown type in RowWritable: " + clazz);
}
return new Cell(name, val, clazz);
}
@Override
public String toString() {
return toJson();
}
public String toJson() {
return gson.toJson(row, Row.class);
}
}

32
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/rpcclient/RpcClient.java
Просмотреть файл

@ -1,32 +0,0 @@
package com.youtube.vitess.vtgate.rpcclient;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.Field;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.QueryResponse;
import com.youtube.vitess.vtgate.QueryResult;
import com.youtube.vitess.vtgate.SplitQueryRequest;
import com.youtube.vitess.vtgate.SplitQueryResponse;
import java.util.List;
public interface RpcClient {
public Object begin() throws ConnectionException;
public void commit(Object session) throws ConnectionException;
public void rollback(Object session) throws ConnectionException;
public QueryResponse execute(Query query) throws ConnectionException;
public List<QueryResponse> executeBatchKeyspaceIds(
List<Query> queries, String tabletType, Object session, boolean asTransaction)
throws ConnectionException;
public QueryResult streamNext(List<Field> fields) throws ConnectionException;
public SplitQueryResponse splitQuery(SplitQueryRequest request) throws ConnectionException;
public void close() throws ConnectionException;
}

7
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/rpcclient/RpcClientFactory.java
Просмотреть файл

@ -1,7 +0,0 @@
package com.youtube.vitess.vtgate.rpcclient;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
public interface RpcClientFactory {
RpcClient create(String address, int timeoutMs) throws ConnectionException;
}

24
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/rpcclient/gorpc/BsonRpcClientFactory.java
Просмотреть файл

@ -1,24 +0,0 @@
package com.youtube.vitess.vtgate.rpcclient.gorpc;
import com.google.common.net.HostAndPort;
import com.youtube.vitess.gorpc.Client;
import com.youtube.vitess.gorpc.Exceptions.GoRpcException;
import com.youtube.vitess.gorpc.codecs.bson.BsonClientCodecFactory;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.rpcclient.RpcClient;
import com.youtube.vitess.vtgate.rpcclient.RpcClientFactory;
public class BsonRpcClientFactory implements RpcClientFactory {
@Override
public RpcClient create(String address, int timeoutMs) throws ConnectionException {
try {
HostAndPort hostAndPort = HostAndPort.fromString(address);
Client client = Client.dialHttp(hostAndPort.getHostText(), hostAndPort.getPort(),
GoRpcClient.BSON_RPC_PATH, timeoutMs, new BsonClientCodecFactory());
return new GoRpcClient(client);
} catch (GoRpcException e) {
GoRpcClient.LOGGER.error("vtgate connection exception: ", e);
throw new ConnectionException(e.getMessage());
}
}
}

258
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/rpcclient/gorpc/Bsonify.java
Просмотреть файл

@ -1,258 +0,0 @@
package com.youtube.vitess.vtgate.rpcclient.gorpc;
import com.google.common.primitives.Ints;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.BatchQuery;
import com.youtube.vitess.vtgate.BatchQueryResponse;
import com.youtube.vitess.vtgate.BindVariable;
import com.youtube.vitess.vtgate.Field;
import com.youtube.vitess.vtgate.Field.Flag;
import com.youtube.vitess.vtgate.KeyRange;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import com.youtube.vitess.vtgate.QueryResponse;
import com.youtube.vitess.vtgate.QueryResult;
import com.youtube.vitess.vtgate.Row;
import com.youtube.vitess.vtgate.Row.Cell;
import com.youtube.vitess.vtgate.SplitQueryRequest;
import com.youtube.vitess.vtgate.SplitQueryResponse;
import org.apache.commons.codec.binary.Hex;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
import org.bson.types.BasicBSONList;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class Bsonify {
public static BSONObject queryToBson(Query query) {
BSONObject b = new BasicBSONObject();
b.put("Sql", query.getSql());
b.put("Keyspace", query.getKeyspace());
b.put("TabletType", query.getTabletType());
if (query.getBindVars() != null) {
b.put("BindVariables", bindVarsToBSON(query.getBindVars()));
}
if (query.getKeyspaceIds() != null) {
b.put("KeyspaceIds", query.getKeyspaceIds());
} else {
b.put("KeyRanges", query.getKeyRanges());
}
if (query.getSession() != null) {
b.put("Session", query.getSession());
}
return b;
}
public static BSONObject bindVarsToBSON(List<BindVariable> bindVars) {
BSONObject bindVariables = new BasicBSONObject();
for (BindVariable b : bindVars) {
if (b.getType().equals(BindVariable.Type.NULL)) {
bindVariables.put(b.getName(), null);
}
if (b.getType().equals(BindVariable.Type.LONG)) {
bindVariables.put(b.getName(), b.getLongVal());
}
if (b.getType().equals(BindVariable.Type.UNSIGNED_LONG)) {
bindVariables.put(b.getName(), b.getULongVal());
}
if (b.getType().equals(BindVariable.Type.DOUBLE)) {
bindVariables.put(b.getName(), b.getDoubleVal());
}
if (b.getType().equals(BindVariable.Type.BYTE_ARRAY)) {
bindVariables.put(b.getName(), b.getByteArrayVal());
}
}
return bindVariables;
}
public static BSONObject batchQueryToBson(BatchQuery batchQuery) {
BSONObject b = new BasicBSONObject();
List<Map<String, Object>> queries = new LinkedList<>();
Iterator<String> sqlIter = batchQuery.getSqls().iterator();
Iterator<List<BindVariable>> bvIter = batchQuery.getBindVarsList().iterator();
while (sqlIter.hasNext()) {
Map<String, Object> query = new HashMap<>();
query.put("Sql", sqlIter.next());
List<BindVariable> bindVars = bvIter.next();
if (bindVars != null) {
query.put("BindVariables", bindVarsToBSON(bindVars));
}
queries.add(query);
}
b.put("Queries", queries);
b.put("Keyspace", batchQuery.getKeyspace());
b.put("TabletType", batchQuery.getTabletType());
b.put("KeyspaceIds", batchQuery.getKeyspaceIds());
if (batchQuery.getSession() != null) {
b.put("Session", batchQuery.getSession());
}
return b;
}
public static QueryResponse bsonToQueryResponse(BSONObject reply) {
String error = null;
if (reply.containsField("Error")) {
byte[] err = (byte[]) reply.get("Error");
if (err.length > 0) {
error = new String(err);
}
}
QueryResult queryResult = null;
BSONObject result = (BSONObject) reply.get("Result");
if (result != null) {
queryResult = bsonToQueryResult(result, null);
}
Object session = null;
if (reply.containsField("Session")) {
session = reply.get("Session");
}
return new QueryResponse(queryResult, session, error);
}
public static BatchQueryResponse bsonToBatchQueryResponse(BSONObject reply) {
String error = null;
if (reply.containsField("Error")) {
byte[] err = (byte[]) reply.get("Error");
if (err.length > 0) {
error = new String(err);
}
}
Object session = null;
if (reply.containsField("Session")) {
session = reply.get("Session");
}
List<QueryResult> qrs = new LinkedList<>();
BasicBSONList results = (BasicBSONList) reply.get("List");
for (Object result : results) {
qrs.add(bsonToQueryResult((BSONObject) result, null));
}
return new BatchQueryResponse(qrs, session, error);
}
public static QueryResult bsonToQueryResult(BSONObject result, List<Field> fields) {
if (fields == null) {
fields = bsonToFields(result);
}
List<Row> rows = bsonToRows(result, fields);
long rowsAffected = (long) result.get("RowsAffected");
long lastRowId = (long) result.get("InsertId");
return new QueryResult(rows, fields, rowsAffected, lastRowId);
}
public static List<Field> bsonToFields(BSONObject result) {
List<Field> fieldList = new LinkedList<>();
BasicBSONList fields = (BasicBSONList) result.get("Fields");
for (Object field : fields) {
BSONObject fieldBson = (BSONObject) field;
String fieldName = new String((byte[]) fieldBson.get("Name"));
int mysqlType = Ints.checkedCast((Long) fieldBson.get("Type"));
long mysqlFlags = Flag.VT_ZEROVALUE_FLAG.mysqlFlag;
Object flags = fieldBson.get("Flags");
if (flags != null) {
mysqlFlags = (Long) flags;
}
fieldList.add(Field.newFieldFromMysql(fieldName, mysqlType, mysqlFlags));
}
return fieldList;
}
public static List<Row> bsonToRows(BSONObject result, List<Field> fields) {
List<Row> rowList = new LinkedList<>();
BasicBSONList rows = (BasicBSONList) result.get("Rows");
for (Object row : rows) {
LinkedList<Cell> cells = new LinkedList<>();
BasicBSONList cols = (BasicBSONList) row;
Iterator<Field> fieldsIter = fields.iterator();
for (Object col : cols) {
byte[] val = col != null ? (byte[]) col : null;
Field field = fieldsIter.next();
cells.add(field.convertValueToCell(val));
}
rowList.add(new Row(cells));
}
return rowList;
}
public static BSONObject splitQueryRequestToBson(SplitQueryRequest request) {
BSONObject query = new BasicBSONObject();
query.put("Sql", request.getSql());
BSONObject b = new BasicBSONObject();
b.put("Keyspace", request.getKeyspace());
b.put("SplitColumn", request.getSplitColumn());
b.put("Query", query);
b.put("SplitCount", request.getSplitCount());
return b;
}
public static SplitQueryResponse bsonToSplitQueryResponse(BSONObject reply) {
String error = null;
if (reply.containsField("Error")) {
byte[] err = (byte[]) reply.get("Error");
if (err.length > 0) {
error = new String(err);
}
}
BasicBSONList result = (BasicBSONList) reply.get("Splits");
Map<Query, Long> queries = new HashMap<>();
for (Object split : result) {
BSONObject splitObj = (BasicBSONObject) split;
BSONObject query = (BasicBSONObject) (splitObj.get("Query"));
String sql = new String((byte[]) query.get("Sql"));
BSONObject bindVars = (BasicBSONObject) query.get("BindVariables");
List<BindVariable> bindVariables = new LinkedList<>();
for (String key : bindVars.keySet()) {
BindVariable bv = null;
BSONObject val = (BasicBSONObject) bindVars.get(key);
if (val == null) {
bv = BindVariable.forNull(key);
}
int type = (int)val.get("Type");
if (type == 3 /*unsigned long*/) {
bv = BindVariable.forULong(key, (UnsignedLong) val.get("ValueUint"));
}
if (type == 2 /*long*/) {
bv = BindVariable.forLong(key, (Long) val.get("ValueInt"));
}
if (type == 4 /*double*/) {
bv = BindVariable.forDouble(key, (Double) val.get("ValueFloat"));
}
if (type == 1 /*byte[]*/) {
bv = BindVariable.forBytes(key, (byte[]) val.get("ValueBytes"));
}
if (bv == null) {
throw new RuntimeException("invalid bind variable type: " + val.getClass());
}
bindVariables.add(bv);
}
BSONObject keyrangePart = (BasicBSONObject)splitObj.get("KeyRangePart");
String keyspace = new String((byte[]) keyrangePart.get("Keyspace"));
List<KeyRange> keyranges = new ArrayList<>();
for (Object o : (List<?>) keyrangePart.get("KeyRanges")) {
BSONObject keyrange = (BasicBSONObject) o;
String start = Hex.encodeHexString((byte[]) keyrange.get("Start"));
String end = Hex.encodeHexString((byte[]) keyrange.get("End"));
KeyRange kr = new KeyRange(KeyspaceId.valueOf(start), KeyspaceId.valueOf(end));
keyranges.add(kr);
}
Query q = new QueryBuilder(sql, keyspace, "rdonly").setKeyRanges(keyranges)
.setBindVars(bindVariables).setStreaming(true).build();
long size = (long) splitObj.get("Size");
queries.put(q, size);
}
return new SplitQueryResponse(queries, error);
}
}

134
java/vtgate-client/src/main/java/com/youtube/vitess/vtgate/rpcclient/gorpc/GoRpcClient.java
Просмотреть файл

@ -1,134 +0,0 @@
package com.youtube.vitess.vtgate.rpcclient.gorpc;
import com.youtube.vitess.gorpc.Client;
import com.youtube.vitess.gorpc.Exceptions.ApplicationException;
import com.youtube.vitess.gorpc.Exceptions.GoRpcException;
import com.youtube.vitess.gorpc.Response;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.Field;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.QueryResponse;
import com.youtube.vitess.vtgate.QueryResult;
import com.youtube.vitess.vtgate.SplitQueryRequest;
import com.youtube.vitess.vtgate.SplitQueryResponse;
import com.youtube.vitess.vtgate.rpcclient.RpcClient;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
import java.util.List;
public class GoRpcClient implements RpcClient {
public static final Logger LOGGER = LogManager.getLogger(GoRpcClient.class.getName());
public static final String BSON_RPC_PATH = "/_bson_rpc_";
private Client client;
public GoRpcClient(Client client) {
this.client = client;
}
@Override
public Object begin() throws ConnectionException {
Response response = call("VTGate.Begin", new BasicBSONObject());
return response.getReply();
}
@Override
public QueryResponse execute(Query query) throws ConnectionException {
String callMethod = null;
Response response;
if (query.isStreaming()) {
if (query.getKeyspaceIds() != null) {
callMethod = "VTGate.StreamExecuteKeyspaceIds";
} else {
callMethod = "VTGate.StreamExecuteKeyRanges";
}
response = streamCall(callMethod, Bsonify.queryToBson(query));
} else {
if (query.getKeyspaceIds() != null) {
callMethod = "VTGate.ExecuteKeyspaceIds";
} else {
callMethod = "VTGate.ExecuteKeyRanges";
}
response = call(callMethod, Bsonify.queryToBson(query));
}
return Bsonify.bsonToQueryResponse((BSONObject) response.getReply());
}
@Override
public List<QueryResponse> executeBatchKeyspaceIds(
List<Query> queries, String tabletType, Object session, boolean asTransaction)
throws ConnectionException {
throw new UnsupportedOperationException();
}
@Override
public QueryResult streamNext(List<Field> fields) throws ConnectionException {
Response response;
try {
response = client.streamNext();
} catch (GoRpcException | ApplicationException e) {
LOGGER.error("vtgate exception", e);
throw new ConnectionException("vtgate exception: " + e.getMessage());
}
if (response == null) {
return null;
}
BSONObject reply = (BSONObject) response.getReply();
if (reply.containsField("Result")) {
BSONObject result = (BSONObject) reply.get("Result");
return Bsonify.bsonToQueryResult(result, fields);
}
return null;
}
@Override
public void commit(Object session) throws ConnectionException {
call("VTGate.Commit", session);
}
@Override
public void rollback(Object session) throws ConnectionException {
call("VTGate.Rollback", session);
}
@Override
public void close() throws ConnectionException {
try {
client.close();
} catch (GoRpcException e) {
LOGGER.error("vtgate exception", e);
throw new ConnectionException("vtgate exception: " + e.getMessage());
}
}
@Override
public SplitQueryResponse splitQuery(SplitQueryRequest request) throws ConnectionException {
String callMethod = "VTGate.SplitQuery";
Response response = call(callMethod, Bsonify.splitQueryRequestToBson(request));
return Bsonify.bsonToSplitQueryResponse((BSONObject) response.getReply());
}
private Response call(String methodName, Object args) throws ConnectionException {
try {
Response response = client.call(methodName, args);
return response;
} catch (GoRpcException | ApplicationException e) {
LOGGER.error("vtgate exception", e);
throw new ConnectionException("vtgate exception: " + e.getMessage());
}
}
private Response streamCall(String methodName, Object args) throws ConnectionException {
try {
client.streamCall(methodName, args);
return client.streamNext();
} catch (GoRpcException | ApplicationException e) {
LOGGER.error("vtgate exception", e);
throw new ConnectionException("vtgate exception: " + e.getMessage());
}
}
}

5
java/vtgate-client/src/main/resources/log4j.properties
Просмотреть файл

@ -1,5 +0,0 @@
log4j.rootLogger=ERROR, consoleAppender
log4j.appender.consoleAppender=org.apache.log4j.ConsoleAppender
log4j.appender.consoleAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.consoleAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n

46
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/DateTypesTest.java
Просмотреть файл

@ -1,46 +0,0 @@
package com.youtube.vitess.vtgate;
import com.youtube.vitess.vtgate.Field.Flag;
import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.text.ParseException;
@RunWith(JUnit4.class)
public class DateTypesTest {
@Test
public void testDateTimes() throws Exception {
DateTime dt = DateTime.now();
byte[] bytes = BindVariable.forDateTime("", dt).getByteArrayVal();
check(FieldType.VT_TIMESTAMP, dt, bytes);
check(FieldType.VT_DATETIME, dt, bytes);
}
@Test
public void testDate() throws Exception {
DateTime now = DateTime.now();
byte[] bytes = BindVariable.forDate("", now).getByteArrayVal();
DateTime date =
now.withMillisOfSecond(0).withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0);
check(FieldType.VT_DATE, date, bytes);
check(FieldType.VT_NEWDATE, date, bytes);
}
@Test
public void testTime() throws Exception {
DateTime now = DateTime.now();
byte[] bytes = BindVariable.forTime("", now).getByteArrayVal();
DateTime time = now.withMillisOfSecond(0).withYear(1970).withMonthOfYear(1).withDayOfMonth(1);
check(FieldType.VT_TIME, time, bytes);
}
private void check(FieldType typeUnderTest, DateTime dt, byte[] bytes) throws ParseException {
Field f = Field.newFieldForTest(typeUnderTest, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(bytes);
Assert.assertEquals(DateTime.class, o.getClass());
Assert.assertEquals(dt, (DateTime) o);
}
}

97
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/FieldTest.java
Просмотреть файл

@ -1,97 +0,0 @@
package com.youtube.vitess.vtgate;
import com.google.common.collect.Lists;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.Field.Flag;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.math.BigDecimal;
import java.util.List;
@RunWith(JUnit4.class)
public class FieldTest {
@Test
public void testDouble() {
List<FieldType> typesToTest = Lists.newArrayList(FieldType.VT_DOUBLE);
String val = "1000.01";
for (FieldType type : typesToTest) {
Field f = Field.newFieldForTest(type, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(val.getBytes());
Assert.assertEquals(Double.class, o.getClass());
Assert.assertEquals(1000.01, ((Double) o).doubleValue(), 0.01);
}
}
@Test
public void testBigDecimal() {
List<FieldType> typesToTest = Lists.newArrayList(FieldType.VT_DECIMAL, FieldType.VT_NEWDECIMAL);
String val = "1000.01";
for (FieldType type : typesToTest) {
Field f = Field.newFieldForTest(type, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(val.getBytes());
Assert.assertEquals(BigDecimal.class, o.getClass());
Assert.assertEquals(1000.01, ((BigDecimal) o).doubleValue(), 0.01);
}
}
@Test
public void testInteger() {
List<FieldType> typesToTest =
Lists.newArrayList(FieldType.VT_TINY, FieldType.VT_SHORT, FieldType.VT_INT24);
String val = "1000";
for (FieldType type : typesToTest) {
Field f = Field.newFieldForTest(type, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(val.getBytes());
Assert.assertEquals(Integer.class, o.getClass());
Assert.assertEquals(1000, ((Integer) o).intValue());
}
}
@Test
public void testLong_LONG() {
String val = "1000";
Field f = Field.newFieldForTest(FieldType.VT_LONG, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(val.getBytes());
Assert.assertEquals(Long.class, o.getClass());
Assert.assertEquals(1000L, ((Long) o).longValue());
}
@Test
public void testLong_LONGLONG() {
String val = "10000000000000";
Field f = Field.newFieldForTest(FieldType.VT_LONGLONG, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(val.getBytes());
Assert.assertEquals(Long.class, o.getClass());
Assert.assertEquals(10000000000000L, ((Long) o).longValue());
}
@Test
public void testLong_LONGLONG_UNSIGNED() {
String val = "10000000000000";
Field f = Field.newFieldForTest(FieldType.VT_LONGLONG, Flag.VT_UNSIGNED_FLAG);
Object o = f.convert(val.getBytes());
Assert.assertEquals(UnsignedLong.class, o.getClass());
Assert.assertEquals(10000000000000L, ((UnsignedLong) o).longValue());
}
@Test
public void testNull() {
Field f = Field.newFieldForTest(FieldType.VT_NULL, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(null);
Assert.assertEquals(null, o);
}
@Test
public void testFloat() {
String val = "1000.01";
Field f = Field.newFieldForTest(FieldType.VT_FLOAT, Flag.VT_ZEROVALUE_FLAG);
Object o = f.convert(val.getBytes());
Assert.assertEquals(Float.class, o.getClass());
Assert.assertEquals(1000.01, ((Float) o).floatValue(), 0.1);
}
}

67
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/QueryBuilderTest.java
Просмотреть файл

@ -1,67 +0,0 @@
package com.youtube.vitess.vtgate;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.util.Arrays;
@RunWith(JUnit4.class)
public class QueryBuilderTest {
@Test
public void testValidQueryWithKeyspaceIds() {
String sql = "select 1 from dual";
KeyspaceId kid = KeyspaceId.valueOf("80");
QueryBuilder builder =
new QueryBuilder("select 1 from dual", "test_keyspace", "master").addKeyspaceId(kid);
Query query = builder.build();
Assert.assertEquals(sql, query.getSql());
Assert.assertEquals("test_keyspace", query.getKeyspace());
Assert.assertEquals("master", query.getTabletType());
Assert.assertEquals(null, query.getBindVars());
Assert.assertEquals(1, query.getKeyspaceIds().size());
Assert.assertTrue(Arrays.equals(kid.getBytes(), query.getKeyspaceIds().get(0)));
Assert.assertEquals(null, query.getKeyRanges());
}
@Test
public void testValidQueryWithKeyRanges() {
String sql = "select 1 from dual";
QueryBuilder builder =
new QueryBuilder("select 1 from dual", "test_keyspace", "master").addKeyRange(KeyRange.ALL);
Query query = builder.build();
Assert.assertEquals(sql, query.getSql());
Assert.assertEquals("test_keyspace", query.getKeyspace());
Assert.assertEquals("master", query.getTabletType());
Assert.assertEquals(null, query.getBindVars());
Assert.assertEquals(1, query.getKeyRanges().size());
Assert.assertEquals(KeyRange.ALL.toMap(), query.getKeyRanges().get(0));
Assert.assertEquals(null, query.getKeyspaceIds());
}
@Test
public void testNoKeyspaceIdOrKeyrange() {
QueryBuilder builder = new QueryBuilder("select 1 from dual", "test_keyspace", "master");
try {
builder.build();
Assert.fail("did not raise IllegalStateException");
} catch (IllegalStateException e) {
Assert.assertEquals("query must have either keyspaceIds or keyRanges", e.getMessage());
}
}
@Test
public void testBothKeyspaceIdAndKeyrange() {
QueryBuilder builder = new QueryBuilder("select 1 from dual", "test_keyspace", "master")
.addKeyRange(KeyRange.ALL).addKeyspaceId(KeyspaceId.valueOf("80"));
try {
builder.build();
Assert.fail("did not raise IllegalStateException");
} catch (IllegalStateException e) {
Assert.assertEquals("query cannot have both keyspaceIds and keyRanges", e.getMessage());
}
}
}

234
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/DataTypesIT.java
Просмотреть файл

@ -1,234 +0,0 @@
package com.youtube.vitess.vtgate.integration;
import com.google.common.collect.Lists;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.BindVariable;
import com.youtube.vitess.vtgate.KeyRange;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import com.youtube.vitess.vtgate.Row;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.cursor.Cursor;
import com.youtube.vitess.vtgate.TestEnv;
import com.youtube.vitess.vtgate.integration.util.Util;
import org.joda.time.DateTime;
import org.joda.time.format.ISODateTimeFormat;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@RunWith(JUnit4.class)
public class DataTypesIT {
public static TestEnv testEnv = getTestEnv();
@BeforeClass
public static void setUpVtGate() throws Exception {
Util.setupTestEnv(testEnv);
}
@AfterClass
public static void tearDownVtGate() throws Exception {
Util.teardownTestEnv(testEnv);
}
@Test
public void testInts() throws Exception {
String createTable =
"create table vtocc_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, year year, primary key(tiny)) comment 'vtocc_nocache'\n" + "";
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
vtgate.execute(getQuery(createTable));
vtgate.commit();
String insertSql =
"insert into vtocc_ints values(:tiny, :tinyu, :small, :smallu, :medium, :mediumu, :normal, :normalu, :big, :bigu, :year)";
Query insertQuery = new QueryBuilder(insertSql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forInt("tiny", -128))
.addBindVar(BindVariable.forInt("tinyu", 255))
.addBindVar(BindVariable.forInt("small", -32768))
.addBindVar(BindVariable.forInt("smallu", 65535))
.addBindVar(BindVariable.forInt("medium", -8388608))
.addBindVar(BindVariable.forInt("mediumu", 16777215))
.addBindVar(BindVariable.forLong("normal", -2147483648L))
.addBindVar(BindVariable.forLong("normalu", 4294967295L))
.addBindVar(BindVariable.forLong("big", -9223372036854775808L))
.addBindVar(BindVariable.forULong("bigu", UnsignedLong.valueOf("18446744073709551615")))
.addBindVar(BindVariable.forShort("year", (short) 2012))
.addKeyRange(KeyRange.ALL)
.build();
vtgate.begin();
vtgate.execute(insertQuery);
vtgate.commit();
String selectSql = "select * from vtocc_ints where tiny = -128";
Query selectQuery =
new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(KeyRange.ALL).build();
Cursor cursor = vtgate.execute(selectQuery);
Assert.assertEquals(1, cursor.getRowsAffected());
Row row = cursor.next();
Assert.assertTrue(row.getInt("tiny").equals(-128));
Assert.assertTrue(row.getInt("tinyu").equals(255));
Assert.assertTrue(row.getInt("small").equals(-32768));
Assert.assertTrue(row.getInt("smallu").equals(65535));
Assert.assertTrue(row.getInt("medium").equals(-8388608));
Assert.assertTrue(row.getInt("mediumu").equals(16777215));
Assert.assertTrue(row.getLong("normal").equals(-2147483648L));
Assert.assertTrue(row.getLong("normalu").equals(4294967295L));
Assert.assertTrue(row.getLong("big").equals(-9223372036854775808L));
Assert.assertTrue(row.getULong("bigu").equals(UnsignedLong.valueOf("18446744073709551615")));
Assert.assertTrue(row.getShort("year").equals((short) 2012));
vtgate.close();
}
@Test
public void testFracts() throws Exception {
String createTable =
"create table vtocc_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id)) comment 'vtocc_nocache'\n" + "";
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
vtgate.execute(getQuery(createTable));
vtgate.commit();
String insertSql = "insert into vtocc_fracts values(:id, :deci, :num, :f, :d)";
Query insertQuery = new QueryBuilder(insertSql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forInt("id", 1))
.addBindVar(BindVariable.forDouble("deci", 1.99))
.addBindVar(BindVariable.forDouble("num", 2.99))
.addBindVar(BindVariable.forFloat("f", 3.99F))
.addBindVar(BindVariable.forDouble("d", 4.99))
.addKeyRange(KeyRange.ALL)
.build();
vtgate.begin();
vtgate.execute(insertQuery);
vtgate.commit();
String selectSql = "select * from vtocc_fracts where id = 1";
Query selectQuery =
new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(KeyRange.ALL).build();
Cursor cursor = vtgate.execute(selectQuery);
Assert.assertEquals(1, cursor.getRowsAffected());
Row row = cursor.next();
Assert.assertTrue(row.getLong("id").equals(1L));
Assert.assertTrue(row.getBigDecimal("deci").equals(new BigDecimal("1.99")));
Assert.assertTrue(row.getBigDecimal("num").equals(new BigDecimal("2.99")));
Assert.assertTrue(row.getFloat("f").equals(3.99F));
Assert.assertTrue(row.getDouble("d").equals(4.99));
vtgate.close();
}
@Test
public void testStrings() throws Exception {
String createTable =
"create table vtocc_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb)) comment 'vtocc_nocache'\n" + "";
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
vtgate.execute(getQuery(createTable));
vtgate.commit();
String insertSql =
"insert into vtocc_strings values(:vb, :c, :vc, :b, :tb, :bl, :ttx, :tx, :en, :s)";
Query insertQuery = new QueryBuilder(insertSql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forBytes("vb", "a".getBytes()))
.addBindVar(BindVariable.forBytes("c", "b".getBytes()))
.addBindVar(BindVariable.forBytes("vc", "c".getBytes()))
.addBindVar(BindVariable.forBytes("b", "d".getBytes()))
.addBindVar(BindVariable.forBytes("tb", "e".getBytes()))
.addBindVar(BindVariable.forBytes("bl", "f".getBytes()))
.addBindVar(BindVariable.forBytes("ttx", "g".getBytes()))
.addBindVar(BindVariable.forBytes("tx", "h".getBytes()))
.addBindVar(BindVariable.forString("en", "a"))
.addBindVar(BindVariable.forString("s", "a,b"))
.addKeyRange(KeyRange.ALL)
.build();
vtgate.begin();
vtgate.execute(insertQuery);
vtgate.commit();
String selectSql = "select * from vtocc_strings where vb = 'a'";
Query selectQuery =
new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(KeyRange.ALL).build();
Cursor cursor = vtgate.execute(selectQuery);
Assert.assertEquals(1, cursor.getRowsAffected());
Row row = cursor.next();
Assert.assertTrue(Arrays.equals(row.getBytes("vb"), "a".getBytes()));
Assert.assertTrue(Arrays.equals(row.getBytes("c"), "b".getBytes()));
Assert.assertTrue(Arrays.equals(row.getBytes("vc"), "c".getBytes()));
// binary(4) column will be suffixed with three 0 bytes
Assert.assertTrue(
Arrays.equals(row.getBytes("b"), ByteBuffer.allocate(4).put((byte) 'd').array()));
Assert.assertTrue(Arrays.equals(row.getBytes("tb"), "e".getBytes()));
Assert.assertTrue(Arrays.equals(row.getBytes("bl"), "f".getBytes()));
Assert.assertTrue(Arrays.equals(row.getBytes("ttx"), "g".getBytes()));
Assert.assertTrue(Arrays.equals(row.getBytes("tx"), "h".getBytes()));
// Assert.assertTrue(row.getString("en").equals("a"));
// Assert.assertTrue(row.getString("s").equals("a,b"));
vtgate.close();
}
@Test
public void testMisc() throws Exception {
String createTable =
"create table vtocc_misc(id int, b bit(8), d date, dt datetime, t time, primary key(id)) comment 'vtocc_nocache'\n" + "";
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
vtgate.execute(getQuery(createTable));
vtgate.commit();
String insertSql = "insert into vtocc_misc values(:id, :b, :d, :dt, :t)";
Query insertQuery = new QueryBuilder(insertSql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forInt("id", 1))
.addBindVar(BindVariable.forBytes("b", ByteBuffer.allocate(1).put((byte) 1).array()))
.addBindVar(BindVariable.forDate("d", DateTime.parse("2012-01-01")))
.addBindVar(BindVariable.forDateTime("dt", DateTime.parse("2012-01-01T15:45:45")))
.addBindVar(BindVariable.forTime("t",
DateTime.parse("15:45:45", ISODateTimeFormat.timeElementParser())))
.addKeyRange(KeyRange.ALL)
.build();
vtgate.begin();
vtgate.execute(insertQuery);
vtgate.commit();
String selectSql = "select * from vtocc_misc where id = 1";
Query selectQuery =
new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(KeyRange.ALL).build();
Cursor cursor = vtgate.execute(selectQuery);
Assert.assertEquals(1, cursor.getRowsAffected());
Row row = cursor.next();
Assert.assertTrue(row.getLong("id").equals(1L));
Assert.assertTrue(
Arrays.equals(row.getBytes("b"), ByteBuffer.allocate(1).put((byte) 1).array()));
Assert.assertTrue(row.getDateTime("d").equals(DateTime.parse("2012-01-01")));
Assert.assertTrue(row.getDateTime("dt").equals(DateTime.parse("2012-01-01T15:45:45")));
Assert.assertTrue(row.getDateTime("t").equals(
DateTime.parse("15:45:45", ISODateTimeFormat.timeElementParser())));
vtgate.close();
}
private Query getQuery(String sql) {
return new QueryBuilder(sql, testEnv.getKeyspace(), "master").addKeyRange(KeyRange.ALL).build();
}
/**
* Create env with two shards each having a master and replica
*/
static TestEnv getTestEnv() {
Map<String, List<String>> shardKidMap = new HashMap<>();
shardKidMap.put("-", Lists.newArrayList("527875958493693904"));
TestEnv env = Util.getTestEnv(shardKidMap, "test_keyspace");
env.addTablet("replica", 1);
return env;
}
}

135
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/FailuresIT.java
Просмотреть файл

@ -1,135 +0,0 @@
package com.youtube.vitess.vtgate.integration;
import com.google.common.collect.Lists;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.BindVariable;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.Exceptions.DatabaseException;
import com.youtube.vitess.vtgate.Exceptions.IntegrityException;
import com.youtube.vitess.vtgate.KeyRange;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.TestEnv;
import com.youtube.vitess.vtgate.integration.util.Util;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Test failures and exceptions
*/
@RunWith(JUnit4.class)
public class FailuresIT {
public static TestEnv testEnv = getTestEnv();
@BeforeClass
public static void setUpVtGate() throws Exception {
Util.setupTestEnv(testEnv);
}
@AfterClass
public static void tearDownVtGate() throws Exception {
Util.teardownTestEnv(testEnv);
}
@Before
public void createTable() throws Exception {
Util.createTable(testEnv);
}
@Test
public void testIntegrityException() throws Exception {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
String insertSql = "insert into vtgate_test(id, keyspace_id) values (:id, :keyspace_id)";
KeyspaceId kid = testEnv.getAllKeyspaceIds().get(0);
Query insertQuery = new QueryBuilder(insertSql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forULong("id", UnsignedLong.valueOf("1")))
.addBindVar(BindVariable.forULong("keyspace_id", ((UnsignedLong) kid.getId())))
.addKeyspaceId(kid).build();
vtgate.begin();
vtgate.execute(insertQuery);
vtgate.commit();
vtgate.begin();
try {
vtgate.execute(insertQuery);
Assert.fail("failed to throw exception");
} catch (IntegrityException e) {
} finally {
vtgate.rollback();
vtgate.close();
}
}
@Test
public void testTimeout() throws ConnectionException, DatabaseException {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 200, testEnv.getRpcClientFactory());
// Check timeout error raised for slow query
Query sleepQuery = new QueryBuilder("select sleep(0.5) from dual", testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).build();
try {
vtgate.execute(sleepQuery);
Assert.fail("did not raise timeout exception");
} catch (ConnectionException e) {
}
vtgate.close();
vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 2000, testEnv.getRpcClientFactory());
// Check no timeout error for fast query
sleepQuery = new QueryBuilder("select sleep(0.01) from dual", testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).build();
vtgate.execute(sleepQuery);
vtgate.close();
}
@Test
public void testTxPoolFull() throws Exception {
List<VtGate> vtgates = new ArrayList<>();
boolean failed = false;
try {
// Transaction cap is 20
for (int i = 0; i < 25; i++) {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgates.add(vtgate);
vtgate.begin();
// Run a query to actually begin a transaction with the tablets
Query query = new QueryBuilder("delete from vtgate_test", testEnv.getKeyspace(), "master")
.addKeyRange(KeyRange.ALL).build();
vtgate.execute(query);
}
} catch (DatabaseException e) {
if (e.getMessage().contains("tx_pool_full")) {
failed = true;
}
} finally {
for (VtGate vtgate : vtgates) {
vtgate.close();
}
}
if (!failed) {
Assert.fail("failed to raise tx_pool_full exception");
}
}
/**
* Create env with two shards each having a master and replica
*/
static TestEnv getTestEnv() {
Map<String, List<String>> shardKidMap = new HashMap<>();
shardKidMap.put("-", Lists.newArrayList("527875958493693904"));
TestEnv env = Util.getTestEnv(shardKidMap, "test_keyspace");
env.addTablet("replica", 1);
return env;
}
}

57
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/StreamingServerShutdownIT.java
Просмотреть файл

@ -1,57 +0,0 @@
package com.youtube.vitess.vtgate.integration;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.cursor.Cursor;
import com.youtube.vitess.vtgate.TestEnv;
import com.youtube.vitess.vtgate.integration.util.Util;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class StreamingServerShutdownIT {
static TestEnv testEnv = VtGateIT.getTestEnv();
@Before
public void setUpVtGate() throws Exception {
Util.setupTestEnv(testEnv);
Util.createTable(testEnv);
}
@After
public void tearDownVtGate() throws Exception {
Util.teardownTestEnv(testEnv);
}
@Test
public void testShutdownServerWhileStreaming() throws Exception {
Util.insertRows(testEnv, 1, 2000);
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
String selectSql = "select A.* from vtgate_test A join vtgate_test B";
Query joinQuery = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).setStreaming(true).build();
Cursor cursor = vtgate.execute(joinQuery);
int count = 0;
try {
while (cursor.hasNext()) {
count++;
if (count == 1) {
Util.teardownTestEnv(testEnv);
}
cursor.next();
}
vtgate.close();
Assert.fail("failed to raise exception");
} catch (RuntimeException e) {
Assert.assertTrue(e.getMessage().length() > 0);
}
}
}

146
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/StreamingVtGateIT.java
Просмотреть файл

@ -1,146 +0,0 @@
package com.youtube.vitess.vtgate.integration;
import com.google.common.collect.Iterables;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.BindVariable;
import com.youtube.vitess.vtgate.KeyRange;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import com.youtube.vitess.vtgate.TestEnv;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.cursor.Cursor;
import com.youtube.vitess.vtgate.cursor.StreamCursor;
import com.youtube.vitess.vtgate.integration.util.Util;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.util.Collections;
import java.util.List;
/**
* Test cases for streaming queries in VtGate
*
*/
@RunWith(JUnit4.class)
public class StreamingVtGateIT {
public static TestEnv testEnv = VtGateIT.getTestEnv();
@BeforeClass
public static void setUpVtGate() throws Exception {
Util.setupTestEnv(testEnv);
}
@AfterClass
public static void tearDownVtGate() throws Exception {
Util.teardownTestEnv(testEnv);
}
@Before
public void createTable() throws Exception {
Util.createTable(testEnv);
}
@Test
public void testStreamCursorType() throws Exception {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
String selectSql = "select * from vtgate_test";
Query allRowsQuery = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).setStreaming(true).build();
Cursor cursor = vtgate.execute(allRowsQuery);
Assert.assertEquals(StreamCursor.class, cursor.getClass());
vtgate.close();
}
/**
* Test StreamExecuteKeyspaceIds query on a single shard
*/
@Test
public void testStreamExecuteKeyspaceIds() throws Exception {
int rowCount = 10;
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, rowCount);
}
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
for (String shardName : testEnv.getShardKidMap().keySet()) {
String selectSql = "select A.* from vtgate_test A join vtgate_test B join vtgate_test C";
Query query = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getKeyspaceIds(shardName)).setStreaming(true).build();
Cursor cursor = vtgate.execute(query);
int count = Iterables.size(cursor);
Assert.assertEquals((int) Math.pow(rowCount, 3), count);
}
vtgate.close();
}
/**
* Same as testStreamExecuteKeyspaceIds but for StreamExecuteKeyRanges
*/
@Test
public void testStreamExecuteKeyRanges() throws Exception {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
int rowCount = 10;
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, rowCount);
}
for (String shardName : testEnv.getShardKidMap().keySet()) {
List<KeyspaceId> kids = testEnv.getKeyspaceIds(shardName);
KeyRange kr = new KeyRange(Collections.min(kids), Collections.max(kids));
String selectSql = "select A.* from vtgate_test A join vtgate_test B join vtgate_test C";
Query query = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(kr)
.setStreaming(true).build();
Cursor cursor = vtgate.execute(query);
int count = Iterables.size(cursor);
Assert.assertEquals((int) Math.pow(rowCount, 3), count);
}
vtgate.close();
}
/**
* Test scatter streaming queries fetch rows from all shards
*/
@Test
public void testScatterStreamingQuery() throws Exception {
int rowCount = 10;
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, rowCount);
}
String selectSql = "select A.* from vtgate_test A join vtgate_test B join vtgate_test C";
Query query = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).setStreaming(true).build();
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
Cursor cursor = vtgate.execute(query);
int count = Iterables.size(cursor);
Assert.assertEquals(2 * (int) Math.pow(rowCount, 3), count);
vtgate.close();
}
@Test
@Ignore("currently failing as vtgate doesn't set the error")
public void testStreamingWrites() throws Exception {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
String insertSql = "insert into vtgate_test " + "(id, name, age, percent, keyspace_id) "
+ "values (:id, :name, :age, :percent, :keyspace_id)";
KeyspaceId kid = testEnv.getAllKeyspaceIds().get(0);
Query query = new QueryBuilder(insertSql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forULong("id", UnsignedLong.valueOf("" + 1)))
.addBindVar(BindVariable.forString("name", ("name_" + 1)))
.addBindVar(BindVariable.forULong("keyspace_id", (UnsignedLong) kid.getId()))
.addKeyspaceId(kid)
.setStreaming(true)
.build();
vtgate.execute(query);
vtgate.commit();
vtgate.close();
}
}

370
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/VtGateIT.java
Просмотреть файл

@ -1,370 +0,0 @@
package com.youtube.vitess.vtgate.integration;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.BindVariable;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.KeyRange;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import com.youtube.vitess.vtgate.Row;
import com.youtube.vitess.vtgate.Row.Cell;
import com.youtube.vitess.vtgate.TestEnv;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.cursor.Cursor;
import com.youtube.vitess.vtgate.cursor.CursorImpl;
import com.youtube.vitess.vtgate.integration.util.Util;
import org.apache.commons.codec.binary.Hex;
import org.joda.time.DateTime;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@RunWith(JUnit4.class)
public class VtGateIT {
public static TestEnv testEnv = getTestEnv();
@BeforeClass
public static void setUpVtGate() throws Exception {
Util.setupTestEnv(testEnv);
}
@AfterClass
public static void tearDownVtGate() throws Exception {
Util.teardownTestEnv(testEnv);
}
@Before
public void createTable() throws Exception {
Util.createTable(testEnv);
}
/**
* Test selects using ExecuteKeyspaceIds
*/
@Test
public void testExecuteKeyspaceIds() throws Exception {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
// Ensure empty table
String selectSql = "select * from vtgate_test";
Query allRowsQuery = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").setKeyspaceIds(
testEnv.getAllKeyspaceIds()).build();
Cursor cursor = vtgate.execute(allRowsQuery);
Assert.assertEquals(CursorImpl.class, cursor.getClass());
Assert.assertEquals(0, cursor.getRowsAffected());
Assert.assertEquals(0, cursor.getLastRowId());
Assert.assertFalse(cursor.hasNext());
vtgate.close();
// Insert 10 rows
Util.insertRows(testEnv, 1000, 10);
vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
cursor = vtgate.execute(allRowsQuery);
Assert.assertEquals(10, cursor.getRowsAffected());
Assert.assertEquals(0, cursor.getLastRowId());
Assert.assertTrue(cursor.hasNext());
// Fetch all rows from the first shard
KeyspaceId firstKid = testEnv.getAllKeyspaceIds().get(0);
Query query =
new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyspaceId(firstKid).build();
cursor = vtgate.execute(query);
// Check field types and values
Row row = cursor.next();
Cell idCell = row.next();
Assert.assertEquals("id", idCell.getName());
Assert.assertEquals(Long.class, idCell.getType());
Long id = row.getLong(idCell.getName());
Cell nameCell = row.next();
Assert.assertEquals("name", nameCell.getName());
Assert.assertEquals(byte[].class, nameCell.getType());
Assert.assertTrue(
Arrays.equals(("name_" + id.toString()).getBytes(), row.getBytes(nameCell.getName())));
Cell ageCell = row.next();
Assert.assertEquals("age", ageCell.getName());
Assert.assertEquals(Integer.class, ageCell.getType());
Assert.assertEquals(Integer.valueOf(2 * id.intValue()), row.getInt(ageCell.getName()));
vtgate.close();
}
/**
* Test queries are routed to the right shard based on based on keyspace ids
*/
@Test
public void testQueryRouting() throws Exception {
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, 10);
}
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
String allRowsSql = "select * from vtgate_test";
for (String shardName : testEnv.getShardKidMap().keySet()) {
Query shardRows = new QueryBuilder(allRowsSql, testEnv.getKeyspace(), "master").setKeyspaceIds(
testEnv.getKeyspaceIds(shardName)).build();
Cursor cursor = vtgate.execute(shardRows);
Assert.assertEquals(10, cursor.getRowsAffected());
}
vtgate.close();
}
@Test
public void testDateFieldTypes() throws Exception {
DateTime dt = DateTime.now().minusDays(2).withMillisOfSecond(0);
Util.insertRows(testEnv, 10, 1, dt);
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
Query allRowsQuery = new QueryBuilder("select * from vtgate_test", testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).build();
Row row = vtgate.execute(allRowsQuery).next();
Assert.assertTrue(dt.equals(row.getDateTime("timestamp_col")));
Assert.assertTrue(dt.equals(row.getDateTime("datetime_col")));
Assert.assertTrue(dt.withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0)
.equals(row.getDateTime("date_col")));
Assert.assertTrue(
dt.withYear(1970).withMonthOfYear(1).withDayOfMonth(1).equals(row.getDateTime("time_col")));
vtgate.close();
}
/**
* Test ALL keyrange fetches rows from all shards
*/
@Test
public void testAllKeyRange() throws Exception {
// Insert 10 rows across the shards
Util.insertRows(testEnv, 1000, 10);
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
String selectSql = "select * from vtgate_test";
Query allRowsQuery =
new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(KeyRange.ALL).build();
Cursor cursor = vtgate.execute(allRowsQuery);
// Verify all rows returned
Assert.assertEquals(10, cursor.getRowsAffected());
vtgate.close();
}
/**
* Test reads using Keyrange query
*/
@Test
public void testKeyRangeReads() throws Exception {
int rowsPerShard = 10;
// insert rows in each shard using ExecuteKeyspaceIds
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, rowsPerShard);
}
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
String selectSql = "select * from vtgate_test";
// Check ALL KeyRange query returns rows from both shards
Query allRangeQuery =
new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(KeyRange.ALL).build();
Cursor cursor = vtgate.execute(allRangeQuery);
Assert.assertEquals(rowsPerShard * 2, cursor.getRowsAffected());
// Check KeyRange query limited to a single shard returns 10 rows each
for (String shardName : testEnv.getShardKidMap().keySet()) {
List<KeyspaceId> shardKids = testEnv.getKeyspaceIds(shardName);
KeyspaceId minKid = Collections.min(shardKids);
KeyspaceId maxKid = Collections.max(shardKids);
KeyRange shardKeyRange = new KeyRange(minKid, maxKid);
Query shardRangeQuery = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(
shardKeyRange).build();
cursor = vtgate.execute(shardRangeQuery);
Assert.assertEquals(rowsPerShard, cursor.getRowsAffected());
}
// Now make a cross-shard KeyRange and check all rows are returned
Iterator<String> shardNameIter = testEnv.getShardKidMap().keySet().iterator();
KeyspaceId kidShard1 = testEnv.getKeyspaceIds(shardNameIter.next()).get(2);
KeyspaceId kidShard2 = testEnv.getKeyspaceIds(shardNameIter.next()).get(2);
KeyRange crossShardKeyrange;
if (kidShard1.compareTo(kidShard2) < 0) {
crossShardKeyrange = new KeyRange(kidShard1, kidShard2);
} else {
crossShardKeyrange = new KeyRange(kidShard2, kidShard1);
}
Query shardRangeQuery = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").addKeyRange(
crossShardKeyrange).build();
cursor = vtgate.execute(shardRangeQuery);
Assert.assertEquals(rowsPerShard * 2, cursor.getRowsAffected());
vtgate.close();
}
/**
* Test inserts using KeyRange query
*/
@Test
public void testKeyRangeWrites() throws Exception {
Random random = new Random();
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
String sql = "insert into vtgate_test " + "(id, name, keyspace_id, age) "
+ "values (:id, :name, :keyspace_id, :age)";
int count = 20;
// Insert 20 rows per shard
for (String shardName : testEnv.getShardKidMap().keySet()) {
List<KeyspaceId> kids = testEnv.getKeyspaceIds(shardName);
KeyspaceId minKid = Collections.min(kids);
KeyspaceId maxKid = Collections.max(kids);
KeyRange kr = new KeyRange(minKid, maxKid);
for (int i = 0; i < count; i++) {
KeyspaceId kid = kids.get(i % kids.size());
Query query = new QueryBuilder(sql, testEnv.getKeyspace(), "master")
.addBindVar(
BindVariable.forULong("id", UnsignedLong.valueOf("" + Math.abs(random.nextInt()))))
.addBindVar(BindVariable.forString("name", ("name_" + i)))
.addBindVar(BindVariable.forULong("keyspace_id", (UnsignedLong) kid.getId()))
.addBindVar(BindVariable.forNull("age"))
.addKeyRange(kr)
.build();
vtgate.execute(query);
}
}
vtgate.commit();
vtgate.close();
// Check 40 rows exist in total
vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
String selectSql = "select * from vtgate_test";
Query allRowsQuery = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").setKeyspaceIds(
testEnv.getAllKeyspaceIds()).build();
Cursor cursor = vtgate.execute(allRowsQuery);
Assert.assertEquals(count * 2, cursor.getRowsAffected());
// Check 20 rows exist per shard
for (String shardName : testEnv.getShardKidMap().keySet()) {
Query shardRows = new QueryBuilder(selectSql, testEnv.getKeyspace(), "master").setKeyspaceIds(
testEnv.getKeyspaceIds(shardName)).build();
cursor = vtgate.execute(shardRows);
Assert.assertEquals(count, cursor.getRowsAffected());
}
vtgate.close();
}
@Test
public void testSplitQuery() throws Exception {
// Insert 20 rows per shard
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, 20);
}
Util.waitForTablet("rdonly", 40, 3, testEnv);
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
Map<Query, Long> queries =
vtgate.splitQuery("test_keyspace", "select id,keyspace_id from vtgate_test", 1, "");
vtgate.close();
// Verify 2 splits, one per shard
Assert.assertEquals(2, queries.size());
Set<String> shardsInSplits = new HashSet<>();
for (Query q : queries.keySet()) {
Assert.assertEquals("select id,keyspace_id from vtgate_test", q.getSql());
Assert.assertEquals("test_keyspace", q.getKeyspace());
Assert.assertEquals("rdonly", q.getTabletType());
Assert.assertEquals(0, q.getBindVars().size());
Assert.assertEquals(null, q.getKeyspaceIds());
String start = Hex.encodeHexString(q.getKeyRanges().get(0).get("Start"));
String end = Hex.encodeHexString(q.getKeyRanges().get(0).get("End"));
shardsInSplits.add(start + "-" + end);
}
// Verify the keyrange queries in splits cover the entire keyspace
Assert.assertTrue(shardsInSplits.containsAll(testEnv.getShardKidMap().keySet()));
}
@Test
public void testSplitQueryMultipleSplitsPerShard() throws Exception {
int rowCount = 30;
Util.insertRows(testEnv, 1, 30);
Map<String, List<BindVariable>> expectedSqls = Maps.newHashMap();
expectedSqls.put("select id, keyspace_id from vtgate_test where id < :_splitquery_end",
Lists.newArrayList(BindVariable.forInt("_splitquery_end", 10)));
expectedSqls.put(
"select id, keyspace_id from vtgate_test where id >= :_splitquery_start "
+ "and id < :_splitquery_end",
Lists.newArrayList(BindVariable.forInt("_splitquery_start", 10),
BindVariable.forInt("_splitquery_end", 19)));
expectedSqls.put("select id, keyspace_id from vtgate_test where id >= :_splitquery_start",
Lists.newArrayList(BindVariable.forInt("_splitquery_start", 19)));
Util.waitForTablet("rdonly", rowCount, 3, testEnv);
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
int splitCount = 6;
Map<Query, Long> queries =
vtgate.splitQuery("test_keyspace", "select id,keyspace_id from vtgate_test", splitCount, "");
vtgate.close();
// Verify 6 splits, 3 per shard
Assert.assertEquals(splitCount, queries.size());
Set<String> shardsInSplits = new HashSet<>();
for (Query q : queries.keySet()) {
String sql = q.getSql();
List<BindVariable> bindVars = expectedSqls.get(sql);
Assert.assertNotNull(bindVars);
Assert.assertEquals("test_keyspace", q.getKeyspace());
Assert.assertEquals("rdonly", q.getTabletType());
Assert.assertEquals(bindVars.size(), q.getBindVars().size());
Assert.assertEquals(null, q.getKeyspaceIds());
String start = Hex.encodeHexString(q.getKeyRanges().get(0).get("Start"));
String end = Hex.encodeHexString(q.getKeyRanges().get(0).get("End"));
shardsInSplits.add(start + "-" + end);
}
// Verify the keyrange queries in splits cover the entire keyspace
Assert.assertTrue(shardsInSplits.containsAll(testEnv.getShardKidMap().keySet()));
}
@Test
public void testSplitQueryInvalidTable() throws Exception {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
try {
vtgate.splitQuery("test_keyspace", "select id from invalid_table", 1, "");
Assert.fail("failed to raise connection exception");
} catch (ConnectionException e) {
Assert.assertTrue(
e.getMessage().contains("query validation error: can't find table in schema"));
} finally {
vtgate.close();
}
}
/**
* Create env with two shards each having a master and replica
*/
static TestEnv getTestEnv() {
Map<String, List<String>> shardKidMap = new LinkedHashMap<>();
shardKidMap.put("80-",
Lists.newArrayList("9767889778372766922", "9742070682920810358", "10296850775085416642"));
shardKidMap.put("-80",
Lists.newArrayList("527875958493693904", "626750931627689502", "345387386794260318"));
TestEnv env = Util.getTestEnv(shardKidMap, "test_keyspace");
env.addTablet("rdonly", 1);
return env;
}
}

251
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/hadoop/MapReduceIT.java
Просмотреть файл

@ -1,251 +0,0 @@
package com.youtube.vitess.vtgate.integration.hadoop;
import com.google.common.collect.Lists;
import com.google.common.primitives.UnsignedLong;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.youtube.vitess.vtgate.Exceptions.InvalidFieldException;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.hadoop.VitessInputFormat;
import com.youtube.vitess.vtgate.hadoop.writables.KeyspaceIdWritable;
import com.youtube.vitess.vtgate.hadoop.writables.RowWritable;
import com.youtube.vitess.vtgate.TestEnv;
import com.youtube.vitess.vtgate.integration.util.Util;
import com.youtube.vitess.vtgate.hadoop.utils.GsonAdapters;
import junit.extensions.TestSetup;
import junit.framework.TestSuite;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.HadoopTestCase;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MapReduceTestUtil;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Integration tests for MapReductions in Vitess. These tests use an in-process Hadoop cluster via
* {@link HadoopTestCase}. These tests are JUnit3 style because of this dependency. Vitess setup for
* these tests require at least one rdonly instance per shard.
*
*/
public class MapReduceIT extends HadoopTestCase {
public static TestEnv testEnv = getTestEnv();
private static Path outputPath = new Path("/tmp");
public MapReduceIT() throws IOException {
super(HadoopTestCase.LOCAL_MR, HadoopTestCase.LOCAL_FS, 1, 1);
}
@Override
public void setUp() throws Exception {
super.setUp();
Util.createTable(testEnv);
}
/**
* Run a mapper only MR job and verify all the rows in the source table were outputted into HDFS.
*/
public void testDumpTableToHDFS() throws Exception {
// Insert 20 rows per shard
int rowsPerShard = 20;
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, rowsPerShard);
}
Util.waitForTablet("rdonly", 40, 3, testEnv);
// Configurations for the job, output from mapper as Text
Configuration conf = createJobConf();
Job job = new Job(conf);
job.setJobName("table");
job.setJarByClass(VitessInputFormat.class);
job.setMapperClass(TableMapper.class);
VitessInputFormat.setInput(job,
"localhost:" + testEnv.getPort(),
testEnv.getKeyspace(),
"select keyspace_id, name from vtgate_test",
4,
testEnv.getRpcClientFactory().getClass());
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(RowWritable.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setNumReduceTasks(0);
Path outDir = new Path(outputPath, "mrvitess/output");
FileSystem fs = FileSystem.get(conf);
if (fs.exists(outDir)) {
fs.delete(outDir, true);
}
FileOutputFormat.setOutputPath(job, outDir);
job.waitForCompletion(true);
assertTrue(job.isSuccessful());
String[] outputLines = MapReduceTestUtil.readOutput(outDir, conf).split("\n");
// there should be one line per row in the source table
assertEquals(rowsPerShard * 2, outputLines.length);
Set<String> actualKids = new HashSet<>();
Set<String> actualNames = new HashSet<>();
// Rows are keyspace ids are written as JSON since this is
// TextOutputFormat. Parse and verify we've gotten all the keyspace
// ids and rows.
Gson gson = new GsonBuilder()
.registerTypeHierarchyAdapter(byte[].class, GsonAdapters.BYTE_ARRAY)
.registerTypeAdapter(UnsignedLong.class, GsonAdapters.UNSIGNED_LONG)
.registerTypeAdapter(Class.class, GsonAdapters.CLASS)
.create();
for (String line : outputLines) {
String kidJson = line.split("\t")[0];
Map<String, String> m = new HashMap<>();
m = gson.fromJson(kidJson, m.getClass());
actualKids.add(m.get("id"));
String rowJson = line.split("\t")[1];
Map<String, Map<String, Map<String, String>>> map = new HashMap<>();
map = gson.fromJson(rowJson, map.getClass());
actualNames.add(
new String(Base64.decodeBase64(map.get("contents").get("name").get("value"))));
}
Set<String> expectedKids = new HashSet<>();
for (KeyspaceId kid : testEnv.getAllKeyspaceIds()) {
expectedKids.add(((UnsignedLong) kid.getId()).toString());
}
assertEquals(expectedKids.size(), actualKids.size());
assertTrue(actualKids.containsAll(expectedKids));
Set<String> expectedNames = new HashSet<>();
for (int i = 1; i <= rowsPerShard; i++) {
expectedNames.add("name_" + i);
}
assertEquals(rowsPerShard, actualNames.size());
assertTrue(actualNames.containsAll(expectedNames));
}
/**
* Map all rows and aggregate by keyspace id at the reducer.
*/
public void testReducerAggregateRows() throws Exception {
int rowsPerShard = 20;
for (String shardName : testEnv.getShardKidMap().keySet()) {
Util.insertRowsInShard(testEnv, shardName, rowsPerShard);
}
Util.waitForTablet("rdonly", 40, 3, testEnv);
Configuration conf = createJobConf();
Job job = new Job(conf);
job.setJobName("table");
job.setJarByClass(VitessInputFormat.class);
job.setMapperClass(TableMapper.class);
VitessInputFormat.setInput(job,
"localhost:" + testEnv.getPort(),
testEnv.getKeyspace(),
"select keyspace_id, name from vtgate_test",
1,
testEnv.getRpcClientFactory().getClass());
job.setMapOutputKeyClass(KeyspaceIdWritable.class);
job.setMapOutputValueClass(RowWritable.class);
job.setReducerClass(CountReducer.class);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(LongWritable.class);
job.setOutputFormatClass(TextOutputFormat.class);
Path outDir = new Path(outputPath, "mrvitess/output");
FileSystem fs = FileSystem.get(conf);
if (fs.exists(outDir)) {
fs.delete(outDir, true);
}
FileOutputFormat.setOutputPath(job, outDir);
job.waitForCompletion(true);
assertTrue(job.isSuccessful());
String[] outputLines = MapReduceTestUtil.readOutput(outDir, conf).split("\n");
assertEquals(6, outputLines.length);
int totalRowsReduced = 0;
for (String line : outputLines) {
totalRowsReduced += Integer.parseInt(line);
}
assertEquals(rowsPerShard * 2, totalRowsReduced);
}
public static class TableMapper extends
Mapper<NullWritable, RowWritable, KeyspaceIdWritable, RowWritable> {
@Override
public void map(NullWritable key, RowWritable value, Context context) throws IOException,
InterruptedException {
try {
KeyspaceId id = new KeyspaceId();
id.setId(value.getRow().getULong("keyspace_id"));
context.write(new KeyspaceIdWritable(id), value);
} catch (InvalidFieldException e) {
throw new IOException(e);
}
}
}
public static class CountReducer extends
Reducer<KeyspaceIdWritable, RowWritable, NullWritable, LongWritable> {
@Override
public void reduce(KeyspaceIdWritable key, Iterable<RowWritable> values, Context context)
throws IOException, InterruptedException {
long count = 0;
Iterator<RowWritable> iter = values.iterator();
while (iter.hasNext()) {
count++;
iter.next();
}
context.write(NullWritable.get(), new LongWritable(count));
}
}
/**
* Create env with two shards each having a master and rdonly
*/
static TestEnv getTestEnv() {
Map<String, List<String>> shardKidMap = new HashMap<>();
shardKidMap.put("-80",
Lists.newArrayList("527875958493693904", "626750931627689502", "345387386794260318"));
shardKidMap.put("80-",
Lists.newArrayList("9767889778372766922", "9742070682920810358", "10296850775085416642"));
TestEnv env = Util.getTestEnv(shardKidMap, "test_keyspace");
env.addTablet("rdonly", 1);
return env;
}
public static TestSetup suite() {
return new TestSetup(new TestSuite(MapReduceIT.class)) {
@Override
protected void setUp() throws Exception {
Util.setupTestEnv(testEnv);
}
@Override
protected void tearDown() throws Exception {
Util.teardownTestEnv(testEnv);
}
};
}
}

188
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/integration/util/Util.java
Просмотреть файл

@ -1,188 +0,0 @@
package com.youtube.vitess.vtgate.integration.util;
import com.google.common.primitives.UnsignedLong;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import com.youtube.vitess.vtgate.BindVariable;
import com.youtube.vitess.vtgate.Exceptions.ConnectionException;
import com.youtube.vitess.vtgate.Exceptions.DatabaseException;
import com.youtube.vitess.vtgate.KeyspaceId;
import com.youtube.vitess.vtgate.Query;
import com.youtube.vitess.vtgate.Query.QueryBuilder;
import com.youtube.vitess.vtgate.TestEnv;
import com.youtube.vitess.vtgate.VtGate;
import com.youtube.vitess.vtgate.cursor.Cursor;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.junit.Assert;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
public class Util {
static final Logger logger = LogManager.getLogger(Util.class.getName());
public static final String PROPERTY_KEY_VTGATE_TEST_ENV = "vtgate.test.env";
/**
* Setup MySQL, Vttablet and VtGate instances required for the tests. This uses a Python helper
* script to start and stop instances.
*/
public static void setupTestEnv(TestEnv testEnv) throws Exception {
ProcessBuilder pb = new ProcessBuilder(testEnv.getSetupCommand());
pb.redirectErrorStream(true);
Process p = pb.start();
BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));
// The port for VtGate is dynamically assigned and written to
// stdout as a JSON string.
String line;
while ((line = br.readLine()) != null) {
logger.info("java_vtgate_test_helper: " + line);
if (!line.startsWith("{")) {
continue;
}
try {
Type mapType = new TypeToken<Map<String, Integer>>() {}.getType();
Map<String, Integer> map = new Gson().fromJson(line, mapType);
testEnv.setPythonScriptProcess(p);
testEnv.setPort(map.get("port"));
return;
} catch (JsonSyntaxException e) {
logger.error("JsonSyntaxException parsing setup command output: " + line, e);
}
}
Assert.fail("setup script failed to parse vtgate port");
}
/**
* Teardown the test instances, if any.
*/
public static void teardownTestEnv(TestEnv testEnv) throws Exception {
Process process = testEnv.getPythonScriptProcess();
if (process != null) {
logger.info("sending empty line to java_vtgate_test_helper to stop test setup");
process.getOutputStream().write("\n".getBytes());
process.getOutputStream().flush();
process.waitFor();
testEnv.setPythonScriptProcess(null);
}
}
public static void insertRows(TestEnv testEnv, int startId, int count) throws ConnectionException,
DatabaseException {
insertRows(testEnv, startId, count, new DateTime());
}
public static void insertRows(TestEnv testEnv, int startId, int count, DateTime dateTime)
throws ConnectionException, DatabaseException {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
String insertSql = "insert into vtgate_test "
+ "(id, name, age, percent, datetime_col, timestamp_col, date_col, time_col, keyspace_id) "
+ "values (:id, :name, :age, :percent, :datetime_col, :timestamp_col, :date_col, :time_col, :keyspace_id)";
for (int i = startId; i < startId + count; i++) {
KeyspaceId kid = testEnv.getAllKeyspaceIds().get(i % testEnv.getAllKeyspaceIds().size());
Query query = new QueryBuilder(insertSql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forULong("id", UnsignedLong.valueOf("" + i)))
.addBindVar(BindVariable.forBytes("name", ("name_" + i).getBytes()))
.addBindVar(BindVariable.forInt("age", i * 2))
.addBindVar(BindVariable.forDouble("percent", new Double(i / 100.0)))
.addBindVar(BindVariable.forULong("keyspace_id", (UnsignedLong) kid.getId()))
.addBindVar(BindVariable.forDateTime("datetime_col", dateTime))
.addBindVar(BindVariable.forDateTime("timestamp_col", dateTime))
.addBindVar(BindVariable.forDate("date_col", dateTime))
.addBindVar(BindVariable.forTime("time_col", dateTime))
.addKeyspaceId(kid)
.build();
vtgate.execute(query);
}
vtgate.commit();
vtgate.close();
}
/**
* Insert rows to a specific shard using ExecuteKeyspaceIds
*/
public static void insertRowsInShard(TestEnv testEnv, String shardName, int count)
throws DatabaseException, ConnectionException {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
String sql = "insert into vtgate_test " + "(id, name, keyspace_id) "
+ "values (:id, :name, :keyspace_id)";
List<KeyspaceId> kids = testEnv.getKeyspaceIds(shardName);
for (int i = 1; i <= count; i++) {
KeyspaceId kid = kids.get(i % kids.size());
Query query = new QueryBuilder(sql, testEnv.getKeyspace(), "master")
.addBindVar(BindVariable.forULong("id", UnsignedLong.valueOf("" + i)))
.addBindVar(BindVariable.forBytes("name", ("name_" + i).getBytes()))
.addBindVar(BindVariable.forULong("keyspace_id", (UnsignedLong) kid.getId()))
.addKeyspaceId(kid)
.build();
vtgate.execute(query);
}
vtgate.commit();
vtgate.close();
}
public static void createTable(TestEnv testEnv) throws Exception {
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
vtgate.begin();
vtgate.execute(new QueryBuilder("drop table if exists vtgate_test", testEnv.getKeyspace(), "master")
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).build());
String createTable = "create table vtgate_test (id bigint auto_increment,"
+ " name varchar(64), age SMALLINT, percent DECIMAL(5,2),"
+ " keyspace_id bigint(20) unsigned NOT NULL, datetime_col DATETIME,"
+ " timestamp_col TIMESTAMP, date_col DATE, time_col TIME, primary key (id))"
+ " Engine=InnoDB";
vtgate.execute(new QueryBuilder(createTable, testEnv.getKeyspace(), "master").setKeyspaceIds(
testEnv.getAllKeyspaceIds()).build());
vtgate.commit();
vtgate.close();
}
/**
* Wait until the specified tablet type has received at least rowCount rows in vtgate_test from
* the master. If the criteria isn't met after the specified number of attempts raise an
* exception.
*/
public static void waitForTablet(String tabletType, int rowCount, int attempts, TestEnv testEnv)
throws Exception {
String sql = "select * from vtgate_test";
VtGate vtgate = VtGate.connect("localhost:" + testEnv.getPort(), 0, testEnv.getRpcClientFactory());
for (int i = 0; i < attempts; i++) {
try {
Cursor cursor = vtgate.execute(new QueryBuilder(sql, testEnv.getKeyspace(), tabletType)
.setKeyspaceIds(testEnv.getAllKeyspaceIds()).build());
if (cursor.getRowsAffected() >= rowCount) {
vtgate.close();
return;
}
} catch (DatabaseException e) {
}
Thread.sleep(1000);
}
vtgate.close();
throw new Exception(tabletType + " fails to catch up");
}
public static TestEnv getTestEnv(Map<String, List<String>> shardKidMap, String keyspace) {
String testEnvClass = System.getProperty(PROPERTY_KEY_VTGATE_TEST_ENV);
try {
Class<?> clazz = Class.forName(testEnvClass);
TestEnv env = (TestEnv)clazz.newInstance();
env.setKeyspace(keyspace);
env.setShardKidMap(shardKidMap);
return env;
} catch (ClassNotFoundException|IllegalAccessException|InstantiationException e) {
throw new RuntimeException(e);
}
}
}

105
java/vtgate-client/src/test/java/com/youtube/vitess/vtgate/rpcclient/gorpc/BsonifyTest.java
Просмотреть файл

@ -1,105 +0,0 @@
package com.youtube.vitess.vtgate.rpcclient.gorpc;
import com.google.common.primitives.UnsignedLong;
import com.youtube.vitess.vtgate.Exceptions.InvalidFieldException;
import com.youtube.vitess.vtgate.Field.Flag;
import com.youtube.vitess.vtgate.QueryResult;
import com.youtube.vitess.vtgate.Row;
import com.youtube.vitess.vtgate.Row.Cell;
import com.youtube.vitess.vtgate.cursor.Cursor;
import com.youtube.vitess.vtgate.cursor.CursorImpl;
import com.youtube.vitess.vtgate.FieldType;
import org.bson.BSONObject;
import org.bson.BasicBSONObject;
import org.bson.types.BasicBSONList;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.math.BigDecimal;
@RunWith(JUnit4.class)
public class BsonifyTest {
@Test
public void testResultParse() throws InvalidFieldException {
BSONObject result = new BasicBSONObject();
result.put("RowsAffected", 12L);
result.put("InsertId", 12345L);
BasicBSONList fields = new BasicBSONList();
fields.add(newField("col_0", FieldType.VT_DECIMAL, Flag.VT_ZEROVALUE_FLAG));
fields.add(newField("col_1", FieldType.VT_TINY, Flag.VT_ZEROVALUE_FLAG));
fields.add(newField("col_2", FieldType.VT_SHORT, Flag.VT_ZEROVALUE_FLAG));
fields.add(newField("col_3", FieldType.VT_LONG, Flag.VT_ZEROVALUE_FLAG));
fields.add(newField("col_4", FieldType.VT_LONGLONG, Flag.VT_ZEROVALUE_FLAG));
fields.add(newField("col_5", FieldType.VT_LONGLONG, Flag.VT_UNSIGNED_FLAG));
result.put("Fields", fields);
// Fill each column with the following different values: 0, 1, 2
BasicBSONList rows = new BasicBSONList();
int rowCount = 2;
for (int i = 0; i <= rowCount; i++) {
BasicBSONList row = new BasicBSONList();
row.add(new Double(i).toString().getBytes());
row.add(String.valueOf(i).getBytes());
row.add(String.valueOf(i).getBytes());
row.add(new Long(i).toString().getBytes());
row.add(new Long(i).toString().getBytes());
row.add(new Long(i).toString().getBytes());
Assert.assertEquals(fields.size(), row.size());
rows.add(row);
}
result.put("Rows", rows);
QueryResult qr = Bsonify.bsonToQueryResult(result, null);
Cursor cursor = new CursorImpl(qr);
Assert.assertEquals(12L, cursor.getRowsAffected());
Assert.assertEquals(12345L, cursor.getLastRowId());
for (int i = 0; i <= rowCount; i++) {
Row row = cursor.next();
Cell cell0 = row.next();
Assert.assertEquals("col_0", cell0.getName());
Assert.assertEquals(BigDecimal.class, cell0.getType());
Assert.assertEquals(new BigDecimal(String.format("%d.0", i)), row.getBigDecimal(cell0.getName()));
Cell cell1 = row.next();
Assert.assertEquals("col_1", cell1.getName());
Assert.assertEquals(Integer.class, cell1.getType());
Assert.assertEquals(new Integer(i), row.getInt(cell1.getName()));
Cell cell2 = row.next();
Assert.assertEquals("col_2", cell2.getName());
Assert.assertEquals(Integer.class, cell2.getType());
Assert.assertEquals(new Integer(i), row.getInt(cell2.getName()));
Cell cell3 = row.next();
Assert.assertEquals("col_3", cell3.getName());
Assert.assertEquals(Long.class, cell3.getType());
Assert.assertEquals(new Long(i), row.getLong(cell3.getName()));
Cell cell4 = row.next();
Assert.assertEquals("col_4", cell4.getName());
Assert.assertEquals(Long.class, cell4.getType());
Assert.assertEquals(new Long(i), row.getLong(cell4.getName()));
Cell cell5 = row.next();
Assert.assertEquals("col_5", cell5.getName());
Assert.assertEquals(UnsignedLong.class, cell5.getType());
Assert.assertEquals(UnsignedLong.valueOf(String.format("%d", i)), row.getULong(cell5.getName()));
}
// No more rows left.
Assert.assertFalse(cursor.hasNext());
}
private BSONObject newField(String name, FieldType fieldType, Flag flag) {
BSONObject field = new BasicBSONObject();
field.put("Name", name.getBytes());
field.put("Type", (long) fieldType.mysqlType);
field.put("Flags", (long) flag.mysqlFlag);
return field;
}
}

6
java/vtgate-client/src/test/resources/log4j.properties
Просмотреть файл

@ -1,6 +0,0 @@
log4j.rootLogger=INFO, consoleAppender
log4j.appender.consoleAppender=org.apache.log4j.ConsoleAppender
log4j.appender.consoleAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.consoleAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
OFF

24
php/README.md
Просмотреть файл

@ -6,14 +6,27 @@ See `demo.php` for a simple example of using the API.
## Prerequisites
PHP 5.3+ is required.
PHP 5.5+ is required.
This library uses the BSON codec library contained within the MongoDB plugin.
For example, you can install this on Debian/Ubuntu like this:
### gRPC Extension Module
Install the [gRPC extension module](https://pecl.php.net/package/gRPC).
For example, on Debian/Ubuntu:
``` sh
$ sudo apt-get install php5-dev php5-cli php-pear
$ sudo pecl install mongo
$ sudo pecl install grpc-beta
```
### gRPC Dependencies
To download the dependencies of the gRPC PHP library, run Composer:
``` sh
$ cd vitess/php
vitess/php$ curl -sS https://getcomposer.org/installer | php
vitess/php$ php composer.phar install
```
## Unit Tests
@ -29,7 +42,8 @@ $ chmod +x $VTROOT/bin/phpunit
Then run the tests like this:
``` sh
vitess/php$ phpunit tests
vitess$ . dev.env
vitess$ make php_test
```
### Coverage

20
php/demo.php
Просмотреть файл

@ -4,13 +4,13 @@
* This is a sample for using the PHP Vitess client.
*
* Before running this, start up a local demo cluster by running:
* vitess$ test/demo.py
* vitess$ examples/demo/run.py
*
* The demo.py script will print the vtgate port, which you pass in like this:
* vitess/php$ php demo.php --server localhost:port
* Then in another terminal:
* vitess/php$ php demo.php --server=localhost:12346
*/
require_once ('./src/VTGateConn.php');
require_once ('./src/BsonRpcClient.php');
require_once ('./src/VTGrpcClient.php');
$opts = getopt('', array(
'server:'
@ -18,9 +18,7 @@ $opts = getopt('', array(
// Create a connection.
$ctx = VTContext::getDefault();
$client = new BsonRpcClient();
$client->dial($ctx, $opts['server']);
$conn = new VTGateConn($client);
$conn = new VTGateConn(new VTGrpcClient($opts['server']));
// Insert something.
$tx = $conn->begin($ctx);
@ -30,5 +28,9 @@ $tx->execute($ctx, 'INSERT INTO user (name) VALUES (:name)', array(
$tx->commit($ctx);
// Read it back.
$result = $conn->execute($ctx, 'SELECT * FROM user', array(), VTTabletType::MASTER);
print_r($result->rows);
$cursor = $conn->execute($ctx, 'SELECT * FROM user', array(), \topodata\TabletType::MASTER);
while (($row = $cursor->next()) !== FALSE) {
print_r($row);
}
$conn->close();

46
php/src/BsonRpcClient.php
Просмотреть файл

@ -1,46 +0,0 @@
<?php
/*
* NOTE: This module requires bson_encode() and bson_decode(),
* which can be obtained by installing the MongoDB driver.
* For example, on Debian/Ubuntu:
* $ sudo apt-get install php5-dev php5-cli php-pear
* $ sudo pecl install mongo
*/
ini_set('mongo.native_long', 1);
require_once (dirname(__FILE__) . '/GoRpcClient.php');
class BsonRpcClient extends GoRpcClient {
const LEN_PACK_FORMAT = 'V';
const LEN_PACK_SIZE = 4;
public function dial(VTContext $ctx, $addr) {
parent::dial($ctx, "tcp://$addr", '/_bson_rpc_');
}
protected function sendRequest(VTContext $ctx, GoRpcRequest $req) {
$this->write($ctx, bson_encode($req->header));
if ($req->body === NULL)
$this->write($ctx, bson_encode(array()));
else
$this->write($ctx, bson_encode($req->body));
}
protected function readResponse(VTContext $ctx) {
// Read the header.
$data = $this->readN($ctx, self::LEN_PACK_SIZE);
$unpack = unpack(self::LEN_PACK_FORMAT, $data);
$len = $unpack[1];
$header = $data . $this->readN($ctx, $len - self::LEN_PACK_SIZE);
// Read the body.
$data = $this->readN($ctx, self::LEN_PACK_SIZE);
$unpack = unpack(self::LEN_PACK_FORMAT, $data);
$len = $unpack[1];
$body = $data . $this->readN($ctx, $len - self::LEN_PACK_SIZE);
// Decode and return.
return new GoRpcResponse(bson_decode($header), bson_decode($body));
}
}

167
php/src/GoRpcClient.php
Просмотреть файл

@ -1,167 +0,0 @@
<?php
/*
* This is an implementation of a Vitess-compatible Go-style RPC layer for PHP.
*/
// TODO(enisoc): Implement timeouts.
class GoRpcException extends Exception {
}
class GoRpcRemoteError extends GoRpcException {
}
class GoRpcRequest {
public $header;
public $body;
public function __construct($seq, $method, $body) {
$this->header = array(
'ServiceMethod' => $method,
'Seq' => $seq
);
$this->body = $body;
}
public function seq() {
return $this->header['Seq'];
}
}
function unwrap_bin_data($value) {
if (is_object($value) && get_class($value) == 'MongoBinData')
return $value->bin;
if (is_array($value)) {
foreach ($value as $key => $child)
$value[$key] = unwrap_bin_data($child);
}
return $value;
}
class GoRpcResponse {
public $header;
public $reply;
public function __construct($header, $body) {
$this->header = unwrap_bin_data($header);
$this->reply = unwrap_bin_data($body);
}
public function error() {
return $this->header['Error'];
}
public function seq() {
return $this->header['Seq'];
}
public function isEOS() {
return $this->error() == self::END_OF_STREAM;
}
const END_OF_STREAM = 'EOS';
}
abstract class GoRpcClient {
protected $seq = 0;
protected $stream = NULL;
abstract protected function sendRequest(VTContext $ctx, GoRpcRequest $req);
abstract protected function readResponse(VTContext $ctx);
private $in_stream_call = FALSE;
public function dial(VTContext $ctx, $addr, $path) {
// Connect to $addr.
$fp = stream_socket_client($addr, $errno, $errstr);
if ($fp === FALSE)
throw new GoRpcException("can't connect to $addr: $errstr ($errno)");
$this->stream = $fp;
// Initiate request for $path.
$this->write($ctx, "CONNECT $path HTTP/1.0\n\n");
// Read until handshake is completed.
$data = '';
while (strpos($data, "\n\n") === FALSE)
$data .= $this->read($ctx, 1024);
}
public function close() {
if ($this->stream !== NULL) {
fclose($this->stream);
$this->stream = NULL;
}
}
public function call(VTContext $ctx, $method, $request) {
if ($this->in_stream_call) {
throw new Exception("GoRpcClient: can't make another call until results of streaming call are completely fetched.");
}
$req = new GoRpcRequest($this->nextSeq(), $method, $request);
$this->sendRequest($ctx, $req);
$resp = $this->readResponse($ctx);
if ($resp->seq() != $req->seq())
throw new GoRpcException("$method: request sequence mismatch");
if ($resp->error())
throw new GoRpcRemoteError("$method: " . $resp->error());
return $resp;
}
public function streamCall(VTContext $ctx, $method, $request) {
if ($this->in_stream_call) {
throw new Exception("GoRpcClient: can't make another call until results of streaming call are completely fetched.");
}
$req = new GoRpcRequest($this->nextSeq(), $method, $request);
$this->sendRequest($ctx, $req);
$this->in_stream_call = TRUE;
}
public function streamNext(VTContext $ctx) {
if (! $this->in_stream_call) {
throw new Exception("GoRpcClient: streamNext called when not in streaming call.");
}
$resp = $this->readResponse($ctx);
if ($resp->seq() != $this->seq) {
throw new GoRpcException("$method: request sequence mismatch");
}
if ($resp->isEOS()) {
$this->in_stream_call = FALSE;
return FALSE;
}
if ($resp->error()) {
throw new GoRpcRemoteError("$method: " . $resp->error());
}
return $resp;
}
protected function read(VTContext $ctx, $max_len) {
if (feof($this->stream))
throw new GoRpcException("unexpected EOF while reading from stream");
$packet = fread($this->stream, $max_len);
if ($packet === FALSE)
throw new GoRpcException("can't read from stream");
return $packet;
}
protected function readN(VTContext $ctx, $target_len) {
// Read exactly $target_len bytes or bust.
$data = '';
while (($len = strlen($data)) < $target_len)
$data .= $this->read($ctx, $target_len - $len);
return $data;
}
protected function write(VTContext $ctx, $data) {
if (fwrite($this->stream, $data) === FALSE)
throw new GoRpcException("can't write to stream");
}
protected function nextSeq() {
return ++ $this->seq;
}
}

6
php/src/VTContext.php
Просмотреть файл

@ -1,13 +1,11 @@
<?php
class VTDeadlineExceededException extends Exception {
}
require_once (dirname(__FILE__) . '/VTException.php');
/**
* VTContext is an immutable object carrying request-specific deadlines and
* credentials (callerId).
*
* Example usage:
* <p>Example usage:
* <pre>
* $ctx = VTContext::getDefault()->withDeadlineAfter(0.5); // 500ms timeout
* $conn->doSomething($ctx, ...);

103
php/src/VTCursor.php Normal file
Просмотреть файл

@ -0,0 +1,103 @@
<?php
require_once (dirname(__FILE__) . '/VTProto.php');
require_once (dirname(__FILE__) . '/VTException.php');
class VTCursor {
private $queryResult;
private $pos = - 1;
private $rows;
public function __construct(\query\QueryResult $query_result) {
$this->queryResult = $query_result;
if ($query_result->hasRows()) {
$this->rows = $query_result->getRowsList();
}
}
public function getRowsAffected() {
return $this->queryResult->getRowsAffected();
}
public function getInsertId() {
return $this->queryResult->getInsertId();
}
public function getFields() {
return $this->queryResult->getFieldsList();
}
public function close() {
}
public function next() {
if ($this->rows && ++ $this->pos < count($this->rows)) {
return $this->rows[$this->pos]->getValuesList();
} else {
return FALSE;
}
}
}
class VTStreamCursor {
private $pos = - 1;
private $rows;
private $fields;
private $stream;
public function __construct($stream) {
$this->stream = $stream;
}
public function getFields() {
if ($this->fields === NULL) {
// The first QueryResult should have the fields.
if (! $this->nextQueryResult()) {
throw new VTException("stream ended before fields were received");
}
}
return $this->fields;
}
public function close() {
if ($this->stream) {
$this->stream->close();
$this->stream = NULL;
}
}
public function next() {
// Get the next row from the current QueryResult.
if ($this->rows && ++ $this->pos < count($this->rows)) {
return $this->rows[$this->pos]->getValuesList();
}
// Get the next QueryResult. Loop in case we get a QueryResult with no rows (e.g. only fields).
while ($this->nextQueryResult()) {
// Get the first row from the new QueryResult.
if ($this->rows && ++ $this->pos < count($this->rows)) {
return $this->rows[$this->pos]->getValuesList();
}
}
// No more rows and no more QueryResults.
return FALSE;
}
private function nextQueryResult() {
if (($response = $this->stream->next()) !== FALSE) {
$query_result = $response->getResult();
$this->rows = $query_result->getRowsList();
$this->pos = - 1;
if ($this->fields === NULL) {
// The first QueryResult should have the fields.
$this->fields = $query_result->getFieldsList();
}
return TRUE;
} else {
// No more QueryResults.
$this->rows = NULL;
return FALSE;
}
}
}

20
php/src/VTException.php Normal file
Просмотреть файл

@ -0,0 +1,20 @@
<?php
class VTException extends Exception {
}
class VTBadInputError extends VTException {
}
class VTIntegrityError extends VTException {
}
class VTTransientError extends VTException {
}
class VTDeadlineExceededError extends VTTransientError {
}
class VTUnauthenticatedError extends VTException {
}

454
php/src/VTGateConn.php
Просмотреть файл

@ -1,137 +1,8 @@
<?php
require_once (dirname(__FILE__) . '/VTProto.php');
require_once (dirname(__FILE__) . '/VTContext.php');
class VTGateTx {
protected $client;
protected $session;
public function __construct($client, $session) {
$this->client = $client;
$this->session = $session;
}
private function callExecute(VTContext $ctx, $query, array $bind_vars, $tablet_type, $not_in_transaction, $method, $req = array()) {
if (is_null($this->session)) {
throw new Exception('execute called while not in transaction.');
}
$req['Session'] = $this->session;
$req['Query'] = VTBoundQuery::buildBsonP3($query, $bind_vars);
$req['TabletType'] = $tablet_type;
$req['NotInTransaction'] = $not_in_transaction;
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$resp = $this->client->call($ctx, $method, $req)->reply;
if (array_key_exists('Session', $resp) && $resp['Session']) {
$this->session = $resp['Session'];
} else {
$this->session = NULL;
}
VTProto::checkError($resp);
return VTQueryResult::fromBsonP3($resp['Result']);
}
private function callExecuteBatch(VTContext $ctx, $queries, $tablet_type, $as_transaction, $method, $req = array()) {
if (is_null($this->session)) {
throw new Exception('execute called while not in transaction.');
}
$req['Session'] = $this->session;
$req['Queries'] = $queries;
$req['TabletType'] = $tablet_type;
$req['AsTransaction'] = $as_transaction;
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$resp = $this->client->call($ctx, $method, $req)->reply;
if (array_key_exists('Session', $resp) && $resp['Session']) {
$this->session = $resp['Session'];
} else {
$this->session = NULL;
}
VTProto::checkError($resp);
$results = array();
foreach ($resp['Results'] as $result) {
$results[] = VTQueryResult::fromBsonP3($result);
}
return $results;
}
public function execute(VTContext $ctx, $query, array $bind_vars, $tablet_type = VTTabletType::MASTER, $not_in_transaction = FALSE) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, $not_in_transaction, 'VTGateP3.Execute');
}
public function executeShards(VTContext $ctx, $query, $keyspace, array $shards, array $bind_vars, $tablet_type = VTTabletType::MASTER, $not_in_transaction = FALSE) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, $not_in_transaction, 'VTGateP3.ExecuteShards', array(
'Keyspace' => $keyspace,
'Shards' => $shards
));
}
public function executeKeyspaceIds(VTContext $ctx, $query, $keyspace, array $keyspace_ids, array $bind_vars, $tablet_type = VTTabletType::MASTER, $not_in_transaction = FALSE) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, $not_in_transaction, 'VTGateP3.ExecuteKeyspaceIds', array(
'Keyspace' => $keyspace,
'KeyspaceIds' => VTKeyspaceId::buildBsonP3Array($keyspace_ids)
));
}
public function executeKeyRanges(VTContext $ctx, $query, $keyspace, array $key_ranges, array $bind_vars, $tablet_type = VTTabletType::MASTER, $not_in_transaction = FALSE) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, $not_in_transaction, 'VTGateP3.ExecuteKeyRanges', array(
'Keyspace' => $keyspace,
'KeyRanges' => VTKeyRange::buildBsonP3Array($key_ranges)
));
}
public function executeEntityIds(VTContext $ctx, $query, $keyspace, $entity_column_name, array $entity_keyspace_ids, array $bind_vars, $tablet_type = VTTabletType::MASTER, $not_in_transaction = FALSE) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, $not_in_transaction, 'VTGateP3.ExecuteEntityIds', array(
'Keyspace' => $keyspace,
'EntityColumnName' => $entity_column_name,
'EntityKeyspaceIds' => VTEntityId::buildBsonP3Array($entity_keyspace_ids)
));
}
public function executeBatchShards(VTContext $ctx, array $bound_shard_queries, $tablet_type = VTTabletType::MASTER, $as_transaction = TRUE) {
return $this->callExecuteBatch($ctx, VTBoundShardQuery::buildBsonP3Array($bound_shard_queries), $tablet_type, $as_transaction, 'VTGateP3.ExecuteBatchShards');
}
public function executeBatchKeyspaceIds(VTContext $ctx, array $bound_keyspace_id_queries, $tablet_type = VTTabletType::MASTER, $as_transaction = TRUE) {
return $this->callExecuteBatch($ctx, VTBoundKeyspaceIdQuery::buildBsonP3Array($bound_keyspace_id_queries), $tablet_type, $as_transaction, 'VTGateP3.ExecuteBatchKeyspaceIds');
}
public function commit(VTContext $ctx) {
if (is_null($this->session)) {
throw new Exception('commit called while not in transaction.');
}
$req = array(
'Session' => $this->session
);
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$resp = $this->client->call($ctx, 'VTGateP3.Commit', $req)->reply;
$this->session = NULL;
}
public function rollback(VTContext $ctx) {
if (is_null($this->session)) {
throw new Exception('rollback called while not in transaction.');
}
$req = array(
'Session' => $this->session
);
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$resp = $this->client->call($ctx, 'VTGateP3.Rollback', $req)->reply;
$this->session = NULL;
}
}
require_once (dirname(__FILE__) . '/VTCursor.php');
require_once (dirname(__FILE__) . '/VTGateTx.php');
class VTGateConn {
protected $client;
@ -140,200 +11,191 @@ class VTGateConn {
$this->client = $client;
}
private function callExecute(VTContext $ctx, $query, array $bind_vars, $tablet_type, $method, $req = array()) {
$req['Query'] = VTBoundQuery::buildBsonP3($query, $bind_vars);
$req['TabletType'] = $tablet_type;
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$resp = $this->client->call($ctx, $method, $req)->reply;
VTProto::checkError($resp);
return VTQueryResult::fromBsonP3($resp['Result']);
}
private function callExecuteBatch(VTContext $ctx, $queries, $tablet_type, $as_transaction, $method, $req = array()) {
$req['Queries'] = $queries;
$req['TabletType'] = $tablet_type;
$req['AsTransaction'] = $as_transaction;
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$resp = $this->client->call($ctx, $method, $req)->reply;
VTProto::checkError($resp);
$results = array();
if (array_key_exists('Results', $resp)) {
foreach ($resp['Results'] as $result) {
$results[] = VTQueryResult::fromBsonP3($result);
}
}
return $results;
}
private function callStreamExecute(VTContext $ctx, $query, array $bind_vars, $tablet_type, $method, $req = array()) {
$req['Query'] = VTBoundQuery::buildBsonP3($query, $bind_vars);
$req['TabletType'] = $tablet_type;
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$this->client->streamCall($ctx, $method, $req);
return new VTStreamResults($ctx, $this->client);
}
public function execute(VTContext $ctx, $query, array $bind_vars, $tablet_type) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.Execute');
$request = new \vtgate\ExecuteRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->execute($ctx, $request);
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeShards(VTContext $ctx, $query, $keyspace, array $shards, array $bind_vars, $tablet_type) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.ExecuteShards', array(
'Keyspace' => $keyspace,
'Shards' => $shards
));
$request = new \vtgate\ExecuteShardsRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setKeyspace($keyspace);
$request->setShards($shards);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeShards($ctx, $request);
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeKeyspaceIds(VTContext $ctx, $query, $keyspace, array $keyspace_ids, array $bind_vars, $tablet_type) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.ExecuteKeyspaceIds', array(
'Keyspace' => $keyspace,
'KeyspaceIds' => VTKeyspaceId::buildBsonP3Array($keyspace_ids)
));
$request = new \vtgate\ExecuteKeyspaceIdsRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setKeyspace($keyspace);
$request->setKeyspaceIds($keyspace_ids);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeKeyspaceIds($ctx, $request);
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeKeyRanges(VTContext $ctx, $query, $keyspace, array $key_ranges, array $bind_vars, $tablet_type) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.ExecuteKeyRanges', array(
'Keyspace' => $keyspace,
'KeyRanges' => VTKeyRange::buildBsonP3Array($key_ranges)
));
$request = new \vtgate\ExecuteKeyRangesRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setKeyspace($keyspace);
VTProto::addKeyRanges($request, $key_ranges);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeKeyRanges($ctx, $request);
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeEntityIds(VTContext $ctx, $query, $keyspace, $entity_column_name, array $entity_keyspace_ids, array $bind_vars, $tablet_type) {
return $this->callExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.ExecuteEntityIds', array(
'Keyspace' => $keyspace,
'EntityColumnName' => $entity_column_name,
'EntityKeyspaceIds' => VTEntityId::buildBsonP3Array($entity_keyspace_ids)
));
$request = new \vtgate\ExecuteEntityIdsRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setKeyspace($keyspace);
$request->setEntityColumnName($entity_column_name);
VTProto::addEntityKeyspaceIds($request, $entity_keyspace_ids);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeEntityIds($ctx, $request);
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeBatchShards(VTContext $ctx, array $bound_shard_queries, $tablet_type, $as_transaction) {
return $this->callExecuteBatch($ctx, VTBoundShardQuery::buildBsonP3Array($bound_shard_queries), $tablet_type, $as_transaction, 'VTGateP3.ExecuteBatchShards');
}
public function executeBatchKeyspaceIds(VTContext $ctx, array $bound_keyspace_id_queries, $tablet_type, $as_transaction) {
return $this->callExecuteBatch($ctx, VTBoundKeyspaceIdQuery::buildBsonP3Array($bound_keyspace_id_queries), $tablet_type, $as_transaction, 'VTGateP3.ExecuteBatchKeyspaceIds');
}
public function streamExecute(VTContext $ctx, $query, array $bind_vars, $tablet_type) {
return $this->callStreamExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.StreamExecute');
}
public function streamExecuteShards(VTContext $ctx, $query, $keyspace, array $shards, array $bind_vars, $tablet_type) {
return $this->callStreamExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.StreamExecuteShards', array(
'Keyspace' => $keyspace,
'Shards' => $shards
));
}
public function streamExecuteKeyspaceIds(VTContext $ctx, $query, $keyspace, array $keyspace_ids, array $bind_vars, $tablet_type) {
return $this->callStreamExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.StreamExecuteKeyspaceIds', array(
'Keyspace' => $keyspace,
'KeyspaceIds' => VTKeyspaceId::buildBsonP3Array($keyspace_ids)
));
}
public function streamExecuteKeyRanges(VTContext $ctx, $query, $keyspace, array $key_ranges, array $bind_vars, $tablet_type) {
return $this->callStreamExecute($ctx, $query, $bind_vars, $tablet_type, 'VTGateP3.StreamExecuteKeyRanges', array(
'Keyspace' => $keyspace,
'KeyRanges' => VTKeyRange::buildBsonP3Array($key_ranges)
));
}
public function begin(VTContext $ctx) {
$req = array();
$request = new \vtgate\ExecuteBatchShardsRequest();
VTProto::addQueries($request, $bound_shard_queries);
$request->setTabletType($tablet_type);
$request->setAsTransaction($as_transaction);
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
$request->setCallerId($ctx->getCallerId());
}
$resp = $this->client->call($ctx, 'VTGateP3.Begin', $req)->reply;
return new VTGateTx($this->client, $resp['Session']);
}
public function splitQuery(VTContext $ctx, $keyspace, $query, array $bind_vars, $split_column, $split_count) {
$req = array(
'Keyspace' => $keyspace,
'Query' => VTBoundQuery::buildBsonP3($query, $bind_vars),
'SplitColumn' => $split_column,
'SplitCount' => $split_count
);
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
}
$resp = $this->client->call($ctx, 'VTGateP3.SplitQuery', $req)->reply;
$response = $this->client->executeBatchShards($ctx, $request);
VTProto::checkError($response);
$results = array();
if (array_key_exists('Splits', $resp)) {
foreach ($resp['Splits'] as $split) {
$results[] = VTSplitQueryPart::fromBsonP3($split);
}
foreach ($response->getResultsList() as $result) {
$results[] = new VTCursor($result);
}
return $results;
}
public function getSrvKeyspace(VTContext $ctx, $keyspace) {
$req = array(
'Keyspace' => $keyspace
);
public function executeBatchKeyspaceIds(VTContext $ctx, array $bound_keyspace_id_queries, $tablet_type, $as_transaction) {
$request = new \vtgate\ExecuteBatchKeyspaceIdsRequest();
VTProto::addQueries($request, $bound_keyspace_id_queries);
$request->setTabletType($tablet_type);
$request->setAsTransaction($as_transaction);
if ($ctx->getCallerId()) {
$req['CallerId'] = $ctx->getCallerId()->toBsonP3();
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeBatchKeyspaceIds($ctx, $request);
VTProto::checkError($response);
$results = array();
foreach ($response->getResultsList() as $result) {
$results[] = new VTCursor($result);
}
return $results;
}
$resp = $this->client->call($ctx, 'VTGateP3.GetSrvKeyspace', $req)->reply;
return VTSrvKeyspace::fromBsonP3($resp['SrvKeyspace']);
public function streamExecute(VTContext $ctx, $query, array $bind_vars, $tablet_type) {
$request = new \vtgate\StreamExecuteRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$call = $this->client->streamExecute($ctx, $request);
return new VTStreamCursor($call);
}
public function streamExecuteShards(VTContext $ctx, $query, $keyspace, array $shards, array $bind_vars, $tablet_type) {
$request = new \vtgate\StreamExecuteShardsRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setKeyspace($keyspace);
$request->setShards($shards);
$request->setTabletType($tablet_type);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$call = $this->client->streamExecuteShards($ctx, $request);
return new VTStreamCursor($call);
}
public function streamExecuteKeyspaceIds(VTContext $ctx, $query, $keyspace, array $keyspace_ids, array $bind_vars, $tablet_type) {
$request = new \vtgate\StreamExecuteKeyspaceIdsRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setKeyspace($keyspace);
$request->setKeyspaceIds($keyspace_ids);
$request->setTabletType($tablet_type);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$call = $this->client->streamExecuteKeyspaceIds($ctx, $request);
return new VTStreamCursor($call);
}
public function streamExecuteKeyRanges(VTContext $ctx, $query, $keyspace, array $key_ranges, array $bind_vars, $tablet_type) {
$request = new \vtgate\StreamExecuteKeyRangesRequest();
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setKeyspace($keyspace);
VTProto::addKeyRanges($request, $key_ranges);
$request->setTabletType($tablet_type);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$call = $this->client->streamExecuteKeyRanges($ctx, $request);
return new VTStreamCursor($call);
}
public function begin(VTContext $ctx) {
$request = new \vtgate\BeginRequest();
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->begin($ctx, $request);
return new VTGateTx($this->client, $response->getSession());
}
public function splitQuery(VTContext $ctx, $keyspace, $query, array $bind_vars, $split_column, $split_count) {
$request = new \vtgate\SplitQueryRequest();
$request->setKeyspace($keyspace);
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setSplitColumn($split_column);
$request->setSplitCount($split_count);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->splitQuery($ctx, $request);
return $response->getSplitsList();
}
public function getSrvKeyspace(VTContext $ctx, $keyspace) {
$request = new \vtgate\GetSrvKeyspaceRequest();
$request->setKeyspace($keyspace);
$response = $this->client->getSrvKeyspace($ctx, $request);
return $response->getSrvKeyspace();
}
public function close() {
$this->client->close();
}
}
class VTStreamResults {
private $ctx;
private $client;
public function __construct($ctx, $client) {
$this->ctx = $ctx;
$this->client = $client;
}
/**
* fetch reads and returns the next VTQueryResult from the stream.
*
* If there are no more results and the stream has finished successfully,
* it returns FALSE.
*/
public function fetch() {
$resp = $this->client->streamNext($this->ctx);
if ($resp === FALSE) {
return FALSE;
}
VTProto::checkError($resp->reply);
return VTQueryResult::fromBsonP3($resp->reply['Result']);
}
/**
* fetchAll calls fetch in a loop until it returns FALSE, and then returns the
* results as an array.
*/
public function fetchAll() {
$results = array();
while (($result = $this->fetch()) !== FALSE) {
$results[] = $result;
}
return $results;
}
}

193
php/src/VTGateTx.php Normal file
Просмотреть файл

@ -0,0 +1,193 @@
<?php
class VTGateTx {
protected $client;
protected $session;
public function __construct($client, $session) {
$this->client = $client;
$this->session = $session;
}
private function inTransaction() {
return ! is_null($this->session) && $this->session->getInTransaction();
}
public function execute(VTContext $ctx, $query, array $bind_vars, $tablet_type = \topodata\TabletType::MASTER, $not_in_transaction = FALSE) {
if (! $this->inTransaction()) {
throw new VTException('execute called while not in transaction.');
}
$request = new \vtgate\ExecuteRequest();
$request->setSession($this->session);
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setNotInTransaction($not_in_transaction);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->execute($ctx, $request);
$this->session = $response->getSession();
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeShards(VTContext $ctx, $query, $keyspace, array $shards, array $bind_vars, $tablet_type = \topodata\TabletType::MASTER, $not_in_transaction = FALSE) {
if (! $this->inTransaction()) {
throw new VTException('execute called while not in transaction.');
}
$request = new \vtgate\ExecuteShardsRequest();
$request->setSession($this->session);
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setNotInTransaction($not_in_transaction);
$request->setKeyspace($keyspace);
$request->setShards($shards);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeShards($ctx, $request);
$this->session = $response->getSession();
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeKeyspaceIds(VTContext $ctx, $query, $keyspace, array $keyspace_ids, array $bind_vars, $tablet_type = \topodata\TabletType::MASTER, $not_in_transaction = FALSE) {
if (! $this->inTransaction()) {
throw new VTException('execute called while not in transaction.');
}
$request = new \vtgate\ExecuteKeyspaceIdsRequest();
$request->setSession($this->session);
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setNotInTransaction($not_in_transaction);
$request->setKeyspace($keyspace);
$request->setKeyspaceIds($keyspace_ids);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeKeyspaceIds($ctx, $request);
$this->session = $response->getSession();
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeKeyRanges(VTContext $ctx, $query, $keyspace, array $key_ranges, array $bind_vars, $tablet_type = \topodata\TabletType::MASTER, $not_in_transaction = FALSE) {
if (! $this->inTransaction()) {
throw new VTException('execute called while not in transaction.');
}
$request = new \vtgate\ExecuteKeyRangesRequest();
$request->setSession($this->session);
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setNotInTransaction($not_in_transaction);
$request->setKeyspace($keyspace);
VTProto::addKeyRanges($request, $key_ranges);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeKeyRanges($ctx, $request);
$this->session = $response->getSession();
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeEntityIds(VTContext $ctx, $query, $keyspace, $entity_column_name, array $entity_keyspace_ids, array $bind_vars, $tablet_type = \topodata\TabletType::MASTER, $not_in_transaction = FALSE) {
if (! $this->inTransaction()) {
throw new VTException('execute called while not in transaction.');
}
$request = new \vtgate\ExecuteEntityIdsRequest();
$request->setSession($this->session);
$request->setQuery(VTProto::BoundQuery($query, $bind_vars));
$request->setTabletType($tablet_type);
$request->setNotInTransaction($not_in_transaction);
$request->setKeyspace($keyspace);
$request->setEntityColumnName($entity_column_name);
VTProto::addEntityKeyspaceIds($request, $entity_keyspace_ids);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeEntityIds($ctx, $request);
$this->session = $response->getSession();
VTProto::checkError($response);
return new VTCursor($response->getResult());
}
public function executeBatchShards(VTContext $ctx, array $bound_shard_queries, $tablet_type = \topodata\TabletType::MASTER, $as_transaction = TRUE) {
if (! $this->inTransaction()) {
throw new VTException('execute called while not in transaction.');
}
$request = new \vtgate\ExecuteBatchShardsRequest();
$request->setSession($this->session);
VTProto::addQueries($request, $bound_shard_queries);
$request->setTabletType($tablet_type);
$request->setAsTransaction($as_transaction);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeBatchShards($ctx, $request);
$this->session = $response->getSession();
VTProto::checkError($response);
$results = array();
foreach ($response->getResultsList() as $result) {
$results[] = new VTCursor($result);
}
return $results;
}
public function executeBatchKeyspaceIds(VTContext $ctx, array $bound_keyspace_id_queries, $tablet_type = \topodata\TabletType::MASTER, $as_transaction = TRUE) {
if (! $this->inTransaction()) {
throw new VTException('execute called while not in transaction.');
}
$request = new \vtgate\ExecuteBatchKeyspaceIdsRequest();
$request->setSession($this->session);
VTProto::addQueries($request, $bound_keyspace_id_queries);
$request->setTabletType($tablet_type);
$request->setAsTransaction($as_transaction);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->executeBatchKeyspaceIds($ctx, $request);
$this->session = $response->getSession();
VTProto::checkError($response);
$results = array();
foreach ($response->getResultsList() as $result) {
$results[] = new VTCursor($result);
}
return $results;
}
public function commit(VTContext $ctx) {
if (! $this->inTransaction()) {
throw new VTException('commit called while not in transaction.');
}
$request = new \vtgate\CommitRequest();
$request->setSession($this->session);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->commit($ctx, $request);
$this->session = NULL;
}
public function rollback(VTContext $ctx) {
if (! $this->inTransaction()) {
throw new VTException('rollback called while not in transaction.');
}
$request = new \vtgate\RollbackRequest();
$request->setSession($this->session);
if ($ctx->getCallerId()) {
$request->setCallerId($ctx->getCallerId());
}
$response = $this->client->rollback($ctx, $request);
$this->session = NULL;
}
}

157
php/src/VTGrpcClient.php Normal file
Просмотреть файл

@ -0,0 +1,157 @@
<?php
require_once (dirname(__FILE__) . '/proto/vtgateservice.php');
require_once (dirname(__FILE__) . '/VTRpcClient.php');
class VTGrpcClient implements VTRpcClient {
protected $stub;
public function __construct($addr, $opts = []) {
$this->stub = new \vtgateservice\VitessClient($addr, $opts);
}
public function execute(VTContext $ctx, \vtgate\ExecuteRequest $request) {
list ( $response, $status ) = $this->stub->Execute($request)->wait();
self::checkError($status);
return $response;
}
public function executeShards(VTContext $ctx, \vtgate\ExecuteShardsRequest $request) {
list ( $response, $status ) = $this->stub->ExecuteShards($request)->wait();
self::checkError($status);
return $response;
}
public function executeKeyspaceIds(VTContext $ctx, \vtgate\ExecuteKeyspaceIdsRequest $request) {
list ( $response, $status ) = $this->stub->ExecuteKeyspaceIds($request)->wait();
self::checkError($status);
return $response;
}
public function executeKeyRanges(VTContext $ctx, \vtgate\ExecuteKeyRangesRequest $request) {
list ( $response, $status ) = $this->stub->ExecuteKeyRanges($request)->wait();
self::checkError($status);
return $response;
}
public function executeEntityIds(VTContext $ctx, \vtgate\ExecuteEntityIdsRequest $request) {
list ( $response, $status ) = $this->stub->ExecuteEntityIds($request)->wait();
self::checkError($status);
return $response;
}
public function executeBatchShards(VTContext $ctx, \vtgate\ExecuteBatchShardsRequest $request) {
list ( $response, $status ) = $this->stub->ExecuteBatchShards($request)->wait();
self::checkError($status);
return $response;
}
public function executeBatchKeyspaceIds(VTContext $ctx, \vtgate\ExecuteBatchKeyspaceIdsRequest $request) {
list ( $response, $status ) = $this->stub->ExecuteBatchKeyspaceIds($request)->wait();
self::checkError($status);
return $response;
}
public function streamExecute(VTContext $ctx, \vtgate\StreamExecuteRequest $request) {
return new VTGrpcStreamResponse($this->stub->StreamExecute($request));
}
public function streamExecuteShards(VTContext $ctx, \vtgate\StreamExecuteShardsRequest $request) {
return new VTGrpcStreamResponse($this->stub->StreamExecuteShards($request));
}
public function streamExecuteKeyspaceIds(VTContext $ctx, \vtgate\StreamExecuteKeyspaceIdsRequest $request) {
return new VTGrpcStreamResponse($this->stub->StreamExecuteKeyspaceIds($request));
}
public function streamExecuteKeyRanges(VTContext $ctx, \vtgate\StreamExecuteKeyRangesRequest $request) {
return new VTGrpcStreamResponse($this->stub->StreamExecuteKeyRanges($request));
}
public function begin(VTContext $ctx, \vtgate\BeginRequest $request) {
list ( $response, $status ) = $this->stub->Begin($request)->wait();
self::checkError($status);
return $response;
}
public function commit(VTContext $ctx, \vtgate\CommitRequest $request) {
list ( $response, $status ) = $this->stub->Commit($request)->wait();
self::checkError($status);
return $response;
}
public function rollback(VTContext $ctx, \vtgate\RollbackRequest $request) {
list ( $response, $status ) = $this->stub->Rollback($request)->wait();
self::checkError($status);
return $response;
}
public function getSrvKeyspace(VTContext $ctx, \vtgate\GetSrvKeyspaceRequest $request) {
list ( $response, $status ) = $this->stub->GetSrvKeyspace($request)->wait();
self::checkError($status);
return $response;
}
public function splitQuery(VTContext $ctx, \vtgate\SplitQueryRequest $request) {
list ( $response, $status ) = $this->stub->SplitQuery($request)->wait();
self::checkError($status);
return $response;
}
public function close() {
$this->stub->close();
}
public static function checkError($status) {
if ($status) {
switch ($status->code) {
case Grpc\STATUS_OK:
break;
case Grpc\STATUS_INVALID_ARGUMENT:
throw new VTBadInputError($status->details);
case Grpc\STATUS_DEADLINE_EXCEEDED:
throw new VTDeadlineExceededError($status->details);
case Grpc\STATUS_ALREADY_EXISTS:
throw new VTIntegrityError($status->details);
case Grpc\STATUS_UNAUTHENTICATED:
throw new VTUnauthenticatedError($status->details);
case Grpc\STATUS_UNAVAILABLE:
throw new VTTransientError($status->details);
default:
throw new VTException("{$status->code}: {$status->details}");
}
}
}
}
class VTGrpcStreamResponse {
private $call;
private $iterator;
public function __construct($call) {
$this->call = $call;
$this->iterator = $call->responses();
if (! $this->iterator->valid()) {
// No responses were returned. Check for error.
VTGrpcClient::checkError($this->call->getStatus());
}
}
public function next() {
if ($this->iterator->valid()) {
$value = $this->iterator->current();
$this->iterator->next();
return $value;
} else {
// Now that the responses are done, check for gRPC status.
VTGrpcClient::checkError($this->call->getStatus());
// If no exception is raised, indicate successful end of stream.
return FALSE;
}
}
public function close() {
$this->call->cancel();
}
}

766
php/src/VTProto.php
Просмотреть файл

@ -1,16 +1,13 @@
<?php
require_once (dirname(__FILE__) . '/../vendor/autoload.php');
require_once (dirname(__FILE__) . '/proto/query.php');
require_once (dirname(__FILE__) . '/proto/vtgate.php');
require_once (dirname(__FILE__) . '/proto/topodata.php');
require_once (dirname(__FILE__) . '/proto/vtrpc.php');
/*
* This module is used for interacting with associative arrays that represent
* the proto3 structures for Vitess. Currently there's no protobuf compiler
* plugin for PHP that supports maps (a protobuf feature we use), so we can't
* generate the code needed to connect through gRPC. An official PHP plugin for
* proto3 is in the works, and the plan is to replace this with generated code
* when that's ready.
*
* In the meantime, we produce associative arrays that encode into a BSON
* representation that's compatible with our proto3 structures. We then send
* them over our legacy BSON-based GoRPC+ protocol.
* This module contains helper functions and classes for interacting with the
* Vitess protobufs.
*/
/**
@ -20,627 +17,196 @@
* This is necessary because PHP doesn't have a real unsigned int type.
*/
class VTUnsignedInt {
public $bsonValue;
public $value;
public function __construct($value) {
if (is_int($value)) {
$this->bsonValue = $value;
} else if (is_string($value)) {
$this->bsonValue = new MongoInt64($value);
$this->value = $value;
} else {
throw new Exception('Unsupported type for VTUnsignedInt');
throw new VTException('Unsupported type for VTUnsignedInt');
}
}
}
class VTTabletType {
const IDLE = 1;
const MASTER = 2;
const REPLICA = 3;
const RDONLY = 4;
const SPARE = 5;
const EXPERIMENTAL = 6;
const SCHEMA_UPGRADE = 7;
const BACKUP = 8;
const RESTORE = 9;
const WORKER = 10;
const SCRAP = 11;
}
class VTBindVariable {
const TYPE_NULL = 0;
const TYPE_BYTES = 1;
const TYPE_INT = 2;
const TYPE_UINT = 3;
const TYPE_FLOAT = 4;
const TYPE_BYTES_LIST = 5;
const TYPE_INT_LIST = 6;
const TYPE_UINT_LIST = 7;
const TYPE_FLOAT_LIST = 8;
/**
* buildBsonP3 creates a BindVariable bsonp3 object.
*/
public static function buildBsonP3($value) {
if (is_null($value)) {
return array(
'Type' => self::TYPE_NULL
);
} else if (is_string($value)) {
return array(
'Type' => self::TYPE_BYTES,
'ValueBytes' => new MongoBinData($value)
);
} else if (is_int($value)) {
return array(
'Type' => self::TYPE_INT,
'ValueInt' => $value
);
} else if (is_float($value)) {
return array(
'Type' => self::TYPE_FLOAT,
'ValueFloat' => $value
);
} else if (is_object($value)) {
switch (get_class($value)) {
case 'VTUnsignedInt':
return array(
'Type' => self::TYPE_UINT,
'ValueUint' => $value->bsonValue
);
}
} else if (is_array($value)) {
return self::buildBsonP3ForList($value);
}
throw new Exception('Unknown bind variable type.');
}
public static function buildBsonP3ForList(array $list) {
if (count($list) == 0) {
// The list is empty, so it has no type. VTTablet will reject an empty
// list anyway, so we'll just pretend it was a list of bytes.
return array(
'Type' => self::TYPE_BYTES_LIST
);
}
// Check type of first item to determine type of list.
// We only support lists whose elements have uniform types.
if (is_string($list[0])) {
$value = array();
foreach ($list as $val) {
$value[] = new MongoBinData($val);
}
return array(
'Type' => self::TYPE_BYTES_LIST,
'ValueBytesList' => $value
);
} else if (is_int($list[0])) {
return array(
'Type' => self::TYPE_INT_LIST,
'ValueIntList' => $list
);
} else if (is_float($list[0])) {
return array(
'Type' => self::TYPE_FLOAT_LIST,
'ValueFloatList' => $list
);
} else if (is_object($list[0])) {
switch (get_class($list[0])) {
case 'VTUnsignedInt':
$value = array();
foreach ($list as $val) {
$value[] = $val->bsonValue;
}
return array(
'Type' => self::TYPE_UINT_LIST,
'ValueUintList' => $value
);
}
}
throw new Exception('Unknown list bind variable type.');
}
/**
* fromBsonP3 returns a PHP object from a bsonp3 BindVariable object.
*/
public static function fromBsonP3($bson) {
if (! array_key_exists('Type', $bson)) {
return NULL;
}
switch ($bson['Type']) {
case self::TYPE_NULL:
return NULL;
case self::TYPE_BYTES:
return $bson['ValueBytes'];
case self::TYPE_INT:
return $bson['ValueInt'];
case self::TYPE_UINT:
return new VTUnsignedInt($bson['ValueUint']);
case self::TYPE_FLOAT:
return $bson['ValueFloat'];
case self::TYPE_BYTES_LIST:
return $bson['ValueBytesList'];
case self::TYPE_INT_LIST:
return $bson['ValueIntList'];
case self::TYPE_UINT_LIST:
$result = array();
foreach ($bson['ValueUintList'] as $val) {
$result[] = new VTUnsignedInt($val);
}
return $result;
case self::TYPE_FLOAT_LIST:
return $bson['ValueFloatList'];
}
throw new Exception('Unknown bind variable type.');
}
}
class VTBoundQuery {
public $query;
public $bindVars;
public function __construct($query = '', $bindVars = array()) {
$this->query = $query;
$this->bindVars = $bindVars;
}
public static function fromBsonP3($bson) {
$result = new VTBoundQuery();
if (array_key_exists('Sql', $bson)) {
$result->query = $bson['Sql'];
}
if (array_key_exists('BindVariables', $bson)) {
foreach ($bson['BindVariables'] as $key => $val) {
$result->bindVars[$key] = VTBindVariable::fromBsonP3($val);
}
}
return $result;
}
/**
* buildBsonP3 creates a BoundQuery bsonp3 object.
*/
public static function buildBsonP3($query, $vars) {
$bindVars = array();
if ($vars) {
foreach ($vars as $key => $value) {
$bindVars[$key] = VTBindVariable::buildBsonP3($value);
}
}
return array(
'Sql' => $query,
'BindVariables' => $bindVars
);
}
}
class VTBoundShardQuery {
public $query;
public $vars;
public $keyspace;
public $shards;
public function __construct($query, $bind_vars, $keyspace, $shards) {
$this->query = $query;
$this->vars = $bind_vars;
$this->keyspace = $keyspace;
$this->shards = $shards;
}
public function toBsonP3() {
return array(
'Query' => VTBoundQuery::buildBsonP3($this->query, $this->vars),
'Keyspace' => $this->keyspace,
'Shards' => $this->shards
);
}
public static function buildBsonP3Array($bound_queries) {
$result = array();
foreach ($bound_queries as $bound_query) {
$result[] = $bound_query->toBsonP3();
}
return $result;
}
}
class VTBoundKeyspaceIdQuery {
public $query;
public $vars;
public $keyspace;
public $keyspaceIds;
public function __construct($query, $bind_vars, $keyspace, $keyspace_ids) {
$this->query = $query;
$this->vars = $bind_vars;
$this->keyspace = $keyspace;
$this->keyspaceIds = $keyspace_ids;
}
public function toBsonP3() {
return array(
'Query' => VTBoundQuery::buildBsonP3($this->query, $this->vars),
'Keyspace' => $this->keyspace,
'KeyspaceIds' => VTKeyspaceId::buildBsonP3Array($this->keyspaceIds)
);
}
public static function buildBsonP3Array($bound_queries) {
$result = array();
foreach ($bound_queries as $bound_query) {
$result[] = $bound_query->toBsonP3();
}
return $result;
}
}
class VTField {
const TYPE_DECIMAL = 0;
const TYPE_TINY = 1;
const TYPE_SHORT = 2;
const TYPE_LONG = 3;
const TYPE_FLOAT = 4;
const TYPE_DOUBLE = 5;
const TYPE_NULL = 6;
const TYPE_TIMESTAMP = 7;
const TYPE_LONGLONG = 8;
const TYPE_INT24 = 9;
const TYPE_DATE = 10;
const TYPE_TIME = 11;
const TYPE_DATETIME = 12;
const TYPE_YEAR = 13;
const TYPE_NEWDATE = 14;
const TYPE_VARCHAR = 15;
const TYPE_BIT = 16;
const TYPE_NEWDECIMAL = 246;
const TYPE_ENUM = 247;
const TYPE_SET = 248;
const TYPE_TINY_BLOB = 249;
const TYPE_MEDIUM_BLOB = 250;
const TYPE_LONG_BLOB = 251;
const TYPE_BLOB = 252;
const TYPE_VAR_STRING = 253;
const TYPE_STRING = 254;
const TYPE_GEOMETRY = 255;
const VT_ZEROVALUE_FLAG = 0;
const VT_NOT_NULL_FLAG = 1;
const VT_PRI_KEY_FLAG = 2;
const VT_UNIQUE_KEY_FLAG = 4;
const VT_MULTIPLE_KEY_FLAG = 8;
const VT_BLOB_FLAG = 16;
const VT_UNSIGNED_FLAG = 32;
const VT_ZEROFILL_FLAG = 64;
const VT_BINARY_FLAG = 128;
const VT_ENUM_FLAG = 256;
const VT_AUTO_INCREMENT_FLAG = 512;
const VT_TIMESTAMP_FLAG = 1024;
const VT_SET_FLAG = 2048;
const VT_NO_DEFAULT_VALUE_FLAG = 4096;
const VT_ON_UPDATE_NOW_FLAG = 8192;
const VT_NUM_FLAG = 32768;
public $name = '';
public $type = 0;
public $flags = 0;
public static function fromBsonP3($bson) {
$result = new VTField();
if (array_key_exists('Name', $bson)) {
$result->name = $bson['Name'];
}
if (array_key_exists('Type', $bson)) {
$result->type = $bson['Type'];
}
if (array_key_exists('Flags', $bson)) {
$result->flags = $bson['Flags'];
}
return $result;
}
}
class VTQueryResult {
public $fields = array();
public $rowsAffected = 0;
public $insertId = 0;
public $rows = array();
public static function fromBsonP3($bson) {
$result = new VTQueryResult();
if (array_key_exists('Fields', $bson)) {
foreach ($bson['Fields'] as $field) {
$result->fields[] = VTField::fromBsonP3($field);
}
}
if (array_key_exists('RowsAffected', $bson)) {
$result->rowsAffected = $bson['RowsAffected'];
}
if (array_key_exists('InsertId', $bson)) {
$result->insertId = $bson['InsertId'];
}
if (array_key_exists('Rows', $bson)) {
foreach ($bson['Rows'] as $row) {
$result->rows[] = $row['Values'];
}
}
return $result;
}
}
class VTProto {
public static function checkError($resp) {
// TODO(enisoc): Implement app-level error checking.
}
}
class VTCallerId {
public $principal;
public $component;
public $subcomponent;
public function __construct($principal, $component, $subcomponent) {
$this->principal = $principal;
$this->component = $component;
$this->subcomponent = $subcomponent;
}
public function toBsonP3() {
return array(
'Principal' => $this->principal,
'Component' => $this->component,
'Subcomponent' => $this->subcomponent
);
}
}
class VTKeyRange {
public static function buildBsonP3Array($key_ranges) {
$result = array();
foreach ($key_ranges as $key_range) {
$result[] = array(
'Start' => new MongoBinData($key_range[0]),
'End' => new MongoBinData($key_range[1])
);
public static function checkError($response) {
$error = $response->getError();
if ($error) {
switch ($error->getCode()) {
case \vtrpc\ErrorCode::SUCCESS:
break;
case \vtrpc\ErrorCode::BAD_INPUT:
throw new VTBadInputError($error->getMessage());
case \vtrpc\ErrorCode::DEADLINE_EXCEEDED:
throw new VTDeadlineExceededError($error->getMessage());
case \vtrpc\ErrorCode::INTEGRITY_ERROR:
throw new VTIntegrityError($error->getMessage());
case \vtrpc\ErrorCode::TRANSIENT_ERROR:
throw new VTTransientError($error->getMessage());
case \vtrpc\ErrorCode::UNAUTHENTICATED:
throw new VTUnauthenticatedError($error->getMessage());
default:
throw new VTException($error->getCode() . ': ' . $error->getMessage());
}
}
return $result;
}
public static function fromBsonP3($bson) {
$result = array(
'',
''
);
if (array_key_exists('Start', $bson)) {
$result[0] = $bson['Start'];
public static function BoundQuery($query, $vars) {
$bound_query = new \query\BoundQuery();
$bound_query->setSql($query);
if ($vars) {
foreach ($vars as $key => $value) {
$entry = new \query\BoundQuery\BindVariablesEntry();
$entry->setKey($key);
$entry->setValue(self::BindVariable($value));
$bound_query->addBindVariables($entry);
}
}
if (array_key_exists('End', $bson)) {
$result[1] = $bson['End'];
}
return $result;
return $bound_query;
}
}
class VTKeyspaceId {
public static function BindVariable($value) {
$bind_var = new \query\BindVariable();
if (is_null($value)) {
$bind_var->setType(\query\BindVariable\Type::TYPE_NULL);
} else if (is_string($value)) {
$bind_var->setType(\query\BindVariable\Type::TYPE_BYTES);
$bind_var->setValueBytes($value);
} else if (is_int($value)) {
$bind_var->setType(\query\BindVariable\Type::TYPE_INT);
$bind_var->setValueInt($value);
} else if (is_float($value)) {
$bind_var->setType(\query\BindVariable\Type::TYPE_FLOAT);
$bind_var->setValueFloat($value);
} else if (is_object($value)) {
switch (get_class($value)) {
case 'VTUnsignedInt':
$bind_var->setType(\query\BindVariable\Type::TYPE_UINT);
$bind_var->setValueUint($value->value);
break;
default:
throw new VTException('Unknown bind variable class: ' . get_class($value));
}
} else if (is_array($value)) {
self::ListBindVariable($bind_var, $value);
} else {
throw new VTException('Unknown bind variable type.');
}
return $bind_var;
}
public static function fromHex($hex) {
protected static function ListBindVariable(&$bind_var, array $list) {
if (count($list) == 0) {
// The list is empty, so it has no type. VTTablet will reject an empty
// list anyway, so we'll just pretend it was a list of bytes.
$bind_var->setType(\query\BindVariable\Type::TYPE_BYTES_LIST);
return;
}
// Check type of first item to determine type of list.
// We only support lists whose elements have uniform types.
if (is_string($list[0])) {
$bind_var->setType(\query\BindVariable\Type::TYPE_BYTES_LIST);
$bind_var->setValueBytesList($list);
} else if (is_int($list[0])) {
$bind_var->setType(\query\BindVariable\Type::TYPE_INT_LIST);
$bind_var->setValueIntList($list);
} else if (is_float($list[0])) {
$bind_var->setType(\query\BindVariable\Type::TYPE_FLOAT_LIST);
$bind_var->setValueFloatList($list);
} else if (is_object($list[0])) {
switch (get_class($list[0])) {
case 'VTUnsignedInt':
$bind_var->setType(\query\BindVariable\Type::TYPE_UINT_LIST);
$value = array();
foreach ($list as $val) {
$value[] = $val->value;
}
$bind_var->setValueUintList($value);
break;
default:
throw new VTException('Unknown list bind variable class: ' . get_class($list[0]));
}
} else {
throw new VTException('Unknown list bind variable type.');
}
}
public static function addQueries($proto, $queries) {
foreach ($queries as $query) {
$proto->addQueries($query);
}
}
public static function KeyspaceIdFromHex($hex) {
return pack('H*', $hex);
}
public static function buildBsonP3Array($keyspace_ids) {
$result = array();
foreach ($keyspace_ids as $kid) {
$result[] = new MongoBinData($kid);
}
return $result;
public static function KeyRangeFromHex($start, $end) {
$value = new \topodata\KeyRange();
$value->setStart(self::KeyspaceIdFromHex($start));
$value->setEnd(self::KeyspaceIdFromHex($end));
return $value;
}
}
class VTKeyspaceIdType {
const UINT64 = 1;
const BYTES = 2;
}
public static function addKeyRanges($proto, $key_ranges) {
foreach ($key_ranges as $key_range) {
$proto->addKeyRanges($key_range);
}
}
class VTEntityId {
const TYPE_BYTES = 1;
const TYPE_INT = 2;
const TYPE_UINT = 3;
const TYPE_FLOAT = 4;
public static function buildBsonP3($entity_id) {
if (is_string($entity_id)) {
return array(
'XidType' => self::TYPE_BYTES,
'XidBytes' => new MongoBinData($entity_id)
);
} else if (is_int($entity_id)) {
return array(
'XidType' => self::TYPE_INT,
'XidInt' => $entity_id
);
} else if (is_float($entity_id)) {
return array(
'XidType' => self::TYPE_FLOAT,
'XidFloat' => $entity_id
);
} else if (is_object($entity_id)) {
switch (get_class($entity_id)) {
public static function EntityId($keyspace_id, $value) {
$eid = new \vtgate\ExecuteEntityIdsRequest\EntityId();
$eid->setKeyspaceId($keyspace_id);
if (is_string($value)) {
$eid->setXidType(\vtgate\ExecuteEntityIdsRequest\EntityId\Type::TYPE_BYTES);
$eid->setXidBytes($value);
} else if (is_int($value)) {
$eid->setXidType(\vtgate\ExecuteEntityIdsRequest\EntityId\Type::TYPE_INT);
$eid->setXidInt($value);
} else if (is_float($value)) {
$eid->setXidType(\vtgate\ExecuteEntityIdsRequest\EntityId\Type::TYPE_FLOAT);
$eid->setXidFloat($value);
} else if (is_object($value)) {
switch (get_class($value)) {
case 'VTUnsignedInt':
return array(
'XidType' => self::TYPE_UINT,
'XidUint' => $entity_id->bsonValue
);
$eid->setXidType(\vtgate\ExecuteEntityIdsRequest\EntityId\Type::TYPE_UINT);
$eid->setXidUint($value->value);
break;
default:
throw new VTException('Unknown entity ID class: ' . get_class($value));
}
} else {
throw new VTException('Unknown entity ID type.');
}
throw new Exception('Unknown entity ID type.');
return $eid;
}
public static function buildBsonP3Array($entity_keyspace_ids) {
$result = array();
public static function addEntityKeyspaceIds($proto, $entity_keyspace_ids) {
foreach ($entity_keyspace_ids as $keyspace_id => $entity_id) {
$eid = self::buildBsonP3($entity_id);
$eid['KeyspaceId'] = new MongoBinData($keyspace_id);
$result[] = $eid;
$proto->addEntityKeyspaceIds(self::EntityId($keyspace_id, $entity_id));
}
return $result;
}
public function BoundShardQuery($query, $bind_vars, $keyspace, $shards) {
$value = new \vtgate\BoundShardQuery();
$value->setQuery(self::BoundQuery($query, $bind_vars));
$value->setKeyspace($keyspace);
$value->setShards($shards);
return $value;
}
public function BoundKeyspaceIdQuery($query, $bind_vars, $keyspace, $keyspace_ids) {
$value = new \vtgate\BoundKeyspaceIdQuery();
$value->setQuery(self::BoundQuery($query, $bind_vars));
$value->setKeyspace($keyspace);
$value->setKeyspaceIds($keyspace_ids);
return $value;
}
}
class VTSplitQueryKeyRangePart {
public $keyspace = '';
public $keyRanges = array();
public static function fromBsonP3($bson) {
$result = new VTSplitQueryKeyRangePart();
if (array_key_exists('Keyspace', $bson)) {
$result->keyspace = $bson['Keyspace'];
}
if (array_key_exists('KeyRanges', $bson) && $bson['KeyRanges']) {
foreach ($bson['KeyRanges'] as $val) {
$result->keyRanges[] = VTKeyRange::fromBsonP3($val);
}
}
return $result;
}
}
class VTSplitQueryShardPart {
public $keyspace = '';
public $shards = array();
public static function fromBsonP3($bson) {
$result = new VTSplitQueryShardPart();
if (array_key_exists('Keyspace', $bson)) {
$result->keyspace = $bson['Keyspace'];
}
if (array_key_exists('Shards', $bson)) {
$result->shards = $bson['Shards'];
}
return $result;
}
}
class VTSplitQueryPart {
public $query = '';
public $keyRangePart;
public $shardPart;
public $size = 0;
public static function fromBsonP3($bson) {
$result = new VTSplitQueryPart();
if (array_key_exists('Query', $bson)) {
$result->query = VTBoundQuery::fromBsonP3($bson['Query']);
}
if (array_key_exists('KeyRangePart', $bson) && $bson['KeyRangePart']) {
$result->keyRangePart = VTSplitQueryKeyRangePart::fromBsonP3($bson['KeyRangePart']);
}
if (array_key_exists('ShardPart', $bson) && $bson['ShardPart']) {
$result->shardPart = VTSplitQueryShardPart::fromBsonP3($bson['ShardPart']);
}
if (array_key_exists('Size', $bson)) {
$result->size = $bson['Size'];
}
return $result;
}
}
class VTShardReference {
public $name;
public $keyRange;
public function __construct($name = '', $keyRange = array('','')) {
$this->name = $name;
$this->keyRange = $keyRange;
}
public static function fromBsonP3($bson) {
$result = new VTShardReference();
if (array_key_exists('Name', $bson)) {
$result->name = $bson['Name'];
}
if (array_key_exists('KeyRange', $bson)) {
$result->keyRange = VTKeyRange::fromBsonP3($bson['KeyRange']);
}
return $result;
}
}
class VTSrvKeyspacePartition {
public $servedType;
public $shardReferences;
public function __construct($servedType = 0, $shardReferences = array()) {
$this->servedType = $servedType;
$this->shardReferences = $shardReferences;
}
public static function fromBsonP3($bson) {
$result = new VTSrvKeyspacePartition();
if (array_key_exists('ServedType', $bson)) {
$result->servedType = $bson['ServedType'];
}
if (array_key_exists('ShardReferences', $bson) && $bson['ShardReferences']) {
foreach ($bson['ShardReferences'] as $val) {
$result->shardReferences[] = VTShardReference::fromBsonP3($val);
}
}
return $result;
}
}
class VTSrvKeyspaceServedFrom {
public $tabletType;
public $keyspace;
public function __construct($tabletType = 0, $keyspace = '') {
$this->tabletType = $tabletType;
$this->keyspace = $keyspace;
}
public static function fromBsonP3($bson) {
$result = new VTSrvKeyspaceServedFrom();
if (array_key_exists('TabletType', $bson)) {
$result->tabletType = $bson['TabletType'];
}
if (array_key_exists('Keyspace', $bson)) {
$result->keyspace = $bson['Keyspace'];
}
return $result;
}
}
class VTSrvKeyspace {
public $partitions = array();
public $shardingColumnName = '';
public $shardingColumnType = 0;
public $servedFrom = array();
public $splitShardCount = 0;
public static function fromBsonP3($bson) {
$result = new VTSrvKeyspace();
if (array_key_exists('Partitions', $bson) && $bson['Partitions']) {
foreach ($bson['Partitions'] as $val) {
$result->partitions[] = VTSrvKeyspacePartition::fromBsonP3($val);
}
}
if (array_key_exists('ShardingColumnName', $bson)) {
$result->shardingColumnName = $bson['ShardingColumnName'];
}
if (array_key_exists('ShardingColumnType', $bson)) {
$result->shardingColumnType = $bson['ShardingColumnType'];
}
if (array_key_exists('ServedFrom', $bson)) {
foreach ($bson['ServedFrom'] as $val) {
$result->servedFrom[] = VTSrvKeyspaceServedFrom::fromBsonP3($val);
}
}
if (array_key_exists('SplitShardCount', $bson)) {
$result->splitShardCount = $bson['SplitShardCount'];
}
return $result;
}
}

40
php/src/VTRpcClient.php Normal file
Просмотреть файл

@ -0,0 +1,40 @@
<?php
require_once (dirname(__FILE__) . '/proto/vtgate.php');
require_once (dirname(__FILE__) . '/VTContext.php');
interface VTRpcClient {
public function execute(VTContext $ctx, \vtgate\ExecuteRequest $request);
public function executeShards(VTContext $ctx, \vtgate\ExecuteShardsRequest $request);
public function executeKeyspaceIds(VTContext $ctx, \vtgate\ExecuteKeyspaceIdsRequest $request);
public function executeKeyRanges(VTContext $ctx, \vtgate\ExecuteKeyRangesRequest $request);
public function executeEntityIds(VTContext $ctx, \vtgate\ExecuteEntityIdsRequest $request);
public function executeBatchShards(VTContext $ctx, \vtgate\ExecuteBatchShardsRequest $request);
public function executeBatchKeyspaceIds(VTContext $ctx, \vtgate\ExecuteBatchKeyspaceIdsRequest $request);
public function streamExecute(VTContext $ctx, \vtgate\StreamExecuteRequest $request);
public function streamExecuteShards(VTContext $ctx, \vtgate\StreamExecuteShardsRequest $request);
public function streamExecuteKeyspaceIds(VTContext $ctx, \vtgate\StreamExecuteKeyspaceIdsRequest $request);
public function streamExecuteKeyRanges(VTContext $ctx, \vtgate\StreamExecuteKeyRangesRequest $request);
public function begin(VTContext $ctx, \vtgate\BeginRequest $request);
public function commit(VTContext $ctx, \vtgate\CommitRequest $request);
public function rollback(VTContext $ctx, \vtgate\RollbackRequest $request);
public function getSrvKeyspace(VTContext $ctx, \vtgate\GetSrvKeyspaceRequest $request);
public function splitQuery(VTContext $ctx, \vtgate\SplitQueryRequest $request);
public function close();
}

286
php/tests/VTGateConnTest.php
Просмотреть файл

@ -1,17 +1,28 @@
<?php
require_once (dirname(__FILE__) . '/VTTestUtils.php');
require_once (dirname(__FILE__) . '/../src/VTProto.php');
require_once (dirname(__FILE__) . '/../src/VTGateConn.php');
require_once (dirname(__FILE__) . '/../src/BsonRpcClient.php');
require_once (dirname(__FILE__) . '/../src/VTGrpcClient.php');
class VTGateConnTest extends PHPUnit_Framework_TestCase {
private static $proc;
private static $client;
private static $ECHO_QUERY = 'echo://test query';
private static $ERROR_PREFIX = 'error://';
private static $PARTIAL_ERROR_PREFIX = 'partialerror://';
private static $EXECUTE_ERRORS = array(
'bad input' => 'VTBadInputError',
'deadline exceeded' => 'VTDeadlineExceededError',
'integrity error' => 'VTIntegrityError',
'transient error' => 'VTTransientError',
'unauthenticated' => 'VTUnauthenticatedError',
'unknown error' => 'VTException'
);
private static $BIND_VARS; // initialized in setUpBeforeClass()
private static $BIND_VARS_ECHO = 'map[bytes:[104 101 108 108 111] float:1.5 int:123 uint_from_int:345 uint_from_string:678]';
private static $BIND_VARS_ECHO = 'map[bytes:[104 101 108 108 111] float:1.5 int:123 uint_from_int:18446744073709551493]'; // 18446744073709551493 = uint64(-123)
private static $CALLER_ID; // initialized in setUpBeforeClass()
private static $CALLER_ID_ECHO = 'principal:"test_principal" component:"test_component" subcomponent:"test_subcomponent" ';
private static $TABLET_TYPE = VTTabletType::REPLICA;
private static $TABLET_TYPE = \topodata\TabletType::REPLICA;
private static $TABLET_TYPE_ECHO = 'REPLICA';
private static $KEYSPACE = 'test_keyspace';
private static $SHARDS = array(
@ -43,7 +54,7 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
}
socket_close($sock);
$cmd = "$VTROOT/bin/vtgateclienttest -logtostderr -lameduck-period 0 -port $port -service_map bsonrpc-vt-vtgateservice";
$cmd = "$VTROOT/bin/vtgateclienttest -logtostderr -lameduck-period 0 -grpc_port $port -service_map grpc-vtgateservice";
$proc = proc_open($cmd, array(), $pipes);
if (! $proc) {
@ -53,12 +64,11 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
// Wait for connection to be accepted.
$ctx = VTContext::getDefault()->withDeadlineAfter(5.0);
$client = new BsonRpcClient();
$level = error_reporting(error_reporting() & ~ E_WARNING);
while (! $ctx->isCancelled()) {
try {
$client->dial($ctx, "$addr:$port");
} catch (GoRpcException $e) {
$client = new VTGrpcClient("$addr:$port");
} catch (Exception $e) {
usleep(100000);
continue;
}
@ -71,30 +81,26 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
self::$BIND_VARS = array(
'bytes' => 'hello',
'int' => 123,
'uint_from_int' => new VTUnsignedInt(345),
'uint_from_string' => new VTUnsignedInt('678'),
'uint_from_int' => new VTUnsignedInt(- 123),
'float' => 1.5
);
self::$CALLER_ID = new VTCallerId('test_principal', 'test_component', 'test_subcomponent');
self::$CALLER_ID = new \vtrpc\CallerID();
self::$CALLER_ID->setPrincipal('test_principal');
self::$CALLER_ID->setComponent('test_component');
self::$CALLER_ID->setSubcomponent('test_subcomponent');
self::$KEYSPACE_IDS = array(
VTKeyspaceID::fromHex('8000000000000000'),
VTKeyspaceID::fromHex('ff000000000000ef')
VTProto::KeyspaceIdFromHex('8000000000000000'),
VTProto::KeyspaceIdFromHex('ff000000000000ef')
);
self::$KEY_RANGES = array(
array(
'',
VTKeyspaceID::fromHex('8000000000000000')
),
array(
VTKeyspaceID::fromHex('8000000000000000'),
''
)
VTProto::KeyRangeFromHex('', '8000000000000000'),
VTProto::KeyRangeFromHex('8000000000000000', '')
);
self::$ENTITY_KEYSPACE_IDS = array(
VTKeyspaceID::fromHex('1234567800000002') => 'hello',
VTKeyspaceID::fromHex('1234567800000000') => 123,
VTKeyspaceID::fromHex('1234567800000001') => new VTUnsignedInt('456'),
VTKeyspaceID::fromHex('1234567800000002') => 1.5
VTProto::KeyspaceIdFromHex('1234567800000002') => 'hello',
VTProto::KeyspaceIdFromHex('1234567800000000') => 123,
VTProto::KeyspaceIdFromHex('1234567800000001') => new VTUnsignedInt(456),
VTProto::KeyspaceIdFromHex('1234567800000002') => 1.5
);
}
@ -120,10 +126,11 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$this->conn = new VTGateconn(self::$client);
}
private static function getEcho(VTQueryResult $result) {
private static function getEcho($result) {
$echo = array();
foreach ($result->fields as $i => $field) {
$echo[$field->name] = $result->rows[0][$i];
$row = $result->next();
foreach ($result->getFields() as $i => $field) {
$echo[$field->getName()] = $row[$i];
}
return $echo;
}
@ -172,7 +179,7 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$this->assertEquals(self::$TABLET_TYPE_ECHO, $echo['tabletType']);
$results = $conn->executeBatchShards($ctx, array(
new VTBoundShardQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$SHARDS)
VTProto::BoundShardQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$SHARDS)
), self::$TABLET_TYPE, TRUE);
$echo = self::getEcho($results[0]);
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
@ -184,7 +191,7 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$this->assertEquals('true', $echo['asTransaction']);
$results = $conn->executeBatchKeyspaceIds($ctx, array(
new VTBoundKeyspaceIdQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$KEYSPACE_IDS)
VTProto::BoundKeyspaceIdQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$KEYSPACE_IDS)
), self::$TABLET_TYPE, TRUE);
$echo = self::getEcho($results[0]);
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
@ -200,17 +207,13 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$ctx = $this->ctx;
$conn = $this->conn;
$sr = $conn->streamExecute($ctx, self::$ECHO_QUERY, self::$BIND_VARS, self::$TABLET_TYPE);
$results = $sr->fetchAll();
$echo = self::getEcho($results[0]);
$echo = self::getEcho($conn->streamExecute($ctx, self::$ECHO_QUERY, self::$BIND_VARS, self::$TABLET_TYPE));
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
$this->assertEquals(self::$ECHO_QUERY, $echo['query']);
$this->assertEquals(self::$BIND_VARS_ECHO, $echo['bindVars']);
$this->assertEquals(self::$TABLET_TYPE_ECHO, $echo['tabletType']);
$sr = $conn->streamExecuteShards($ctx, self::$ECHO_QUERY, self::$KEYSPACE, self::$SHARDS, self::$BIND_VARS, self::$TABLET_TYPE);
$results = $sr->fetchAll();
$echo = self::getEcho($results[0]);
$echo = self::getEcho($conn->streamExecuteShards($ctx, self::$ECHO_QUERY, self::$KEYSPACE, self::$SHARDS, self::$BIND_VARS, self::$TABLET_TYPE));
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
$this->assertEquals(self::$ECHO_QUERY, $echo['query']);
$this->assertEquals(self::$KEYSPACE, $echo['keyspace']);
@ -218,9 +221,7 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$this->assertEquals(self::$BIND_VARS_ECHO, $echo['bindVars']);
$this->assertEquals(self::$TABLET_TYPE_ECHO, $echo['tabletType']);
$sr = $conn->streamExecuteKeyspaceIds($ctx, self::$ECHO_QUERY, self::$KEYSPACE, self::$KEYSPACE_IDS, self::$BIND_VARS, self::$TABLET_TYPE);
$results = $sr->fetchAll();
$echo = self::getEcho($results[0]);
$echo = self::getEcho($conn->streamExecuteKeyspaceIds($ctx, self::$ECHO_QUERY, self::$KEYSPACE, self::$KEYSPACE_IDS, self::$BIND_VARS, self::$TABLET_TYPE));
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
$this->assertEquals(self::$ECHO_QUERY, $echo['query']);
$this->assertEquals(self::$KEYSPACE, $echo['keyspace']);
@ -228,9 +229,7 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$this->assertEquals(self::$BIND_VARS_ECHO, $echo['bindVars']);
$this->assertEquals(self::$TABLET_TYPE_ECHO, $echo['tabletType']);
$sr = $conn->streamExecuteKeyRanges($ctx, self::$ECHO_QUERY, self::$KEYSPACE, self::$KEY_RANGES, self::$BIND_VARS, self::$TABLET_TYPE);
$results = $sr->fetchAll();
$echo = self::getEcho($results[0]);
$echo = self::getEcho($conn->streamExecuteKeyRanges($ctx, self::$ECHO_QUERY, self::$KEYSPACE, self::$KEY_RANGES, self::$BIND_VARS, self::$TABLET_TYPE));
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
$this->assertEquals(self::$ECHO_QUERY, $echo['query']);
$this->assertEquals(self::$KEYSPACE, $echo['keyspace']);
@ -298,7 +297,7 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$tx = $conn->begin($ctx);
$results = $tx->executeBatchShards($ctx, array(
new VTBoundShardQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$SHARDS)
VTProto::BoundShardQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$SHARDS)
), self::$TABLET_TYPE, TRUE);
$echo = self::getEcho($results[0]);
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
@ -311,7 +310,7 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$this->assertEquals('true', $echo['asTransaction']);
$results = $tx->executeBatchKeyspaceIds($ctx, array(
new VTBoundKeyspaceIdQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$KEYSPACE_IDS)
VTProto::BoundKeyspaceIdQuery(self::$ECHO_QUERY, self::$BIND_VARS, self::$KEYSPACE, self::$KEYSPACE_IDS)
), self::$TABLET_TYPE, TRUE);
$echo = self::getEcho($results[0]);
$this->assertEquals(self::$CALLER_ID_ECHO, $echo['callerId']);
@ -332,24 +331,22 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$input_bind_vars = array(
'bytes' => 'hello',
'float' => 1.5,
'int' => 123,
'uint_from_int' => new VTUnsignedInt(345),
'uint_from_string' => new VTUnsignedInt('678'),
'float' => 1.5
'uint_from_int' => new VTUnsignedInt(345)
);
$expected_bind_vars = array(
'bytes' => 'hello',
'float' => 1.5,
'int' => 123,
'uint_from_int' => new VTUnsignedInt(345),
// uint_from_string will come back to us as an int.
'uint_from_string' => new VTUnsignedInt(678),
'float' => 1.5
'uint_from_int' => new VTUnsignedInt(345)
);
$expected = new VTSplitQueryPart();
$expected->query = new VTBoundQuery(self::$ECHO_QUERY . ':split_column:123', $expected_bind_vars);
$expected->keyRangePart = new VTSplitQueryKeyRangePart();
$expected->keyRangePart->keyspace = self::$KEYSPACE;
$expected = new \vtgate\SplitQueryResponse\Part();
$expected->setQuery(VTProto::BoundQuery(self::$ECHO_QUERY . ':split_column:123', $expected_bind_vars));
$krpart = new \vtgate\SplitQueryResponse\KeyRangePart();
$krpart->setKeyspace(self::$KEYSPACE);
$expected->setKeyRangePart($krpart);
$actual = $conn->splitQuery($ctx, self::$KEYSPACE, self::$ECHO_QUERY, $input_bind_vars, 'split_column', 123);
$this->assertEquals($expected, $actual[0]);
@ -359,19 +356,178 @@ class VTGateConnTest extends PHPUnit_Framework_TestCase {
$ctx = $this->ctx;
$conn = $this->conn;
$expected = new VTSrvKeyspace();
$expected->partitions[] = new VTSrvKeyspacePartition(VTTabletType::REPLICA, array(
new VTShardReference("shard0", array(
VTKeyspaceID::fromHex('4000000000000000'),
VTKeyspaceID::fromHex('8000000000000000')
))
));
$expected->shardingColumnName = 'sharding_column_name';
$expected->shardingColumnType = VTKeyspaceIDType::UINT64;
$expected->servedFrom[] = new VTSrvKeyspaceServedFrom(VTTabletType::MASTER, 'other_keyspace');
$expected->splitShardCount = 128;
$expected = new \topodata\SrvKeyspace();
$partition = new \topodata\SrvKeyspace\KeyspacePartition();
$partition->setServedType(\topodata\TabletType::REPLICA);
$shard_ref = new \topodata\ShardReference();
$shard_ref->setName("shard0");
$shard_ref->setKeyRange(VTProto::KeyRangeFromHex('4000000000000000', '8000000000000000'));
$partition->addShardReferences($shard_ref);
$expected->addPartitions($partition);
$expected->setShardingColumnName('sharding_column_name');
$expected->setShardingColumnType(\topodata\KeyspaceIdType::UINT64);
$served_from = new \topodata\SrvKeyspace\ServedFrom();
$served_from->setTabletType(\topodata\TabletType::MASTER);
$served_from->setKeyspace('other_keyspace');
$expected->addServedFrom($served_from);
$expected->setSplitShardCount(128);
$actual = $conn->getSrvKeyspace($ctx, "big");
$this->assertEquals($expected, $actual);
}
private function checkExecuteErrors($execute, $partial = TRUE) {
foreach (self::$EXECUTE_ERRORS as $error => $class) {
try {
$query = self::$ERROR_PREFIX . $error;
$execute($this->ctx, $this->conn, $query);
$this->fail("no exception thrown for $query");
} catch (Exception $e) {
$this->assertEquals($class, get_class($e), $e->getMessage());
}
if ($partial) {
try {
$query = self::$PARTIAL_ERROR_PREFIX . $error;
$execute($this->ctx, $this->conn, $query);
$this->fail("no exception thrown for $query");
} catch (Exception $e) {
$this->assertEquals($class, get_class($e), $e->getMessage());
}
}
}
}
private function checkTransactionExecuteErrors($execute) {
foreach (self::$EXECUTE_ERRORS as $error => $class) {
try {
$tx = $this->conn->begin($this->ctx);
$query = self::$ERROR_PREFIX . $error;
$execute($this->ctx, $tx, $query);
$this->fail("no exception thrown for $query");
} catch (Exception $e) {
$this->assertEquals($class, get_class($e), $e->getMessage());
}
// Don't close the transaction on partial error.
$tx = $this->conn->begin($this->ctx);
try {
$query = self::$PARTIAL_ERROR_PREFIX . $error;
$execute($this->ctx, $tx, $query);
$this->fail("no exception thrown for $query");
} catch (Exception $e) {
$this->assertEquals($class, get_class($e), $e->getMessage());
}
// The transaction should still be usable now.
$tx->rollback($this->ctx);
// Close the transaction on partial error.
$tx = $this->conn->begin($this->ctx);
try {
$query = self::$PARTIAL_ERROR_PREFIX . $error . '/close transaction';
$execute($this->ctx, $tx, $query);
$this->fail("no exception thrown for $query");
} catch (Exception $e) {
$this->assertEquals($class, get_class($e), $e->getMessage());
}
// The transaction should be unusable now.
try {
$tx->rollback($this->ctx);
$this->fail("no exception thrown for rollback() after closed transaction");
} catch (Exception $e) {
$this->assertEquals('VTException', get_class($e), $e->getMessage());
$this->assertEquals(TRUE, strpos($e->getMessage(), 'not in transaction') !== FALSE);
}
}
}
private function checkStreamExecuteErrors($execute) {
$this->checkExecuteErrors($execute, FALSE);
}
public function testExecuteErrors() {
$this->checkExecuteErrors(function ($ctx, $conn, $query) {
$conn->execute($ctx, $query, self::$BIND_VARS, self::$TABLET_TYPE);
});
$this->checkExecuteErrors(function ($ctx, $conn, $query) {
$conn->executeShards($ctx, $query, self::$KEYSPACE, self::$SHARDS, self::$BIND_VARS, self::$TABLET_TYPE);
});
$this->checkExecuteErrors(function ($ctx, $conn, $query) {
$conn->executeKeyspaceIds($ctx, $query, self::$KEYSPACE, self::$KEYSPACE_IDS, self::$BIND_VARS, self::$TABLET_TYPE);
});
$this->checkExecuteErrors(function ($ctx, $conn, $query) {
$conn->executeKeyRanges($ctx, $query, self::$KEYSPACE, self::$KEY_RANGES, self::$BIND_VARS, self::$TABLET_TYPE);
});
$this->checkExecuteErrors(function ($ctx, $conn, $query) {
$conn->executeEntityIds($ctx, $query, self::$KEYSPACE, self::$ENTITY_COLUMN_NAME, self::$ENTITY_KEYSPACE_IDS, self::$BIND_VARS, self::$TABLET_TYPE);
});
$this->checkExecuteErrors(function ($ctx, $conn, $query) {
$conn->executeBatchShards($ctx, array(
VTProto::BoundShardQuery($query, self::$BIND_VARS, self::$KEYSPACE, self::$SHARDS)
), self::$TABLET_TYPE, TRUE);
});
$this->checkExecuteErrors(function ($ctx, $conn, $query) {
$conn->executeBatchKeyspaceIds($ctx, array(
VTProto::BoundKeyspaceIdQuery($query, self::$BIND_VARS, self::$KEYSPACE, self::$KEYSPACE_IDS)
), self::$TABLET_TYPE, TRUE);
});
}
public function testTransactionExecuteErrors() {
$this->checkTransactionExecuteErrors(function ($ctx, $tx, $query) {
$tx->execute($ctx, $query, self::$BIND_VARS, self::$TABLET_TYPE, TRUE);
});
$this->checkTransactionExecuteErrors(function ($ctx, $tx, $query) {
$tx->executeShards($ctx, $query, self::$KEYSPACE, self::$SHARDS, self::$BIND_VARS, self::$TABLET_TYPE, TRUE);
});
$this->checkTransactionExecuteErrors(function ($ctx, $tx, $query) {
$tx->executeKeyspaceIds($ctx, $query, self::$KEYSPACE, self::$KEYSPACE_IDS, self::$BIND_VARS, self::$TABLET_TYPE, TRUE);
});
$this->checkTransactionExecuteErrors(function ($ctx, $tx, $query) {
$tx->executeKeyRanges($ctx, $query, self::$KEYSPACE, self::$KEY_RANGES, self::$BIND_VARS, self::$TABLET_TYPE, TRUE);
});
$this->checkTransactionExecuteErrors(function ($ctx, $tx, $query) {
$tx->executeEntityIds($ctx, $query, self::$KEYSPACE, self::$ENTITY_COLUMN_NAME, self::$ENTITY_KEYSPACE_IDS, self::$BIND_VARS, self::$TABLET_TYPE, TRUE);
});
$this->checkTransactionExecuteErrors(function ($ctx, $tx, $query) {
$tx->executeBatchShards($ctx, array(
VTProto::BoundShardQuery($query, self::$BIND_VARS, self::$KEYSPACE, self::$SHARDS)
), self::$TABLET_TYPE, TRUE);
});
$this->checkTransactionExecuteErrors(function ($ctx, $tx, $query) {
$tx->executeBatchKeyspaceIds($ctx, array(
VTProto::BoundKeyspaceIdQuery($query, self::$BIND_VARS, self::$KEYSPACE, self::$KEYSPACE_IDS)
), self::$TABLET_TYPE, TRUE);
});
}
public function testStreamExecuteErrors() {
$this->checkStreamExecuteErrors(function ($ctx, $conn, $query) {
$conn->streamExecute($ctx, $query, self::$BIND_VARS, self::$TABLET_TYPE)->next();
});
$this->checkStreamExecuteErrors(function ($ctx, $conn, $query) {
$conn->streamExecuteShards($ctx, $query, self::$KEYSPACE, self::$SHARDS, self::$BIND_VARS, self::$TABLET_TYPE)->next();
});
$this->checkStreamExecuteErrors(function ($ctx, $conn, $query) {
$conn->streamExecuteKeyspaceIds($ctx, $query, self::$KEYSPACE, self::$KEYSPACE_IDS, self::$BIND_VARS, self::$TABLET_TYPE)->next();
});
$this->checkStreamExecuteErrors(function ($ctx, $conn, $query) {
$conn->streamExecuteKeyRanges($ctx, $query, self::$KEYSPACE, self::$KEY_RANGES, self::$BIND_VARS, self::$TABLET_TYPE)->next();
});
}
}

83
php/tests/VTProtoTest.php
Просмотреть файл

@ -1,69 +1,38 @@
<?php
require_once (dirname(__FILE__) . '/VTTestUtils.php');
require_once (dirname(__FILE__) . '/../src/VTProto.php');
class VTProtoTest extends PHPUnit_Framework_TestCase {
public function testBoundQuery() {
$expected = array(
'Sql' => 'test query',
'BindVariables' => array(
'bytes' => array(
'Type' => VTBindVariable::TYPE_BYTES,
'ValueBytes' => new MongoBinData('hello')
),
'int' => array(
'Type' => VTBindVariable::TYPE_INT,
'ValueInt' => 123
),
'uint_from_int' => array(
'Type' => VTBindVariable::TYPE_UINT,
'ValueUint' => 345
),
'uint_from_string' => array(
'Type' => VTBindVariable::TYPE_UINT,
'ValueUint' => new MongoInt64('678')
),
'float' => array(
'Type' => VTBindVariable::TYPE_FLOAT,
'ValueFloat' => 1.5
),
'bytes_list' => array(
'Type' => VTBindVariable::TYPE_BYTES_LIST,
'ValueBytesList' => array(
new MongoBinData('one'),
new MongoBinData('two')
)
),
'int_list' => array(
'Type' => VTBindVariable::TYPE_INT_LIST,
'ValueIntList' => array(
1,
2,
3
)
),
'uint_list' => array(
'Type' => VTBindVariable::TYPE_UINT_LIST,
'ValueUintList' => array(
123,
456
)
),
'float_list' => array(
'Type' => VTBindVariable::TYPE_FLOAT_LIST,
'ValueFloatList' => array(
2.0,
4.0
)
)
)
);
$expected = new \query\BoundQuery();
$expected->setSql('test query');
add_bind_var($expected, \query\BindVariable\Type::TYPE_BYTES, 'setValueBytes', 'bytes', 'hello');
add_bind_var($expected, \query\BindVariable\Type::TYPE_INT, 'setValueInt', 'int', 123);
add_bind_var($expected, \query\BindVariable\Type::TYPE_UINT, 'setValueUint', 'uint_from_int', - 123);
add_bind_var($expected, \query\BindVariable\Type::TYPE_FLOAT, 'setValueFloat', 'float', 1.5);
add_bind_var($expected, \query\BindVariable\Type::TYPE_BYTES_LIST, 'setValueBytesList', 'bytes_list', array(
'one',
'two'
));
add_bind_var($expected, \query\BindVariable\Type::TYPE_INT_LIST, 'setValueIntList', 'int_list', array(
1,
2,
3
));
add_bind_var($expected, \query\BindVariable\Type::TYPE_UINT_LIST, 'setValueUintList', 'uint_list', array(
123,
456
));
add_bind_var($expected, \query\BindVariable\Type::TYPE_FLOAT_LIST, 'setValueFloatList', 'float_list', array(
2.0,
4.0
));
$actual = VTBoundQuery::buildBsonP3('test query', array(
$actual = VTProto::BoundQuery('test query', array(
'bytes' => 'hello',
'int' => 123,
'uint_from_int' => new VTUnsignedInt(345),
'uint_from_string' => new VTUnsignedInt('678'),
'uint_from_int' => new VTUnsignedInt(- 123),
'float' => 1.5,
'bytes_list' => array(
'one',

11
php/tests/VTTestUtils.php Normal file
Просмотреть файл

@ -0,0 +1,11 @@
<?php
function add_bind_var($bound_query, $type, $method, $key, $value) {
$bv = new \query\BindVariable();
$bv->setType($type);
$bv->$method($value);
$entry = new \query\BoundQuery\BindVariablesEntry();
$entry->setKey($key);
$entry->setValue($bv);
$bound_query->addBindVariables($entry);
}

4
test/java_vtgate_test_helper.py
Просмотреть файл

@ -94,7 +94,6 @@ class TestEnv(object):
utils.run_vtctl(['ApplyVSchema', '-vschema_file', self.vschema])
utils.VtGate(port=self.vtgate_port).start(
cache_ttl='500s',
rpc_error_only_in_reply=False
)
except:
self.shutdown()
@ -105,7 +104,8 @@ class TestEnv(object):
# StreamingServerShutdownIT.java expects an EOF from the vtgate
# client and not an error that vttablet killed the query (which is
# seen when vtgate is killed last).
utils.vtgate.kill()
if utils.vtgate:
utils.vtgate.kill()
tablet.kill_tablets(self.tablets)
teardown_procs = [t.teardown_mysql() for t in self.tablets]
utils.wait_procs(teardown_procs, raise_on_error=False)

1
test/python_client_test.py
Просмотреть файл

@ -45,7 +45,6 @@ def setUpModule():
args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
if protocols_flavor().service_map():
args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
args.extend(['-rpc-error-only-in-reply=true'])
vtgateclienttest_process = utils.run_bg(args)
utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)

4
test/utils.py
Просмотреть файл

@ -493,7 +493,7 @@ class VtGate(object):
def start(self, cell='test_nj', retry_delay=1, retry_count=2,
topo_impl=None, cache_ttl='1s',
timeout_total='4s', timeout_per_conn='2s',
extra_args=None, rpc_error_only_in_reply=True):
extra_args=None):
"""Starts the process for this vtgate instance.
If no other instance has been started, saves it into the global
@ -519,8 +519,6 @@ class VtGate(object):
args.extend(['-topo_implementation', topo_impl])
else:
args.extend(environment.topo_server().flags())
if rpc_error_only_in_reply:
args.extend(['-rpc-error-only-in-reply=true'])
if extra_args:
args.extend(extra_args)

15
test/vtgatev2_test.py
Просмотреть файл

@ -23,9 +23,7 @@ from vtdb import keyrange
from vtdb import keyrange_constants
from vtdb import vtdb_logger
from vtdb import vtgate_cursor
from vtdb import vtgatev2
conn_class = vtgatev2
from vtdb import vtgate_client
shard_0_master = tablet.Tablet()
shard_0_replica1 = tablet.Tablet()
@ -184,11 +182,10 @@ def setup_tablets():
utils.VtGate().start()
def get_connection(user=None, password=None, timeout=10.0):
vtgate_addrs = {'vt': [utils.vtgate.addr(),]}
def get_connection(timeout=10.0):
protocol = protocols_flavor().vtgate_python_protocol()
try:
return conn_class.connect(vtgate_addrs, timeout,
user=user, password=password)
return vtgate_client.connect(protocol, utils.vtgate.addr(), timeout)
except Exception:
logging.exception('Connection to vtgate (timeout=%s) failed.', timeout)
raise
@ -914,7 +911,9 @@ class TestFailures(BaseTestCase):
vtgate_conn.commit()
self.fail('Failed to raise DatabaseError exception')
except dbexceptions.DatabaseError:
if conn_class == vtgatev2:
# FIXME(alainjobart) add a method to get the session to vtgate_client,
# instead of poking into it like this.
if protocols_flavor().vtgate_python_protocol() == 'gorpc':
logging.info(
'SHARD SESSIONS: %s', vtgate_conn.session['ShardSessions'])
transaction_id = (