github
/
vitess-gh
зеркало из https://github.com/github/vitess-gh.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Creating a netutil library, moving some common code there. 

LGTM Sugu, Ric.
This commit is contained in:
Alain Jobart 2013-07-15 11:55:37 -07:00
Родитель bbc923036c
Коммит 7ac8dd32cc
11 изменённых файлов: 178 добавлений и 184 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
Makefile
Просмотреть файл

@ -29,6 +29,7 @@ unit_test:
cd go/bytes2; go test
cd go/cache; go test
cd go/cgzip; go test
cd go/cmd/zkns2pdns; go test
cd go/hack; go test
# cd go/logfile; go test
if [ -e "/usr/bin/memcached" ]; then \

18
go/cmd/zkns2pdns/pdns.go
Просмотреть файл

@ -17,12 +17,12 @@ import (
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"path"
"strings"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/relog"
"code.google.com/p/vitess/go/zk"
"code.google.com/p/vitess/go/zk/zkns"
@ -333,19 +333,6 @@ func (pd *pdns) Serve(r io.Reader, w io.Writer) {
}
}
func fqdn() string {
hostname, err := os.Hostname()
if err != nil {
panic(err)
}
cname, err := net.LookupCNAME(hostname)
if err != nil {
panic(err)
}
return strings.TrimRight(cname, ".")
}
func main() {
zknsDomain := flag.String("zkns-domain", "", "The naming hierarchy portion to serve")
zknsRoot := flag.String("zkns-root", "", "The root path from which to resolve")
@ -362,7 +349,8 @@ func main() {
}
zconn := zk.NewMetaConn(false)
zr1 := newZknsResolver(zconn, fqdn(), *zknsDomain, *zknsRoot)
fqdn := netutil.FullyQualifiedHostnameOrPanic()
zr1 := newZknsResolver(zconn, fqdn, *zknsDomain, *zknsRoot)
pd := &pdns{zr1}
pd.Serve(os.Stdin, os.Stdout)
os.Stdout.Close()

15
go/cmd/zkns2pdns/pdns_test.go
Просмотреть файл

@ -6,6 +6,7 @@ import (
"os"
"testing"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/zk"
"launchpad.net/gozk/zookeeper"
)
@ -41,6 +42,8 @@ const (
]}`
)
var fqdn = netutil.FullyQualifiedHostnameOrPanic()
var zconn = &TestZkConn{map[string]string{
"/zk/test/zkns/srv": fakeSRV,
"/zk/test/zkns/cname": fakeCNAME,
@ -56,10 +59,10 @@ var queries = []string{
}
var results = []string{
"OK\tzkns2pdns\nDATA\t_http.srv.zkns.test.zk\tIN\tSRV\t1\t1\t0\t0 8080 test1\nDATA\t_http.srv.zkns.test.zk\tIN\tSRV\t1\t1\t0\t0 8080 test2\nDATA\t_http.srv.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn() + " hostmaster@" + fqdn() + " 0 1800 600 3600 300\nEND\n",
"OK\tzkns2pdns\nDATA\ta.zkns.test.zk\tIN\tA\t1\t1\t0.0.0.1\nDATA\ta.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn() + " hostmaster@" + fqdn() + " 0 1800 600 3600 300\nDATA\ta.zkns.test.zk\tIN\tCNAME\t1\t1\ttest1\nEND\n",
"OK\tzkns2pdns\nDATA\tcname.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn() + " hostmaster@" + fqdn() + " 0 1800 600 3600 300\nDATA\tcname.zkns.test.zk\tIN\tCNAME\t1\t1\ttest1\nEND\n",
"OK\tzkns2pdns\nDATA\tempty.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn() + " hostmaster@" + fqdn() + " 0 1800 600 3600 300\nEND\n",
"OK\tzkns2pdns\nDATA\t_http.srv.zkns.test.zk\tIN\tSRV\t1\t1\t0\t0 8080 test1\nDATA\t_http.srv.zkns.test.zk\tIN\tSRV\t1\t1\t0\t0 8080 test2\nDATA\t_http.srv.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn + " hostmaster@" + fqdn + " 0 1800 600 3600 300\nEND\n",
"OK\tzkns2pdns\nDATA\ta.zkns.test.zk\tIN\tA\t1\t1\t0.0.0.1\nDATA\ta.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn + " hostmaster@" + fqdn + " 0 1800 600 3600 300\nDATA\ta.zkns.test.zk\tIN\tCNAME\t1\t1\ttest1\nEND\n",
"OK\tzkns2pdns\nDATA\tcname.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn + " hostmaster@" + fqdn + " 0 1800 600 3600 300\nDATA\tcname.zkns.test.zk\tIN\tCNAME\t1\t1\ttest1\nEND\n",
"OK\tzkns2pdns\nDATA\tempty.zkns.test.zk\tIN\tSOA\t1\t1\t" + fqdn + " hostmaster@" + fqdn + " 0 1800 600 3600 300\nEND\n",
"OK\tzkns2pdns\nFAIL\n",
}
@ -76,7 +79,7 @@ func testQuery(t *testing.T, query, result string) {
}
defer outpr.Close()
zr1 := newZknsResolver(zconn, fqdn(), ".zkns.test.zk", "/zk/test/zkns")
zr1 := newZknsResolver(zconn, fqdn, ".zkns.test.zk", "/zk/test/zkns")
pd := &pdns{zr1}
go func() {
pd.Serve(inpr, outpw)
@ -166,7 +169,7 @@ func (conn *TestZkConn) Close() error {
panic("Should not be used")
}
func (conn *TestZkConn) RetryChange(path string, flags int, acl []zookeeper.ACL, changeFunc ChangeFunc) error {
func (conn *TestZkConn) RetryChange(path string, flags int, acl []zookeeper.ACL, changeFunc zk.ChangeFunc) error {
panic("Should not be used")
}

140
go/netutil/netutil.go Normal file
Просмотреть файл

@ -0,0 +1,140 @@
// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
This packages contains a few utility functions for network related functions.
*/
package netutil
import (
"fmt"
"math/rand"
"net"
"os"
"sort"
"strconv"
"strings"
)
// byPriorityWeight sorts records by ascending priority and weight.
type byPriorityWeight []*net.SRV
func (s byPriorityWeight) Len() int { return len(s) }
func (s byPriorityWeight) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byPriorityWeight) Less(i, j int) bool {
return s[i].Priority < s[j].Priority ||
(s[i].Priority == s[j].Priority && s[i].Weight < s[j].Weight)
}
// shuffleByWeight shuffles SRV records by weight using the algorithm
// described in RFC 2782.
func (addrs byPriorityWeight) shuffleByWeight() {
sum := 0
for _, addr := range addrs {
sum += int(addr.Weight)
}
for sum > 0 && len(addrs) > 1 {
s := 0
n := rand.Intn(sum + 1)
for i := range addrs {
s += int(addrs[i].Weight)
if s >= n {
if i > 0 {
t := addrs[i]
copy(addrs[1:i+1], addrs[0:i])
addrs[0] = t
}
break
}
}
sum -= int(addrs[0].Weight)
addrs = addrs[1:]
}
}
func (addrs byPriorityWeight) sortRfc2782() {
sort.Sort(addrs)
i := 0
for j := 1; j < len(addrs); j++ {
if addrs[i].Priority != addrs[j].Priority {
addrs[i:j].shuffleByWeight()
i = j
}
}
addrs[i:].shuffleByWeight()
}
// SortRfc2782 reorders SRV records as specified in RFC 2782.
func SortRfc2782(srvs []*net.SRV) {
byPriorityWeight(srvs).sortRfc2782()
}
// SplitHostPort is an extension to net.SplitHostPort that also parses the
// integer port
func SplitHostPort(addr string) (string, int, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return "", 0, err
}
p, err := strconv.ParseInt(port, 10, 16)
if err != nil {
return "", 0, err
}
return host, int(p), nil
}
// FullyQualifiedHostname returns the full hostname with domain
func FullyQualifiedHostname() (string, error) {
hostname, err := os.Hostname()
if err != nil {
return "", err
}
cname, err := net.LookupCNAME(hostname)
if err != nil {
return "", err
}
return strings.TrimRight(cname, "."), nil
}
// FullyQualifiedHostnameOrPanic is the same as FullyQualifiedHostname
// but panics in case of error
func FullyQualifiedHostnameOrPanic() string {
hostname, err := FullyQualifiedHostname()
if err != nil {
panic(err)
}
return hostname
}
// ResolveAddr can resolve an address where the host has been left
// blank, like ":3306"
func ResolveAddr(addr string) (string, error) {
host, port, err := SplitHostPort(addr)
if err != nil {
return "", err
}
if host == "" {
host, err = FullyQualifiedHostname()
if err != nil {
return "", err
}
}
return fmt.Sprintf("%v:%v", host, port), nil
}
// ResolveIpAddr resolves the address:port part into an IP address:port pair
func ResolveIpAddr(addr string) (string, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return "", err
}
ipAddrs, err := net.LookupHost(host)
if err != nil {
return "", err
}
return net.JoinHostPort(ipAddrs[0], port), nil
}

24
go/vt/mysqlctl/mysqld.go
Просмотреть файл

@ -15,7 +15,6 @@ import (
"errors"
"fmt"
"io/ioutil"
"net"
"os"
"os/exec"
"path"
@ -24,6 +23,7 @@ import (
"time"
"code.google.com/p/vitess/go/mysql"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/relog"
vtenv "code.google.com/p/vitess/go/vt/env"
"code.google.com/p/vitess/go/vt/hook"
@ -378,32 +378,16 @@ func deleteTopDir(dir string) (removalErr error) {
}
func (mysqld *Mysqld) Addr() string {
// build the hostname
hostname, err := os.Hostname()
if err != nil {
panic(err) // should never happen
}
hostname, err = net.LookupCNAME(hostname)
if err != nil {
panic(err) // should never happen
}
hostname = strings.TrimRight(hostname, ".")
// and add the port
hostname := netutil.FullyQualifiedHostnameOrPanic()
return fmt.Sprintf("%v:%v", hostname, mysqld.config.MysqlPort)
}
func (mysqld *Mysqld) IpAddr() string {
addr := mysqld.Addr()
host, port, err := net.SplitHostPort(addr)
addr, err := netutil.ResolveIpAddr(mysqld.Addr())
if err != nil {
panic(err) // should never happen
}
ipAddrs, err := net.LookupHost(host)
if err != nil {
panic(err) // should never happen
}
return net.JoinHostPort(ipAddrs[0], port)
return addr
}
// executes some SQL commands using a mysql command line interface process

4
go/vt/naming/naming.go
Просмотреть файл

@ -23,8 +23,8 @@ import (
"fmt"
"net"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/relog"
"code.google.com/p/vitess/go/zk/zkns"
)
type VtnsAddr struct {
@ -102,7 +102,7 @@ func SrvEntries(addrs *VtnsAddrs, namedPort string) (srvs []*net.SRV, err error)
}
srvs = append(srvs, &net.SRV{Target: host, Port: uint16(port)})
}
zkns.Sort(srvs)
netutil.SortRfc2782(srvs)
if srvErr != nil && len(srvs) == 0 {
return nil, fmt.Errorf("SrvEntries failed: no valid endpoints found")
}

69
go/vt/tabletmanager/agent.go
Просмотреть файл

@ -16,15 +16,14 @@ package tabletmanager
import (
"flag"
"fmt"
"net"
"os"
"os/exec"
"path"
"strconv"
"strings"
"sync"
"code.google.com/p/vitess/go/jscfg"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/relog"
"code.google.com/p/vitess/go/vt/env"
"code.google.com/p/vitess/go/vt/naming"
@ -219,73 +218,21 @@ func (agent *ActionAgent) verifyServingAddrs() error {
return agent.ts.UpdateTabletEndpoint(agent.Tablet().Tablet.Cell, agent.Tablet().Keyspace, agent.Tablet().Shard, agent.Tablet().Type, addr)
}
func splitHostPort(addr string) (string, int, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return "", 0, err
}
p, err := strconv.ParseInt(port, 10, 16)
if err != nil {
return "", 0, err
}
return host, int(p), nil
}
func fqdn() (string, error) {
hostname, err := os.Hostname()
if err != nil {
return "", err
}
cname, err := net.LookupCNAME(hostname)
if err != nil {
return "", err
}
return strings.TrimRight(cname, "."), nil
}
// Resolve an address where the host has been left blank, like ":3306"
func resolveAddr(addr string) (string, error) {
host, port, err := splitHostPort(addr)
if err != nil {
return "", err
}
if host == "" {
host, err = fqdn()
if err != nil {
return "", err
}
}
return fmt.Sprintf("%v:%v", host, port), nil
}
func resolveIpAddr(addr string) (string, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return "", err
}
ipAddrs, err := net.LookupHost(host)
if err != nil {
return "", err
}
return net.JoinHostPort(ipAddrs[0], port), nil
}
func VtnsAddrForTablet(tablet *Tablet) (*naming.VtnsAddr, error) {
host, port, err := splitHostPort(tablet.Addr)
host, port, err := netutil.SplitHostPort(tablet.Addr)
if err != nil {
return nil, err
}
entry := naming.NewAddr(tablet.Uid, host, 0)
entry.NamedPortMap["_vtocc"] = port
if tablet.SecureAddr != "" {
host, port, err = splitHostPort(tablet.SecureAddr)
host, port, err = netutil.SplitHostPort(tablet.SecureAddr)
if err != nil {
return nil, err
}
entry.NamedPortMap["_vts"] = port
}
host, port, err = splitHostPort(tablet.MysqlAddr)
host, port, err = netutil.SplitHostPort(tablet.MysqlAddr)
if err != nil {
return nil, err
}
@ -304,21 +251,21 @@ func (agent *ActionAgent) Start(bindAddr, secureAddr, mysqlAddr string) error {
return err
}
bindAddr, err = resolveAddr(bindAddr)
bindAddr, err = netutil.ResolveAddr(bindAddr)
if err != nil {
return err
}
if secureAddr != "" {
secureAddr, err = resolveAddr(secureAddr)
secureAddr, err = netutil.ResolveAddr(secureAddr)
if err != nil {
return err
}
}
mysqlAddr, err = resolveAddr(mysqlAddr)
mysqlAddr, err = netutil.ResolveAddr(mysqlAddr)
if err != nil {
return err
}
mysqlIpAddr, err := resolveIpAddr(mysqlAddr)
mysqlIpAddr, err := netutil.ResolveIpAddr(mysqlAddr)
if err != nil {
return err
}

7
go/vt/tabletmanager/tablet.go
Просмотреть файл

@ -10,6 +10,7 @@ import (
"path"
"code.google.com/p/vitess/go/jscfg"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/relog"
"code.google.com/p/vitess/go/vt/key"
"code.google.com/p/vitess/go/vt/naming"
@ -86,7 +87,7 @@ func (tablet *Tablet) Json() string {
}
func (tablet *Tablet) Hostname() string {
host, _, err := splitHostPort(tablet.Addr)
host, _, err := netutil.SplitHostPort(tablet.Addr)
if err != nil {
panic(err) // should not happen, Addr was checked at creation
}
@ -135,11 +136,11 @@ func NewTablet(cell string, uid uint32, parent naming.TabletAlias, vtAddr, mysql
}
// check the values for vtAddr and mysqlAddr are correct
_, _, err := splitHostPort(vtAddr)
_, _, err := netutil.SplitHostPort(vtAddr)
if err != nil {
return nil, err
}
_, _, err = splitHostPort(mysqlAddr)
_, _, err = netutil.SplitHostPort(mysqlAddr)
if err != nil {
return nil, err
}

18
go/zk/zkctl/zkconf.go
Просмотреть файл

@ -12,13 +12,12 @@ import (
"bytes"
"fmt"
"io/ioutil"
"net"
"os"
"path"
"strconv"
"strings"
"text/template"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/vt/env"
)
@ -162,7 +161,7 @@ func MakeZkConfigFromString(cmdLine string, myId uint32) *ZkConfig {
}
zkConfig.Servers = append(zkConfig.Servers, zkServer)
}
hostname := fqdn()
hostname := netutil.FullyQualifiedHostnameOrPanic()
for _, zkServer := range zkConfig.Servers {
if (myId > 0 && myId == zkServer.ServerId) || (myId == 0 && zkServer.Hostname == hostname) {
zkConfig.ServerId = zkServer.ServerId
@ -175,16 +174,3 @@ func MakeZkConfigFromString(cmdLine string, myId uint32) *ZkConfig {
}
return zkConfig
}
func fqdn() string {
hostname, err := os.Hostname()
if err != nil {
panic(err)
}
cname, err := net.LookupCNAME(hostname)
if err != nil {
panic(err)
}
return strings.TrimRight(cname, ".")
}

61
go/zk/zkns/zkns.go
Просмотреть файл

@ -7,11 +7,10 @@ package zkns
import (
"encoding/json"
"fmt"
"math/rand"
"net"
"sort"
"strings"
"code.google.com/p/vitess/go/netutil"
"code.google.com/p/vitess/go/zk"
)
@ -71,62 +70,6 @@ func ReadAddrs(zconn zk.Conn, zkPath string) (*ZknsAddrs, error) {
return addrs, nil
}
// byPriorityWeight sorts records by ascending priority and weight.
type byPriorityWeight []*net.SRV
func (s byPriorityWeight) Len() int { return len(s) }
func (s byPriorityWeight) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byPriorityWeight) Less(i, j int) bool {
return s[i].Priority < s[j].Priority ||
(s[i].Priority == s[j].Priority && s[i].Weight < s[j].Weight)
}
// shuffleByWeight shuffles SRV records by weight using the algorithm
// described in RFC 2782.
func (addrs byPriorityWeight) shuffleByWeight() {
sum := 0
for _, addr := range addrs {
sum += int(addr.Weight)
}
for sum > 0 && len(addrs) > 1 {
s := 0
n := rand.Intn(sum + 1)
for i := range addrs {
s += int(addrs[i].Weight)
if s >= n {
if i > 0 {
t := addrs[i]
copy(addrs[1:i+1], addrs[0:i])
addrs[0] = t
}
break
}
}
sum -= int(addrs[0].Weight)
addrs = addrs[1:]
}
}
// sort reorders SRV records as specified in RFC 2782.
func (addrs byPriorityWeight) sort() {
sort.Sort(addrs)
i := 0
for j := 1; j < len(addrs); j++ {
if addrs[i].Priority != addrs[j].Priority {
addrs[i:j].shuffleByWeight()
i = j
}
}
addrs[i:].shuffleByWeight()
}
// sort reorders SRV records as specified in RFC 2782.
func Sort(srvs []*net.SRV) {
byPriorityWeight(srvs).sort()
}
// zkPath is the path to a json file in zk. It can also reference a
// named port: /zk/cell/zkns/path:_named_port
func LookupName(zconn zk.Conn, zkPath string) ([]*net.SRV, error) {
@ -153,6 +96,6 @@ func LookupName(zconn zk.Conn, zkPath string) ([]*net.SRV, error) {
}
srvs = append(srvs, srv)
}
Sort(srvs)
netutil.SortRfc2782(srvs)
return srvs, nil
}

5
py/vtdb/update_stream_service.py
Просмотреть файл

@ -60,6 +60,7 @@ class UpdateStreamResponse(object):
self.format()
def format(self):
print "AAAAAAAAAAAAAAAAAAAAAAAA", self.raw_response
if self.raw_response['Error'] == "":
self.Error = None
else:
@ -87,8 +88,8 @@ class UpdateStreamConnection(object):
except gorpc.GoRpcError as e:
raise dbexceptions.OperationalError(*e.args)
except:
logging.exception('gorpc low-level error')
except Exception as e:
logging.exception('gorpc low-level error: %s', str(e))
raise
return update_stream_response.BinlogPosition, update_stream_response.EventData, update_stream_response.Error