From b906aad2dbc312404249e961c588e1455d892348 Mon Sep 17 00:00:00 2001 From: Liang Guo Date: Fri, 8 Aug 2014 12:16:19 -0700 Subject: [PATCH 1/2] Do not use obsolete SrvKeyspace.Shards --- py/vtdb/keyspace.py | 71 ++----------------------------------------- py/vtdb/topology.py | 3 +- test/keyspace_test.py | 10 +----- 3 files changed, 4 insertions(+), 80 deletions(-) diff --git a/py/vtdb/keyspace.py b/py/vtdb/keyspace.py index 6d5282c1db..8a81cf788e 100644 --- a/py/vtdb/keyspace.py +++ b/py/vtdb/keyspace.py @@ -8,8 +8,6 @@ from vtdb import dbexceptions from vtdb import keyrange_constants -ZK_KEYSPACE_PATH = '/zk/local/vt/ns' - pack_keyspace_id = struct.Struct('!Q').pack # Represent the SrvKeyspace object from the toposerver, and provide functions @@ -22,11 +20,6 @@ class Keyspace(object): sharding_col_type = None served_from = None - shard_count = None - shard_max_keys = None # Sorted list of the max keys for each shard. - shard_names = None # Sorted list of shard names - - # these will match the order of shard_max_keys. - # load this object from a SrvKeyspace object generated by vt def __init__(self, name, data): self.name = name @@ -35,13 +28,6 @@ class Keyspace(object): self.sharding_col_name = data.get('ShardingColumnName', "") self.sharding_col_type = data.get('ShardingColumnType', keyrange_constants.KIT_UNSET) self.served_from = data.get('ServedFrom', None) - self.shard_count = len(data['Shards']) - # if we have real values for shards and KeyRange.End, grab them - if self.shard_count > 1 and data['Shards'][0]['KeyRange']['End'] != "": - self.shard_max_keys = [shard['KeyRange']['End'] - for shard in data['Shards']] - # We end up needing the name for addressing so compute this once. - self.shard_names = self._make_shard_names() def get_shards(self, db_type): if not db_type: @@ -73,36 +59,15 @@ class Keyspace(object): shard_max_keys = self.get_shard_max_keys(db_type) if len(shard_max_keys) == 1 and shard_max_keys[0] == keyrange_constants.MAX_KEY: return [keyrange_constants.SHARD_ZERO,] - for i, max_key in enumerate(self.shard_max_keys): + for i, max_key in enumerate(shard_max_keys): min_key = keyrange_constants.MIN_KEY if i > 0: - min_key = self.shard_max_keys[i-1] + min_key = shard_max_keys[i-1] shard_name = '%s-%s' % (min_key.encode('hex').upper(), max_key.encode('hex').upper()) names.append(shard_name) return names - def keyspace_id_to_shard_index_for_db_type(self, keyspace_id, db_type): - if not keyspace_id: - raise ValueError('keyspace_id is not set') - if not db_type: - raise ValueError('db_type is not set') - # Pack this into big-endian and do a byte-wise comparison. - pkid = pack_keyspace_id(keyspace_id) - shard_max_keys = self.get_shard_max_keys(db_type) - if not shard_max_keys: - raise ValueError('Keyspace is not range sharded', self.name) - for shard_index, shard_max in enumerate(shard_max_keys): - if pkid < shard_max: - break - # shard_names = self.get_shard_names(db_type) - # logging.info('Keyspace resolved %s %s %s %s %s %s', keyspace_id, - # pkid.encode('hex').upper(), shard_index, - # shard_names[shard_index], - # [x.encode('hex').upper() for x in shard_max_keys], - # shard_names) - return shard_index - def keyspace_id_to_shard_name_for_db_type(self, keyspace_id, db_type): if not keyspace_id: raise ValueError('keyspace_id is not set') @@ -119,38 +84,6 @@ class Keyspace(object): break return shard_names[shard_index] - def keyspace_id_to_shard_index(self, keyspace_id): - if not keyspace_id: - raise ValueError('keyspace_id is not set') - # Pack this into big-endian and do a byte-wise comparison. - pkid = pack_keyspace_id(keyspace_id) - if not self.shard_max_keys: - raise ValueError('Keyspace is not range sharded', self.name) - for shard_index, shard_max in enumerate(self.shard_max_keys): - if pkid < shard_max: - break - # logging.info('Keyspace resolved %s %s %s %s %s %s', keyspace_id, - # pkid.encode('hex').upper(), shard_index, - # self.shard_names[shard_index], - # [x.encode('hex').upper() for x in self.shard_max_keys], - # self.shard_names) - return shard_index - - def _make_shard_names(self): - names = [] - if self.shard_max_keys: - for i, max_key in enumerate(self.shard_max_keys): - min_key = keyrange_constants.MIN_KEY - if i > 0: - min_key = self.shard_max_keys[i-1] - shard_name = '%s-%s' % (min_key.encode('hex').upper(), - max_key.encode('hex').upper()) - names.append(shard_name) - else: - # Handle non-range shards - names = [str(x) for x in xrange(self.shard_count)] - return names - def read_keyspace(topo_client, keyspace_name): try: diff --git a/py/vtdb/topology.py b/py/vtdb/topology.py index ad72b98dc1..a3faa11fe2 100644 --- a/py/vtdb/topology.py +++ b/py/vtdb/topology.py @@ -170,9 +170,8 @@ def is_sharded_keyspace(keyspace_name, db_type): shard_count = ks.get_shard_count(db_type) return shard_count > 1 -def get_keyrange_from_shard_name(keyspace, shard_name, db_type='replica'): +def get_keyrange_from_shard_name(keyspace, shard_name, db_type): kr = None - # db_type is immaterial here. if not is_sharded_keyspace(keyspace, db_type): if shard_name == keyrange_constants.SHARD_ZERO: kr = keyrange_constants.NON_PARTIAL_KEYRANGE diff --git a/test/keyspace_test.py b/test/keyspace_test.py index f5c3bfea0c..bcd4008b62 100755 --- a/test/keyspace_test.py +++ b/test/keyspace_test.py @@ -23,7 +23,7 @@ from zk import zkocc SHARDED_KEYSPACE = "TEST_KEYSPACE_SHARDED" UNSHARDED_KEYSPACE = "TEST_KEYSPACE_UNSHARDED" -# shards for SHARDED_KEYSPACE +# shards for SHARDED_KEYSPACE # range "" - 80 shard_0_master = tablet.Tablet() shard_0_replica = tablet.Tablet() @@ -250,14 +250,6 @@ class TestKeyspace(unittest.TestCase): unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE) self.assertEqual(set(unsharded_ks.db_types), set(ALL_DB_TYPES)) - - def test_keyspace_id_to_shard_index(self): - sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) - for i, sn in enumerate(shard_names): - for keyspace_id in shard_kid_map[sn]: - self.assertEqual(sharded_ks.keyspace_id_to_shard_index(keyspace_id), i) - self.assertEqual(sharded_ks.keyspace_id_to_shard_index_for_db_type(keyspace_id, 'master'), i) - def test_keyspace_id_to_shard_name(self): sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) for _, sn in enumerate(shard_names): From 88075960c8c5b7a4f9bed49ec030ae379c3d1af2 Mon Sep 17 00:00:00 2001 From: Liang Guo Date: Fri, 8 Aug 2014 14:11:35 -0700 Subject: [PATCH 2/2] Fix keyspace_test --- test/keyspace_test.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/keyspace_test.py b/test/keyspace_test.py index bcd4008b62..6e5c46f14b 100755 --- a/test/keyspace_test.py +++ b/test/keyspace_test.py @@ -213,34 +213,27 @@ class TestKeyspace(unittest.TestCase): def test_shard_count(self): sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) - self.assertEqual(sharded_ks.shard_count, 2) for db_type in ALL_DB_TYPES: self.assertEqual(sharded_ks.get_shard_count(db_type), 2) unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE) - self.assertEqual(unsharded_ks.shard_count, 1) for db_type in ALL_DB_TYPES: self.assertEqual(unsharded_ks.get_shard_count(db_type), 1) def test_shard_names(self): sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) - self.assertEqual(sharded_ks.shard_names, ['-80', '80-']) for db_type in ALL_DB_TYPES: self.assertEqual(sharded_ks.get_shard_names(db_type), ['-80', '80-']) unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE) - self.assertEqual(unsharded_ks.shard_names, ['0']) for db_type in ALL_DB_TYPES: self.assertEqual(unsharded_ks.get_shard_names(db_type), ['0']) def test_shard_max_keys(self): sharded_ks = self._read_keyspace(SHARDED_KEYSPACE) want = ['80', ''] - for i, smk in enumerate(sharded_ks.shard_max_keys): - self.assertEqual(smk.encode('hex').upper(), want[i]) for db_type in ALL_DB_TYPES: for i, smk in enumerate(sharded_ks.get_shard_max_keys(db_type)): self.assertEqual(smk.encode('hex').upper(), want[i]) unsharded_ks = self._read_keyspace(UNSHARDED_KEYSPACE) - self.assertEqual(unsharded_ks.shard_max_keys, None) for db_type in ALL_DB_TYPES: self.assertEqual(unsharded_ks.get_shard_max_keys(db_type), [''])