Remove references to l2vtgates in docs and integration tests

Signed-off-by: Rafael Chacon <rafael@slack-corp.com>

doc/TabletRouting.md

@@ -93,36 +93,7 @@ There are two implementations of the Gateway interface:
   discovery section, one per cell) as a source of tablets, a HealthCheck module
   to watch their health, and a TabletStatsCache to collect all the health
   information. Based on this data, it can find the best tablet to use.
-* l2VTGateGateway: It keeps a map of l2vtgate processes to send queries to. See
-  next section for more details.
   
-## l2vtgate
-
-As we started increasing the number of tablets in a cell, it became clear that a
-bottleneck of the system was going to be how many tablets a single vtgate is
-connecting to. Since vtgate maintains a streaming health check connection per
-tablet, the number of these connections can grow to large numbers. It is common
-for vtgate to watch tablets in other cells, to be able to find the master
-tablet.
-
-So l2vtgate came to exist, based on very similar concepts and interfaces:
-
-* l2vtgate is an extra hop between a vtgate pool and tablets.
-* A l2vtgate pool connects to a subset of tablets, therefore it can have a
-  reasonable number of streaming health connections. Externally, it exposes the
-  QueryService RPC interface (that has the Target for the query, keyspace /
-  shard / tablet type). Internally, it uses a discoveryGateway, as usual.
-* vtgate connects to l2vtgate pools (using the l2VTGateGateway instead of the
-  discoveryGateway). It has a map of which keyspace / shard / tablet type needs
-  to go to wich l2vtgate pool. At this point, vtgate doesn't maintain any health
-  information about the tablets, it lets l2vtgate handle it.
-
-Note l2vtgate is not an ideal solution as it is now. For instance, if there are
-two cells, and the master for a shard can be in either, l2vtgate still has to
-watch the tablets in both cells, to know where the master is. Ideally, we'd want
-l2vtgate to be collocated with the tablets in a given cell, and not go
-cross-cell.
-
 # Extensions, work in progress
 
 ## Regions, cross-cell targeting
@@ -169,31 +140,6 @@ between vtgate and l2vtgate:
 This would also be a good time to merge the vtgate code that uses the VSchema
 with the code that doesn't for SrvKeyspace access.
 
-## Hybrid Gateway
-
-It would be nice to re-organize the code a bit inside vtgate to allow for an
-hybrid gateway, and get rid of l2vtgate alltogether:
-
-* vtgate would use the discoveryGateway to watch the tablets in the current cell
-  (and optionally to any other cell we still want to consider local).
-* vtgate would use l2vtgateGateway to watch the tablets in a different cell.
-* vtgate would expose the RPC APIs currently exposed by the l2vtgate process.
-
-So vtgate would watch the tablets in the local cell only, but also know what
-healthy tablets are in the other cells, and be able to send query to them
-through their vtgate. The extra hop to the other cell vtgate should be a small
-latency price to pay, compared to going cross-cell already.
-
-So queries would go one of two routes:
-
-* client(cell1) -> vtgate(cell1) -> tablet(cell1)
-* client(cell1) -> vtgate(cell1) -> vtgate(cell2) -> tablet(cell2)
-
-If the number of tablets in a given cell is still too high for the local vtgate
-pool, two or more pools can still be created, each of them knowing about a
-subset of the tablets. And they would just forward queries to each others when
-addressing the other tablet set.
-
 ## Config-based routing
 
 Another possible extension would be to group all routing options for vtgate in a

go/vt/srvtopo/target_stats.go

@@ -27,9 +27,6 @@ import (
 // routing of queries.
 // - discovery.TabletStatsCache will implement the discovery part of the
 //   interface, and discoverygateway will have the QueryService.
-// - hybridgateway will also implement this interface: for each l2vtgate pool,
-//   it will establish a StreamHealth connection, and store the returned
-//   health stats.
 type TargetStats interface {
 	// GetAggregateStats returns the aggregate stats for the given Target.
 	// The srvtopo module will use that information to route queries

go/vt/vtgate/vtgate.go

@@ -61,7 +61,6 @@ var (
 	queryPlanCacheSize  = flag.Int64("gate_query_cache_size", 10000, "gate server query cache size, maximum number of queries to be cached. vtgate analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.")
 	legacyAutocommit    = flag.Bool("legacy_autocommit", false, "DEPRECATED: set this flag to true to get the legacy behavior: all transactions will need an explicit begin, and DMLs outside transactions will return an error.")
 	enableForwarding    = flag.Bool("enable_forwarding", false, "if specified, this process will also expose a QueryService interface that allows other vtgates to talk through this vtgate to the underlying tablets.")
-	l2vtgateAddrs       flagutil.StringListValue
 	disableLocalGateway = flag.Bool("disable_local_gateway", false, "if specified, this process will not route any queries to local tablets in the local cell")
 )
 
@@ -1170,7 +1169,3 @@ func unambiguousKeyspaceBSQ(queries []*vtgatepb.BoundShardQuery) string {
 		return keyspace
 	}
 }
-
-func init() {
-	flag.Var(&l2vtgateAddrs, "l2vtgate_addrs", "Specifies a comma-separated list of other l2 vtgate pools to connect to. These other vtgates must run with the --enable_forwarding flag")
-}

test/config.json

@@ -90,17 +90,6 @@
 				"worker_test"
 			]
 		},
-		"initial_sharding_l2vtgate": {
-			"File": "initial_sharding_l2vtgate.py",
-			"Args": [],
-			"Command": [],
-			"Manual": false,
-			"Shard": 2,
-			"RetryMax": 0,
-			"Tags": [
-				"worker_test"
-			]
-		},
 		"legacy_resharding": {
 			"File": "legacy_resharding.py",
 			"Args": [],
@@ -426,17 +415,6 @@
 				"site_test"
 			]
 		},
-		"vtgatev2_l2vtgate": {
-			"File": "vtgatev2_l2vtgate_test.py",
-			"Args": [],
-			"Command": [],
-			"Manual": false,
-			"Shard": 1,
-			"RetryMax": 0,
-			"Tags": [
-				"site_test"
-			]
-		},
 		"vtgatev3": {
 			"File": "vtgatev3_test.py",
 			"Args": [],

test/initial_sharding.py

@@ -35,16 +35,6 @@ import environment
 import tablet
 import utils
 
-# use_l2vtgate is set if we want to use l2vtgate processes.
-# We'll set them up to have:
-# l2vtgate1: covers the initial shard, and -80
-# l2vtgate2: covers 80-
-use_l2vtgate = False
-
-# the l2vtgate processes, if applicable
-l2vtgate1 = None
-l2vtgate2 = None
-
 # initial shard, covers everything
 shard_master = tablet.Tablet()
 shard_replica = tablet.Tablet()
@@ -217,8 +207,6 @@ index by_msg (msg)
                         should_be_here=False)
 
   def test_resharding(self):
-    global l2vtgate1, l2vtgate2
-
     # create the keyspace with just one shard
     shard_master.init_tablet(
         'replica',
@@ -280,33 +268,12 @@ index by_msg (msg)
     # We must start vtgate after tablets are up, or else wait until 1min refresh
     # (that is the tablet_refresh_interval parameter for discovery gateway)
     # we want cache_ttl at zero so we re-read the topology for every test query.
-    if use_l2vtgate:
-      l2vtgate1 = utils.VtGate()
-      l2vtgate1.start(extra_args=['--enable_forwarding'], tablets=
-                      [shard_master, shard_replica, shard_rdonly1])
-      l2vtgate1.wait_for_endpoints('test_keyspace.0.master', 1)
-      l2vtgate1.wait_for_endpoints('test_keyspace.0.replica', 1)
-      l2vtgate1.wait_for_endpoints('test_keyspace.0.rdonly', 1)
 
-      _, l2vtgate1_addr = l2vtgate1.rpc_endpoint()
+    utils.VtGate().start(cache_ttl='0', tablets=[
-
+        shard_master, shard_replica, shard_rdonly1])
-      # Clear utils.vtgate, so it doesn't point to the previous l2vtgate1.
+    utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
-      utils.vtgate = None
+    utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
-      utils.VtGate().start(cache_ttl='0', l2vtgates=[l2vtgate1_addr,],
+    utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1)
-                           extra_args=['-disable_local_gateway'])
-      utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1,
-                                      var='L2VtgateConnections')
-      utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1,
-                                      var='L2VtgateConnections')
-      utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1,
-                                      var='L2VtgateConnections')
-
-    else:
-      utils.VtGate().start(cache_ttl='0', tablets=[
-          shard_master, shard_replica, shard_rdonly1])
-      utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1)
-      utils.vtgate.wait_for_endpoints('test_keyspace.0.replica', 1)
-      utils.vtgate.wait_for_endpoints('test_keyspace.0.rdonly', 1)
 
     # check the Map Reduce API works correctly, should use ExecuteShards,
     # as we're not sharded yet.
@@ -391,62 +358,13 @@ index by_msg (msg)
     # must restart vtgate after tablets are up, or else wait until 1min refresh
     # we want cache_ttl at zero so we re-read the topology for every test query.
     utils.vtgate.kill()
-    if use_l2vtgate:
-      l2vtgate1.kill()
 
-      l2vtgate1 = utils.VtGate()
+    utils.vtgate = None
-      l2vtgate1.start(extra_args=['--enable_forwarding',
+    utils.VtGate().start(cache_ttl='0', tablets=[
-                                  '-tablet_filters',
+        shard_master, shard_replica, shard_rdonly1,
-                                  'test_keyspace|0,test_keyspace|-80'],
+        shard_0_master, shard_0_replica, shard_0_rdonly1,
-                      tablets=[shard_master, shard_replica, shard_rdonly1,
+        shard_1_master, shard_1_replica, shard_1_rdonly1])
-                               shard_0_master, shard_0_replica,
+    var = None
-                               shard_0_rdonly1])
-      l2vtgate1.wait_for_endpoints('test_keyspace.0.master', 1)
-      l2vtgate1.wait_for_endpoints('test_keyspace.0.replica', 1)
-      l2vtgate1.wait_for_endpoints('test_keyspace.0.rdonly', 1)
-      l2vtgate1.wait_for_endpoints('test_keyspace.-80.master', 1)
-      l2vtgate1.wait_for_endpoints('test_keyspace.-80.replica', 1)
-      l2vtgate1.wait_for_endpoints('test_keyspace.-80.rdonly', 1)
-      l2vtgate1.verify_no_endpoint('test_keyspace.80-.master')
-      l2vtgate1.verify_no_endpoint('test_keyspace.80-.replica')
-      l2vtgate1.verify_no_endpoint('test_keyspace.80-.rdonly')
-
-      # FIXME(alainjobart) we clear tablet_types_to_wait, as this
-      # l2vtgate2 doesn't serve the current test_keyspace shard, which
-      # is test_keyspace.0. This is not ideal, we should re-work
-      # which keyspace/shard a l2vtgate can wait for, as the ones
-      # filtered by tablet_filters.
-      l2vtgate2 = utils.VtGate()
-      l2vtgate2.start(extra_args=['--enable_forwarding',
-                                  '-tablet_filters',
-                                  'test_keyspace|80-'], tablets=
-                      [shard_1_master, shard_1_replica, shard_1_rdonly1],
-                      tablet_types_to_wait='')
-      l2vtgate2.wait_for_endpoints('test_keyspace.80-.master', 1)
-      l2vtgate2.wait_for_endpoints('test_keyspace.80-.replica', 1)
-      l2vtgate2.wait_for_endpoints('test_keyspace.80-.rdonly', 1)
-      l2vtgate2.verify_no_endpoint('test_keyspace.0.master')
-      l2vtgate2.verify_no_endpoint('test_keyspace.0.replica')
-      l2vtgate2.verify_no_endpoint('test_keyspace.0.rdonly')
-      l2vtgate2.verify_no_endpoint('test_keyspace.-80.master')
-      l2vtgate2.verify_no_endpoint('test_keyspace.-80.replica')
-      l2vtgate2.verify_no_endpoint('test_keyspace.-80.rdonly')
-
-      _, l2vtgate1_addr = l2vtgate1.rpc_endpoint()
-      _, l2vtgate2_addr = l2vtgate2.rpc_endpoint()
-      utils.vtgate = None
-      utils.VtGate().start(cache_ttl='0', l2vtgates=[l2vtgate1_addr,
-                                                     l2vtgate2_addr,],
-                           extra_args=['-disable_local_gateway'])
-      var = 'L2VtgateConnections'
-
-    else:
-      utils.vtgate = None
-      utils.VtGate().start(cache_ttl='0', tablets=[
-          shard_master, shard_replica, shard_rdonly1,
-          shard_0_master, shard_0_replica, shard_0_rdonly1,
-          shard_1_master, shard_1_replica, shard_1_rdonly1])
-      var = None
 
     # Wait for the endpoints, either local or remote.
     utils.vtgate.wait_for_endpoints('test_keyspace.0.master', 1, var=var)
@@ -626,12 +544,9 @@ index by_msg (msg)
     # make sure rdonly tablets are back to serving before hitting vtgate.
     for t in [shard_0_rdonly1, shard_1_rdonly1]:
       t.wait_for_vttablet_state('SERVING')
-    if use_l2vtgate:
+
-      l2vtgate1.wait_for_endpoints('test_keyspace.-80.rdonly', 1)
+    utils.vtgate.wait_for_endpoints('test_keyspace.-80.rdonly', 1)
-      l2vtgate2.wait_for_endpoints('test_keyspace.80-.rdonly', 1)
+    utils.vtgate.wait_for_endpoints('test_keyspace.80-.rdonly', 1)
-    else:
-      utils.vtgate.wait_for_endpoints('test_keyspace.-80.rdonly', 1)
-      utils.vtgate.wait_for_endpoints('test_keyspace.80-.rdonly', 1)
 
     # check the Map Reduce API works correctly, should use ExecuteKeyRanges
     # on both destination shards now.

test/initial_sharding_l2vtgate.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Re-runs initial_sharding.py with a l2vtgate process."""
-
-import initial_sharding
-import utils
-
-if __name__ == '__main__':
-  initial_sharding.use_l2vtgate = True
-  utils.main(initial_sharding)

test/vtgatev2_l2vtgate_test.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Re-runs vtgatev2_test.py with a l2vtgate process."""
-
-import utils
-import vtgatev2_test
-
-# This test is just re-running an entire vtgatev2_test.py with a
-# l2vtgate process in the middle.
-if __name__ == '__main__':
-  vtgatev2_test.use_l2vtgate = True
-  utils.main(vtgatev2_test)

test/vtgatev2_test.py

@@ -35,16 +35,6 @@ from vtdb import vtdb_logger
 from vtdb import vtgate_client
 from vtdb import vtgate_cursor
 
-# use_l2vtgate controls if we're adding a l2vtgate process in between
-# vtgate and the tablets.
-use_l2vtgate = False
-
-# l2vtgate is the L2VTGate object, if any
-l2vtgate = None
-
-# l2vtgate_addr is the address of the l2vtgate to send to vtgate
-l2vtgate_addr = None
-
 shard_0_master = tablet.Tablet()
 shard_0_replica1 = tablet.Tablet()
 shard_0_replica2 = tablet.Tablet()
@@ -154,8 +144,6 @@ def tearDownModule():
   logging.debug('Tearing down the servers and setup')
   if utils.vtgate:
     utils.vtgate.kill()
-  if l2vtgate:
-    l2vtgate.kill()
   tablet.kill_tablets([shard_0_master,
                        shard_0_replica1, shard_0_replica2,
                        shard_1_master,
@@ -184,7 +172,6 @@ def tearDownModule():
 
 def setup_tablets():
   """Start up a master mysql and vttablet."""
-  global l2vtgate, l2vtgate_addr
 
   logging.debug('Setting up tablets')
   utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
@@ -252,46 +239,22 @@ def setup_tablets():
       'Partitions(rdonly): -80 80-\n'
       'Partitions(replica): -80 80-\n')
 
-  if use_l2vtgate:
-    l2vtgate = utils.VtGate()
-    l2vtgate.start(extra_args=['--enable_forwarding'], tablets=
-                   [shard_0_master, shard_0_replica1, shard_0_replica2,
-                    shard_1_master, shard_1_replica1, shard_1_replica2])
-    _, l2vtgate_addr = l2vtgate.rpc_endpoint()
 
-    # Clear utils.vtgate, so it doesn't point to the previous l2vtgate.
+  utils.VtGate().start(tablets=
-    utils.vtgate = None
+                       [shard_0_master, shard_0_replica1, shard_0_replica2,
-
+                        shard_1_master, shard_1_replica1, shard_1_replica2])
-    # This vgate doesn't watch any local tablets, so we disable_local_gateway.
-    utils.VtGate().start(l2vtgates=[l2vtgate_addr,],
-                         extra_args=['-disable_local_gateway'])
-
-  else:
-    utils.VtGate().start(tablets=
-                         [shard_0_master, shard_0_replica1, shard_0_replica2,
-                          shard_1_master, shard_1_replica1, shard_1_replica2])
 
   wait_for_all_tablets()
 
 
 def restart_vtgate(port):
-  if use_l2vtgate:
+  utils.VtGate(port=port).start(
-    utils.VtGate(port=port).start(l2vtgates=[l2vtgate_addr,],
+      tablets=[shard_0_master, shard_0_replica1, shard_0_replica2,
-                                  extra_args=['-disable_local_gateway'])
+               shard_1_master, shard_1_replica1, shard_1_replica2])
-  else:
-    utils.VtGate(port=port).start(
-        tablets=[shard_0_master, shard_0_replica1, shard_0_replica2,
-                 shard_1_master, shard_1_replica1, shard_1_replica2])
 
 
 def wait_for_endpoints(name, count):
-  if use_l2vtgate:
+  utils.vtgate.wait_for_endpoints(name, count)
-    # Wait for the l2vtgate to have a healthy connection.
-    l2vtgate.wait_for_endpoints(name, count)
-    # Also wait for vtgate to have received the remote healthy connection.
-    utils.vtgate.wait_for_endpoints(name, count, var='L2VtgateConnections')
-  else:
-    utils.vtgate.wait_for_endpoints(name, count)
 
 
 def wait_for_all_tablets():
@@ -411,17 +374,12 @@ class TestCoreVTGateFunctions(BaseTestCase):
         self.assertIn(kid, SHARD_KID_MAP[SHARD_NAMES[shard_index]])
 
     # Do a cross shard range query and assert all rows are fetched.
-    # Use this test to also test the vtgate vars (and l2vtgate vars if
+    # Use this test to also test the vtgate vars are correctly updated.
-    # applicable) are correctly updated.
     v = utils.vtgate.get_vars()
     key0 = 'Execute.' + KEYSPACE_NAME + '.' + SHARD_NAMES[0] + '.master'
     key1 = 'Execute.' + KEYSPACE_NAME + '.' + SHARD_NAMES[1] + '.master'
     before0 = v['VttabletCall']['Histograms'][key0]['Count']
     before1 = v['VttabletCall']['Histograms'][key1]['Count']
-    if use_l2vtgate:
-      lv = l2vtgate.get_vars()
-      lbefore0 = lv['QueryServiceCall']['Histograms'][key0]['Count']
-      lbefore1 = lv['QueryServiceCall']['Histograms'][key1]['Count']
 
     cursor = vtgate_conn.cursor(
         tablet_type='master', keyspace=KEYSPACE_NAME,
@@ -435,12 +393,6 @@ class TestCoreVTGateFunctions(BaseTestCase):
     after1 = v['VttabletCall']['Histograms'][key1]['Count']
     self.assertEqual(after0 - before0, 1)
     self.assertEqual(after1 - before1, 1)
-    if use_l2vtgate:
-      lv = l2vtgate.get_vars()
-      lafter0 = lv['QueryServiceCall']['Histograms'][key0]['Count']
-      lafter1 = lv['QueryServiceCall']['Histograms'][key1]['Count']
-      self.assertEqual(lafter0 - lbefore0, 1)
-      self.assertEqual(lafter1 - lbefore1, 1)
 
   def test_rollback(self):
     vtgate_conn = get_connection()