Merge pull request #907 from thompsonja/k8s_multicell

Update k8s scripts to allow for multiple cells

examples/kubernetes/etcd-down.sh

@@ -8,8 +8,11 @@ set -e
 script_root=`dirname "${BASH_SOURCE}"`
 source $script_root/env.sh
 
+CELLS=${CELLS:-'test'}
+cells=`echo $CELLS | tr ',' ' '`
+
 # Delete replication controllers
-for cell in 'global' 'test'; do
+for cell in 'global' $cells; do
   echo "Stopping etcd replicationcontroller for $cell cell..."
   $KUBECTL stop replicationcontroller etcd-$cell
 

examples/kubernetes/etcd-up.sh

@@ -15,7 +15,10 @@ source $script_root/env.sh
 
 replicas=${ETCD_REPLICAS:-3}
 
-for cell in 'global' 'test'; do
+CELLS=${CELLS:-'test'}
+cells=`echo $CELLS | tr ',' ' '`
+
+for cell in 'global' $cells; do
   # Generate a discovery token.
   echo "Generating discovery token for $cell cell..."
   discovery=$(curl -sL https://discovery.etcd.io/new?size=$replicas)

examples/kubernetes/vitess-down.sh

@@ -3,9 +3,10 @@
 GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME:-'example'}
 SHARDS=${SHARDS:-'0'}
 TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-3}
+CELLS=${CELLS:-'test'}
 
 ./vtgate-down.sh
-SHARDS=$SHARDS TABLETS_PER_SHARD=$TABLETS_PER_SHARD ./vttablet-down.sh
+SHARDS=$SHARDS CELLS=$CELLS TABLETS_PER_SHARD=$TABLETS_PER_SHARD ./vttablet-down.sh
 ./vtctld-down.sh
 ./etcd-down.sh
 

examples/kubernetes/vitess-up.sh

@@ -23,6 +23,10 @@ VTTABLET_TEMPLATE=${VTTABLET_TEMPLATE:-'vttablet-pod-benchmarking-template.yaml'
 VTGATE_TEMPLATE=${VTGATE_TEMPLATE:-'vtgate-controller-benchmarking-template.yaml'}
 VTGATE_COUNT=${VTGATE_COUNT:-0}
 VTDATAROOT_VOLUME=${VTDATAROOT_VOLUME:-''}
+CELLS=${CELLS:-'test'}
+
+cells=`echo $CELLS | tr ',' ' '`
+num_cells=`echo $cells | wc -w`
 
 # Get region from zone (everything to last dash)
 gke_region=`echo $GKE_ZONE | sed "s/-[^-]*$//"`
@@ -81,7 +85,7 @@ fi
 export KUBECTL='kubectl'
 go get github.com/youtube/vitess/go/cmd/vtctlclient
 num_shards=`echo $SHARDS | tr "," " " | wc -w`
-total_tablet_count=$(($num_shards*$TABLETS_PER_SHARD))
+total_tablet_count=$(($num_shards*$TABLETS_PER_SHARD*$num_cells))
 vtgate_count=$VTGATE_COUNT
 if [ $vtgate_count -eq 0 ]; then
   vtgate_count=$(($total_tablet_count/4>3?$total_tablet_count/4:3))
@@ -94,14 +98,17 @@ echo "*  Shards: $SHARDS"
 echo "*  Tablets per shard: $TABLETS_PER_SHARD"
 echo "*  Rdonly per shard: $RDONLY_COUNT"
 echo "*  VTGate count: $vtgate_count"
+echo "*  Cells: $cells"
 echo "****************************"
 
-echo 'Running etcd-up.sh' && ./etcd-up.sh
+echo 'Running etcd-up.sh' && CELLS=$CELLS ./etcd-up.sh
 wait_for_running_tasks etcd-global 3
-wait_for_running_tasks etcd-test 3
+for cell in $cells; do
+  wait_for_running_tasks etcd-$cell 3
+done
 
 echo 'Running vtctld-up.sh' && ./vtctld-up.sh
-echo 'Running vttablet-up.sh' && ./vttablet-up.sh
+echo 'Running vttablet-up.sh' && CELLS=$CELLS ./vttablet-up.sh
 echo 'Running vtgate-up.sh' && ./vtgate-up.sh
 
 wait_for_running_tasks vtctld 1
@@ -118,7 +125,10 @@ kvtctl="$GOPATH/bin/vtctlclient -server $vtctl_server"
 echo Waiting for tablets to be visible in the topology
 counter=0
 while [ $counter -lt $MAX_VTTABLET_TOPO_WAIT_RETRIES ]; do
-  num_tablets=`$kvtctl ListAllTablets test | wc -l`
+  num_tablets=0
+  for cell in $cells; do
+    num_tablets=$(($num_tablets+`$kvtctl ListAllTablets $cell | wc -l`))
+  done
   echo -en "\r$num_tablets out of $total_tablet_count in topology..."
   if [ $num_tablets -eq $total_tablet_count ]
   then
@@ -141,7 +151,7 @@ if [ $split_shard_count -eq 1 ]; then
   split_shard_count=0
 fi
 
-echo -n Setting Keyspace Sharding Info... 
+echo -n Setting Keyspace Sharding Info...
 $kvtctl SetKeyspaceShardingInfo -force -split_shard_count $split_shard_count test_keyspace keyspace_id uint64
 echo Done
 echo -n Rebuilding Keyspace Graph...
@@ -150,7 +160,7 @@ echo Done
 echo -n Reparenting...
 shard_num=1
 for shard in $(echo $SHARDS | tr "," " "); do
-  $kvtctl InitShardMaster -force test_keyspace/$shard test-0000000${shard_num}00
+  $kvtctl InitShardMaster -force test_keyspace/$shard `echo $cells | awk '{print $1}'`-0100000${shard_num}00
   let shard_num=shard_num+1
 done
 echo Done

examples/kubernetes/vttablet-down.sh

@@ -11,7 +11,7 @@ source $script_root/env.sh
 server=$(get_vtctld_addr)
 
 # Delete the pods for all shards
-cell='test'
+CELLS=${CELLS:-'test'}
 keyspace='test_keyspace'
 SHARDS=${SHARDS:-'0'}
 TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-5}
@@ -21,18 +21,22 @@ num_shards=`echo $SHARDS | tr "," " " | wc -w`
 uid_base=$UID_BASE
 
 for shard in `seq 1 $num_shards`; do
-  for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do
-    uid=$[$uid_base + $uid_index]
-    printf -v alias '%s-%010d' $cell $uid
+  cell_index=0
+  for cell in `echo $CELLS | tr "," " "`; do
+    for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do
+      uid=$[$uid_base + $uid_index + $cell_index]
+      printf -v alias '%s-%010d' $cell $uid
 
-    if [ -n "$server" ]; then
-      echo "Removing tablet $alias from Vitess topology..."
-      vtctlclient -server $server ScrapTablet -force $alias
-      vtctlclient -server $server DeleteTablet $alias
-    fi
+      if [ -n "$server" ]; then
+        echo "Removing tablet $alias from Vitess topology..."
+        vtctlclient -server $server ScrapTablet -force $alias
+        vtctlclient -server $server DeleteTablet $alias
+      fi
 
-    echo "Deleting pod for tablet $alias..."
-    $KUBECTL delete pod vttablet-$uid
+      echo "Deleting pod for tablet $alias..."
+      $KUBECTL delete pod vttablet-$cell-$uid
+    done
+    let cell_index=cell_index+100000000
   done
   let uid_base=uid_base+100
 done

examples/kubernetes/vttablet-up.sh

@@ -8,7 +8,7 @@ script_root=`dirname "${BASH_SOURCE}"`
 source $script_root/env.sh
 
 # Create the pods for shard-0
-cell='test'
+CELLS=${CELLS:-'test'}
 keyspace='test_keyspace'
 SHARDS=${SHARDS:-'0'}
 TABLETS_PER_SHARD=${TABLETS_PER_SHARD:-5}
@@ -23,36 +23,37 @@ if [ -n "$VTDATAROOT_VOLUME" ]; then
   vtdataroot_volume="hostDir: {path: ${VTDATAROOT_VOLUME}}"
 fi
 
-index=1
 uid_base=$UID_BASE
 for shard in $(echo $SHARDS | tr "," " "); do
-  echo "Creating $keyspace.shard-$shard pods in cell $cell..."
-  for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do
-    uid=$[$uid_base + $uid_index]
-    printf -v alias '%s-%010d' $cell $uid
-    printf -v tablet_subdir 'vt_%010d' $uid
+  cell_index=0
+  for cell in `echo $CELLS | tr ',' ' '`; do
+    echo "Creating $keyspace.shard-$shard pods in cell $CELL..."
+    for uid_index in `seq 0 $(($TABLETS_PER_SHARD-1))`; do
+      uid=$[$uid_base + $uid_index + $cell_index]
+      printf -v alias '%s-%010d' $cell $uid
+      printf -v tablet_subdir 'vt_%010d' $uid
 
-    echo "Creating pod for tablet $alias..."
+      echo "Creating pod for tablet $alias..."
 
-    # Add xx to beginning or end if there is a dash.  K8s does not allow for
-    # leading or trailing dashes for labels
-    shard_label=`echo $shard | sed s'/[-]$/-xx/' | sed s'/^-/xx-/'`
+      # Add xx to beginning or end if there is a dash.  K8s does not allow for
+      # leading or trailing dashes for labels
+      shard_label=`echo $shard | sed s'/[-]$/-xx/' | sed s'/^-/xx-/'`
 
-    tablet_type=replica
-    if [ $uid_index -gt $(($TABLETS_PER_SHARD-$RDONLY_COUNT-1)) ]; then
-      tablet_type=rdonly
-    fi
+      tablet_type=replica
+      if [ $uid_index -gt $(($TABLETS_PER_SHARD-$RDONLY_COUNT-1)) ]; then
+        tablet_type=rdonly
+      fi
 
-    # Expand template variables
-    sed_script=""
-    for var in alias cell uid keyspace shard shard_label port tablet_subdir vtdataroot_volume tablet_type; do
-      sed_script+="s,{{$var}},${!var},g;"
+      # Expand template variables
+      sed_script=""
+      for var in alias cell uid keyspace shard shard_label port tablet_subdir vtdataroot_volume tablet_type; do
+        sed_script+="s,{{$var}},${!var},g;"
+      done
+
+      # Instantiate template and send to kubectl.
+      cat $VTTABLET_TEMPLATE | sed -e "$sed_script" | $KUBECTL create -f -
     done
-
-    # Instantiate template and send to kubectl.
-    cat $VTTABLET_TEMPLATE | sed -e "$sed_script" | $KUBECTL create -f -
-
-    let index=index+1
+    let cell_index=cell_index+100000000
   done
   let uid_base=uid_base+100
 done