[baseimage]: Build and install redis-dump-load Python 3 package in host image (#4661)

Fix #4656

files/build_templates/sonic_debian_extension.j2

@@ -101,6 +101,13 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
 # Install Python client for Redis
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6"
 
+# Install redis-dump-load Python 3 package
+# Note: the scripts will be overwritten by corresponding Python 2 package
+REDIS_DUMP_LOAD_PY3_WHEEL_NAME=$(basename {{redis_dump_load_py3_wheel_path}})
+sudo cp {{redis_dump_load_py3_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $REDIS_DUMP_LOAD_PY3_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY3_WHEEL_NAME
+
 # Install redis-dump-load Python 2 package
 REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}})
 sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
@@ -110,13 +117,8 @@ sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
 # Install Python module for ipaddress
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install ipaddress
 
-# Install SwSS SDK Python 2 package
-SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
-sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
-sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
-
 # Install SwSS SDK Python 3 package
+# Note: the scripts will be overwritten by corresponding Python 2 package
 if [ -e {{swsssdk_py3_wheel_path}} ]; then
     SWSSSDK_PY3_WHEEL_NAME=$(basename {{swsssdk_py3_wheel_path}})
     sudo cp {{swsssdk_py3_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME
@@ -124,6 +126,12 @@ if [ -e {{swsssdk_py3_wheel_path}} ]; then
     sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY3_WHEEL_NAME
 fi
 
+# Install SwSS SDK Python 2 package
+SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
+sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
+
 # Install sonic-yang-models py3 package, install dependencies
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb

rules/redis-dump-load-py3.mk

@@ -0,0 +1,9 @@
+# redis_dump_load python3 wheel
+
+REDIS_DUMP_LOAD_PY3 = redis_dump_load-1.1-py3-none-any.whl
+$(REDIS_DUMP_LOAD_PY3)_SRC_PATH = $(SRC_PATH)/redis-dump-load
+$(REDIS_DUMP_LOAD_PY3)_PYTHON_VERSION = 3
+# Synthetic dependency just to avoid race condition
+$(REDIS_DUMP_LOAD_PY3)_DEPENDS += $(REDIS_DUMP_LOAD_PY2)
+$(REDIS_DUMP_LOAD_PY3)_TEST = n
+SONIC_PYTHON_WHEELS += $(REDIS_DUMP_LOAD_PY3)

rules/swss-common.mk

@@ -4,7 +4,7 @@ LIBSWSSCOMMON = libswsscommon_1.0.0_$(CONFIGURED_ARCH).deb
 $(LIBSWSSCOMMON)_SRC_PATH = $(SRC_PATH)/sonic-swss-common
 $(LIBSWSSCOMMON)_DEPENDS += $(LIBHIREDIS_DEV) $(LIBNL3_DEV) $(LIBNL_GENL3_DEV) \
                             $(LIBNL_ROUTE3_DEV) $(LIBNL_NF3_DEV) \
-			    $(LIBNL_CLI_DEV) $(SWIG)
+                            $(LIBNL_CLI_DEV) $(SWIG)
 $(LIBSWSSCOMMON)_RDEPENDS += $(LIBHIREDIS) $(LIBNL3) $(LIBNL_GENL3) \
                              $(LIBNL_ROUTE3) $(LIBNL_NF3) $(LIBNL_CLI)
 SONIC_DPKG_DEBS += $(LIBSWSSCOMMON)

rules/swsssdk-py2.mk

@@ -3,4 +3,5 @@
 SWSSSDK_PY2 = swsssdk-2.0.1-py2-none-any.whl
 $(SWSSSDK_PY2)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk
 $(SWSSSDK_PY2)_PYTHON_VERSION = 2
+$(SWSSSDK_PY2)_DEPENDS += $(REDIS_DUMP_LOAD_PY2)
 SONIC_PYTHON_WHEELS += $(SWSSSDK_PY2)

rules/swsssdk-py3.mk

@@ -4,5 +4,5 @@ SWSSSDK_PY3 = swsssdk-2.0.1-py3-none-any.whl
 $(SWSSSDK_PY3)_SRC_PATH = $(SRC_PATH)/sonic-py-swsssdk
 $(SWSSSDK_PY3)_PYTHON_VERSION = 3
 # Synthetic dependency just to avoid race condition
-$(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2)
+$(SWSSSDK_PY3)_DEPENDS += $(SWSSSDK_PY2) $(REDIS_DUMP_LOAD_PY3)
 SONIC_PYTHON_WHEELS += $(SWSSSDK_PY3)

slave.mk

@@ -816,6 +816,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export swsssdk_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SWSSSDK_PY3))"
 	export platform_common_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_COMMON_PY2))"
 	export redis_dump_load_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2))"
+	export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))"
 	export install_debug_image="$(INSTALL_DEBUG_TOOLS)"
 	export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))"
 	export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))"

src/redis-dump-load.patch/0001-Use-pipelines-when-dumping-52.patch

@@ -33,7 +33,7 @@ index df2870b..1b6063b 100755
      if pretty:
          # hack to avoid implementing pretty printing
          fp.write(dumps(host=host, port=port, password=password, db=db,
-@@ -276,28 +276,76 @@ def _read_key(key, r, pretty, encoding):
+@@ -276,28 +276,77 @@ def _read_key(key, r, pretty, encoding):
      return (type, ttl, value)
  
  def _reader(r, pretty, encoding, keys='*'):
@@ -68,7 +68,7 @@ index df2870b..1b6063b 100755
 +        i += 10000
 +
 +def _read_keys(r, encoded_keys, pretty, encoding):
-+    decoded_keys = [encoded_key.decode() for encoded_key in encoded_keys]
++    decoded_keys = [encoded_key.decode(encoding) for encoded_key in encoded_keys]
 +    do_keys = decoded_keys
 +    retries = 5
 +    type_results = None
@@ -84,12 +84,13 @@ index df2870b..1b6063b 100755
 +            p = r.pipeline()
 +            for key in do_keys:
 +                p.type(key)
-+            type_results = p.execute()
++            encoded_type_results = p.execute()
++            type_results = [encoded_type_result.decode('ascii') for encoded_type_result in encoded_type_results]
 +
 +        p = r.pipeline()
 +        for i in range(len(do_keys)):
 +            key = decoded_keys[i]
-+            type = type_results[i].decode('ascii')
++            type = type_results[i]
 +            if type == 'none':
 +                # key was deleted by a concurrent operation on the data store.
 +                # issue noops so that the number of results does not change