Add regression test for nulls in one_row_complex and fix Presto null binary

pyhive/presto.py

@@ -221,7 +221,8 @@ class Cursor(common.DBAPICursor):
         for i, col in enumerate(self.description):
             if col[1] == 'varbinary':
                 for row in rows:
-                    row[i] = base64.b64decode(row[i])
+                    if row[i] is not None:
+                        row[i] = base64.b64decode(row[i])
 
     def _process_response(self, response):
         """Given the JSON response from Presto's REST API, update the internal state with the next

pyhive/tests/sqlalchemy_test_case.py

@@ -40,6 +40,13 @@ class SqlAlchemyTestCase(with_metaclass(abc.ABCMeta, object)):
         self.assertEqual(rows[0].number_of_rows, 1)  # number_of_rows is the column name
         self.assertEqual(len(rows[0]), 1)
 
+    @with_engine_connection
+    def test_one_row_complex_null(self, engine, connection):
+        one_row_complex_null = Table('one_row_complex_null', MetaData(bind=engine), autoload=True)
+        rows = one_row_complex_null.select().execute().fetchall()
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(list(rows[0]), [None] * len(rows[0]))
+
     @with_engine_connection
     def test_reflect_no_such_table(self, engine, connection):
         """reflecttable should throw an exception on an invalid table"""

scripts/make_one_row_complex.sh

@@ -1,25 +1,29 @@
 #!/bin/bash -eux
 
-hive -e '
+COLUMNS='
+`boolean` BOOLEAN,
+`tinyint` TINYINT,
+`smallint` SMALLINT,
+`int` INT,
+`bigint` BIGINT,
+`float` FLOAT,
+`double` DOUBLE,
+`string` STRING,
+`timestamp` TIMESTAMP,
+`binary` BINARY,
+`array` ARRAY<int>,
+`map` MAP<int, int>,
+`struct` STRUCT<a: int, b: int>,
+`union` UNIONTYPE<int, string>,
+`decimal` DECIMAL(10, 1)
+'
+
+hive -e "
 set mapred.job.tracker=local;
 DROP TABLE IF EXISTS one_row_complex;
-CREATE TABLE one_row_complex (
-    `boolean` BOOLEAN,
-    `tinyint` TINYINT,
-    `smallint` SMALLINT,
-    `int` INT,
-    `bigint` BIGINT,
-    `float` FLOAT,
-    `double` DOUBLE,
-    `string` STRING,
-    `timestamp` TIMESTAMP,
-    `binary` BINARY,
-    `array` ARRAY<int>,
-    `map` MAP<int, int>,
-    `struct` STRUCT<a: int, b: int>,
-    `union` UNIONTYPE<int, string>,
-    `decimal` DECIMAL(10,1)
-);
+DROP TABLE IF EXISTS one_row_complex_null;
+CREATE TABLE one_row_complex ($COLUMNS);
+CREATE TABLE one_row_complex_null ($COLUMNS);
 INSERT OVERWRITE TABLE one_row_complex SELECT
     true,
     127,
@@ -28,13 +32,33 @@ INSERT OVERWRITE TABLE one_row_complex SELECT
     9223372036854775807,
     0.5,
     0.25,
-    "a string",
+    'a string',
     0,
-    "123",
+    '123',
     array(1, 2),
     map(1, 2, 3, 4),
-    named_struct("a", 1, "b", 2),
-    create_union(0, 1, "test_string"),
+    named_struct('a', 1, 'b', 2),
+    create_union(0, 1, 'test_string'),
     0.1
 FROM one_row;
-'
+INSERT OVERWRITE TABLE one_row_complex_null SELECT
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    IF(false, array(1, 2), null),
+    IF(false, map(1, 2, 3, 4), null),
+    IF(false, named_struct('a', 1, 'b', 2), null),
+    IF(false, create_union(0, 1, 'test_string'), null),
+    null
+FROM one_row;
+"
+
+# Note: using IF(false, ...) above to work around https://issues.apache.org/jira/browse/HIVE-4022
+# The problem is that a "void" type cannot be inserted into a complex type field.