diff --git a/pyhive/presto.py b/pyhive/presto.py index 39ca613..9326ec8 100644 --- a/pyhive/presto.py +++ b/pyhive/presto.py @@ -221,7 +221,8 @@ class Cursor(common.DBAPICursor): for i, col in enumerate(self.description): if col[1] == 'varbinary': for row in rows: - row[i] = base64.b64decode(row[i]) + if row[i] is not None: + row[i] = base64.b64decode(row[i]) def _process_response(self, response): """Given the JSON response from Presto's REST API, update the internal state with the next diff --git a/pyhive/tests/sqlalchemy_test_case.py b/pyhive/tests/sqlalchemy_test_case.py index 73ceb47..4f37020 100644 --- a/pyhive/tests/sqlalchemy_test_case.py +++ b/pyhive/tests/sqlalchemy_test_case.py @@ -40,6 +40,13 @@ class SqlAlchemyTestCase(with_metaclass(abc.ABCMeta, object)): self.assertEqual(rows[0].number_of_rows, 1) # number_of_rows is the column name self.assertEqual(len(rows[0]), 1) + @with_engine_connection + def test_one_row_complex_null(self, engine, connection): + one_row_complex_null = Table('one_row_complex_null', MetaData(bind=engine), autoload=True) + rows = one_row_complex_null.select().execute().fetchall() + self.assertEqual(len(rows), 1) + self.assertEqual(list(rows[0]), [None] * len(rows[0])) + @with_engine_connection def test_reflect_no_such_table(self, engine, connection): """reflecttable should throw an exception on an invalid table""" diff --git a/scripts/make_one_row_complex.sh b/scripts/make_one_row_complex.sh index 5b025d3..140e6dc 100755 --- a/scripts/make_one_row_complex.sh +++ b/scripts/make_one_row_complex.sh @@ -1,25 +1,29 @@ #!/bin/bash -eux -hive -e ' +COLUMNS=' +`boolean` BOOLEAN, +`tinyint` TINYINT, +`smallint` SMALLINT, +`int` INT, +`bigint` BIGINT, +`float` FLOAT, +`double` DOUBLE, +`string` STRING, +`timestamp` TIMESTAMP, +`binary` BINARY, +`array` ARRAY, +`map` MAP, +`struct` STRUCT, +`union` UNIONTYPE, +`decimal` DECIMAL(10, 1) +' + +hive -e " set mapred.job.tracker=local; DROP TABLE IF EXISTS one_row_complex; -CREATE TABLE one_row_complex ( - `boolean` BOOLEAN, - `tinyint` TINYINT, - `smallint` SMALLINT, - `int` INT, - `bigint` BIGINT, - `float` FLOAT, - `double` DOUBLE, - `string` STRING, - `timestamp` TIMESTAMP, - `binary` BINARY, - `array` ARRAY, - `map` MAP, - `struct` STRUCT, - `union` UNIONTYPE, - `decimal` DECIMAL(10,1) -); +DROP TABLE IF EXISTS one_row_complex_null; +CREATE TABLE one_row_complex ($COLUMNS); +CREATE TABLE one_row_complex_null ($COLUMNS); INSERT OVERWRITE TABLE one_row_complex SELECT true, 127, @@ -28,13 +32,33 @@ INSERT OVERWRITE TABLE one_row_complex SELECT 9223372036854775807, 0.5, 0.25, - "a string", + 'a string', 0, - "123", + '123', array(1, 2), map(1, 2, 3, 4), - named_struct("a", 1, "b", 2), - create_union(0, 1, "test_string"), + named_struct('a', 1, 'b', 2), + create_union(0, 1, 'test_string'), 0.1 FROM one_row; -' +INSERT OVERWRITE TABLE one_row_complex_null SELECT + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + IF(false, array(1, 2), null), + IF(false, map(1, 2, 3, 4), null), + IF(false, named_struct('a', 1, 'b', 2), null), + IF(false, create_union(0, 1, 'test_string'), null), + null +FROM one_row; +" + +# Note: using IF(false, ...) above to work around https://issues.apache.org/jira/browse/HIVE-4022 +# The problem is that a "void" type cannot be inserted into a complex type field.