Merge pull request #187 from mistercrunch/stats_nodelete

HiveStatsCollectionOperator was throwing deadlocks, this should address it
This commit is contained in:
Maxime Beauchemin 2015-04-23 18:04:29 -07:00
Родитель a983fabfaf d1b6e4cae7
Коммит 6f30b531f4
1 изменённых файлов: 12 добавлений и 3 удалений

Просмотреть файл

@ -136,13 +136,22 @@ class HiveStatsCollectionOperator(BaseOperator):
logging.info("Deleting rows from previous runs if they exist")
mysql = MySqlHook(self.mysql_conn_id)
sql = """
DELETE FROM hive_stats
SELECT 1 FROM hive_stats
WHERE
table_name='{self.table}' AND
partition_repr='{part_json}' AND
dttm='{self.dttm}';
dttm='{self.dttm}'
LIMIT 1;
""".format(**locals())
mysql.run(sql)
if mysql.get_records(sql):
sql = """
DELETE FROM hive_stats
WHERE
table_name='{self.table}' AND
partition_repr='{part_json}' AND
dttm='{self.dttm}';
""".format(**locals())
mysql.run(sql)
logging.info("Pivoting and loading cells into the Airflow db")
rows = [