The IAM role is no longer required to run taar_dynamo

The requirement to define an IAM role made it impossible to
run this script against the dev dynamo enviroment.
This commit is contained in:
Victor Ng 2019-01-23 11:46:36 -05:00 коммит произвёл Anthony Miyaguchi
Родитель 5e9fbb76d2
Коммит 6991b5c9be
1 изменённых файлов: 13 добавлений и 2 удалений

Просмотреть файл

@ -199,7 +199,7 @@ class DynamoReducer(object):
client_id = item["client_id"]
item["client_id"] = hash_telemetry_id(client_id)
def push_to_dynamo(self, data_tuple):
def push_to_dynamo(self, data_tuple): # noqa
"""
This connects to DynamoDB and pushes records in `item_list` into
a table.
@ -226,8 +226,15 @@ class DynamoReducer(object):
]
# Obtain credentials from the singleton
cred_args = credentials.getInstance(self._prod_iam_role)
print("Using prod iam role: %s" % self._prod_iam_role)
if self._prod_iam_role is not None:
cred_args = credentials.getInstance(self._prod_iam_role)
else:
cred_args = {}
conn = boto3.resource("dynamodb", region_name=self._region_name, **cred_args)
table = conn.Table(self._table_name)
try:
with table.batch_writer(overwrite_by_pkeys=["client_id"]) as batch:
@ -440,6 +447,10 @@ def main(date, region, table, prod_iam_role, sample_rate):
conf = SparkConf().setAppName(APP_NAME)
spark = SparkSession.builder.config(conf=conf).getOrCreate()
date_obj = datetime.strptime(date, "%Y%m%d")
if prod_iam_role.strip() == "":
prod_iam_role = None
reduction_output = run_etljob(
spark, date_obj, region, table, prod_iam_role, sample_rate
)