Compress push_data.json files right after generating them

This commit is contained in:
Marco Castelluccio 2020-02-26 10:45:45 +01:00
Родитель b454a39d71
Коммит 282b25871f
1 изменённых файлов: 2 добавлений и 3 удалений

Просмотреть файл

@ -128,6 +128,8 @@ class Retriever(object):
with open(f"push_data_{runnable}.json", "w") as f:
json.dump(push_data, f)
zstd_compress(f"push_data_{runnable}.json")
def retrieve_push_data(self):
# Download previous cache.
db.download(ADR_CACHE_DB)
@ -141,9 +143,6 @@ class Retriever(object):
# ActiveData and we'll see if it's enough to train a satisfying model.
self.generate_push_data("group", 3)
zstd_compress("push_data_label.json")
zstd_compress("push_data_group.json")
def generate_test_scheduling_history(self, granularity):
push_data_path = f"push_data_{granularity}.json"
updated = download_check_etag(PUSH_DATA_URL.format(granularity=granularity))