From 9c503883e312d1133b82bde5952e3efc4f3eb454 Mon Sep 17 00:00:00 2001 From: Mat Date: Wed, 22 May 2019 11:23:46 +0100 Subject: [PATCH] Fixes error in sharding (#37) * Fixes sharding * Fixes typo with DATA env value --- .gitignore | 2 ++ {{cookiecutter.project_name}}/.azureml/azml_config.json | 1 - .../TensorFlow_imagenet/src/data/images.py | 2 +- .../TensorFlow_imagenet/src/data/tfrecords.py | 2 +- .../TensorFlow_imagenet/tensorflow_imagenet.py | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 {{cookiecutter.project_name}}/.azureml/azml_config.json diff --git a/.gitignore b/.gitignore index 4536aca..e649f55 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,5 @@ venv.bak/ .vscode/settings.json .dev_env + +.azureml diff --git a/{{cookiecutter.project_name}}/.azureml/azml_config.json b/{{cookiecutter.project_name}}/.azureml/azml_config.json deleted file mode 100644 index 12860f1..0000000 --- a/{{cookiecutter.project_name}}/.azureml/azml_config.json +++ /dev/null @@ -1 +0,0 @@ -{"Id": null, "Scope": "/subscriptions/edf507a2-6235-46c5-b560-fd463ba2e771/resourceGroups/amldistrg/providers/Microsoft.MachineLearningServices/workspaces/workspace"} \ No newline at end of file diff --git a/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/images.py b/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/images.py index 0449eac..dca0d4a 100644 --- a/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/images.py +++ b/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/images.py @@ -178,7 +178,7 @@ def input_fn( if is_training: # Shuffle the input files if distributed: - dataset = dataset.shard(hvd.size(), hvd.local_rank()) + dataset = dataset.shard(hvd.size(), hvd.rank()) dataset = dataset.shuffle(buffer_size=file_shuffle_buffer) # _NUM_TRAIN_FILES diff --git a/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/tfrecords.py b/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/tfrecords.py index 13971c4..469d7d7 100644 --- a/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/tfrecords.py +++ b/{{cookiecutter.project_name}}/TensorFlow_imagenet/src/data/tfrecords.py @@ -136,7 +136,7 @@ def input_fn( if is_training: # Shuffle the input files if distributed: - dataset = dataset.shard(hvd.size(), hvd.local_rank()) + dataset = dataset.shard(hvd.size(), hvd.rank()) dataset = dataset.shuffle(buffer_size=file_shuffle_buffer) # _NUM_TRAIN_FILES diff --git a/{{cookiecutter.project_name}}/TensorFlow_imagenet/tensorflow_imagenet.py b/{{cookiecutter.project_name}}/TensorFlow_imagenet/tensorflow_imagenet.py index 5f8395e..8272271 100644 --- a/{{cookiecutter.project_name}}/TensorFlow_imagenet/tensorflow_imagenet.py +++ b/{{cookiecutter.project_name}}/TensorFlow_imagenet/tensorflow_imagenet.py @@ -100,7 +100,7 @@ def submit_images_local(c, epochs=1): "--data-format": "channels_first", }, dependencies_file="TensorFlow_imagenet/environment_gpu.yml", - docker_args=["-v", f"{env_values['data']}:/data"], + docker_args=["-v", f"{env_values['DATA']}:/data"], wait_for_completion=True, ) print(run) @@ -155,7 +155,7 @@ def submit_tfrecords_local(c, epochs=1): "--data-format": "channels_first", }, dependencies_file="TensorFlow_imagenet/environment_gpu.yml", - docker_args=["-v", f"{env_values['data']}:/data"], + docker_args=["-v", f"{env_values['DATA']}:/data"], wait_for_completion=True, ) print(run)