diff --git a/taskcluster/taskgraph/transforms/job/__init__.py b/taskcluster/taskgraph/transforms/job/__init__.py
index 36d3d5b4cbad..ba5dcf1626b7 100644
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -115,11 +115,8 @@ def rewrite_when_to_optimization(config, jobs):
 
         files_changed = when.get('files-changed')
 
-        # add some common files
-        files_changed.extend([
-            '{}/**'.format(config.path),
-            'taskcluster/taskgraph/**',
-        ])
+        # implicitly add config directories affecting task
+        files_changed.append('{}/**'.format(config.path))
         if 'in-tree' in job.get('worker', {}).get('docker-image', {}):
             files_changed.append('taskcluster/docker/{}/**'.format(
                 job['worker']['docker-image']['in-tree']))