diff --git a/testing/docker/desktop-test/Dockerfile b/testing/docker/desktop-test/Dockerfile
index 849c4e4cf151..50db0ca47e59 100644
--- a/testing/docker/desktop-test/Dockerfile
+++ b/testing/docker/desktop-test/Dockerfile
@@ -60,11 +60,6 @@ RUN npm install -g taskcluster-vcs@2.3.12 \
  && rm -rf ~/.npm
 ENV PATH $PATH:/home/worker/bin
 
-# Remove once running under 'worker' user.  This is necessary for pulseaudio to start
-# XXX: change this back to worker:worker once permissions issues are resolved
-RUN            chown -R root:root /home/worker
-
-
 # TODO Re-enable worker when bug 1093833 lands
 #USER          worker
 
@@ -97,5 +92,7 @@ ADD motd /etc/taskcluster-motd
 ADD taskcluster-interactive-shell /bin/taskcluster-interactive-shell
 RUN chmod +x /bin/taskcluster-interactive-shell
 
+RUN chown -R worker:worker /home/worker
+
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
diff --git a/testing/docker/desktop1604-test/Dockerfile b/testing/docker/desktop1604-test/Dockerfile
index 4c5ceb862064..742e10746f1f 100644
--- a/testing/docker/desktop1604-test/Dockerfile
+++ b/testing/docker/desktop1604-test/Dockerfile
@@ -61,11 +61,6 @@ RUN npm install -g npm@^2.0.0 \
  && rm -rf ~/.npm
 ENV PATH $PATH:/home/worker/bin
 
-# Remove once running under 'worker' user.  This is necessary for pulseaudio to start
-# XXX: change this back to worker:worker once permissions issues are resolved
-RUN            chown -R root:root /home/worker
-
-
 # TODO Re-enable worker when bug 1093833 lands
 #USER          worker
 
@@ -98,5 +93,7 @@ ADD motd /etc/taskcluster-motd
 ADD taskcluster-interactive-shell /bin/taskcluster-interactive-shell
 RUN chmod +x /bin/taskcluster-interactive-shell
 
+RUN chown -R worker:worker /home/worker
+
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]