diff --git a/vp9/encoder/vp9_denoiser.c b/vp9/encoder/vp9_denoiser.c
index 7c21f4708..8fb34f789 100644
--- a/vp9/encoder/vp9_denoiser.c
+++ b/vp9/encoder/vp9_denoiser.c
@@ -55,19 +55,46 @@ int update_running_avg(uint8_t *mc_avg, int mc_avg_stride, uint8_t *avg,
 void vp9_denoiser_denoise(VP9_DENOISER *denoiser,
                           MACROBLOCK *mb, MODE_INFO **grid,
                           int mi_row, int mi_col, BLOCK_SIZE bs) {
-  update_running_avg(denoiser->mc_running_avg_y.buf,
-                     denoiser->mc_running_avg_y.stride,
-                     denoiser->running_avg_y.buf,
-                     denoiser->running_avg_y.stride,
+  update_running_avg(denoiser->mc_running_avg_y.y_buffer,
+                     denoiser->mc_running_avg_y.y_stride,
+                     denoiser->running_avg_y[INTRA_FRAME].y_buffer,
+                     denoiser->running_avg_y[INTRA_FRAME].y_stride,
                      mb->plane[0].src.buf, mb->plane[0].src.stride, bs);
   return;
 }
 
+void copy_frame(YV12_BUFFER_CONFIG dest, YV12_BUFFER_CONFIG src) {
+  memcpy(dest.buffer_alloc, src.buffer_alloc, src.buffer_alloc_sz);
+}
+
 void vp9_denoiser_update_frame_info(VP9_DENOISER *denoiser,
+                                    YV12_BUFFER_CONFIG src,
                                     FRAME_TYPE frame_type,
                                     int refresh_alt_ref_frame,
                                     int refresh_golden_frame,
                                     int refresh_last_frame) {
+  int i;
+  if (frame_type == KEY_FRAME) {
+    copy_frame(denoiser->running_avg_y[LAST_FRAME], src);
+    for (i = 2; i < MAX_REF_FRAMES - 1; i++) {
+      copy_frame(denoiser->running_avg_y[i],
+                 denoiser->running_avg_y[LAST_FRAME]);
+    }
+  } else { /* For non key frames */
+    if (refresh_alt_ref_frame) {
+      copy_frame(denoiser->running_avg_y[ALTREF_FRAME],
+                 denoiser->running_avg_y[INTRA_FRAME]);
+    }
+    if (refresh_golden_frame) {
+      copy_frame(denoiser->running_avg_y[GOLDEN_FRAME],
+                 denoiser->running_avg_y[INTRA_FRAME]);
+    }
+    if (refresh_last_frame) {
+      copy_frame(denoiser->running_avg_y[LAST_FRAME],
+                 denoiser->running_avg_y[INTRA_FRAME]);
+    }
+  }
+
   return;
 }
 
@@ -76,23 +103,22 @@ void vp9_denoiser_update_frame_stats() {
 }
 
 int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
-                       int border) {
+                       int ssx, int ssy, int border) {
+  int i, fail;
   assert(denoiser);
 
-  denoiser->running_avg_y.stride = width + 2 * border;
-
-  denoiser->running_avg_y.buf = calloc(
-      ((2 * border) + width) * ((2 * border) + height), sizeof(uint8_t));
-  if (denoiser->running_avg_y.buf == NULL) {
-    vp9_denoiser_free(denoiser);
-    return 1;
+  for (i = 0; i < MAX_REF_FRAMES; ++i) {
+    fail = vp9_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
+                                ssx, ssy, border);
+    if (fail) {
+      vp9_denoiser_free(denoiser);
+      return 1;
+    }
   }
 
-  denoiser->mc_running_avg_y.stride = width + 2 * border;
-
-  denoiser->mc_running_avg_y.buf = calloc(
-      ((2 * border) + width) * ((2 * border) + height), sizeof(uint8_t));
-  if (denoiser->mc_running_avg_y.buf == NULL) {
+  fail = vp9_alloc_frame_buffer(&denoiser->mc_running_avg_y, width, height,
+                              ssx, ssy, border);
+  if (fail) {
     vp9_denoiser_free(denoiser);
     return 1;
   }
@@ -101,11 +127,14 @@ int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
 }
 
 void vp9_denoiser_free(VP9_DENOISER *denoiser) {
-  if (denoiser->running_avg_y.buf != NULL) {
-    free(denoiser->running_avg_y.buf);
+  int i;
+  for (i = 0; i < MAX_REF_FRAMES; ++i) {
+    if (&denoiser->running_avg_y[i] != NULL) {
+      vp9_free_frame_buffer(&denoiser->running_avg_y[i]);
+    }
   }
-  if (denoiser->mc_running_avg_y.buf != NULL) {
-    free(denoiser->mc_running_avg_y.buf);
+  if (&denoiser->mc_running_avg_y != NULL) {
+    vp9_free_frame_buffer(&denoiser->mc_running_avg_y);
   }
   return;
 }
diff --git a/vp9/encoder/vp9_denoiser.h b/vp9/encoder/vp9_denoiser.h
index a7a8d9329..81fc7ebe5 100644
--- a/vp9/encoder/vp9_denoiser.h
+++ b/vp9/encoder/vp9_denoiser.h
@@ -12,6 +12,7 @@
 #define VP9_ENCODER_DENOISER_H_
 
 #include "vp9/encoder/vp9_block.h"
+#include "vpx_scale/yv12config.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -23,11 +24,12 @@ enum vp9_denoiser_decision {
 };
 
 typedef struct vp9_denoiser {
-  struct buf_2d running_avg_y;
-  struct buf_2d mc_running_avg_y;
+  YV12_BUFFER_CONFIG running_avg_y[MAX_REF_FRAMES];
+  YV12_BUFFER_CONFIG mc_running_avg_y;
 } VP9_DENOISER;
 
 void vp9_denoiser_update_frame_info(VP9_DENOISER *denoiser,
+                                    YV12_BUFFER_CONFIG src,
                                     FRAME_TYPE frame_type,
                                     int refresh_alt_ref_frame,
                                     int refresh_golden_frame,
@@ -40,7 +42,7 @@ void vp9_denoiser_denoise(VP9_DENOISER *denoiser,
 void vp9_denoiser_update_frame_stats();
 
 int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
-                       int border);
+                       int ssx, int ssy, int border);
 
 void vp9_denoiser_free(VP9_DENOISER *denoiser);
 
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index 525eccd56..b464361fd 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -683,6 +683,12 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
 
 #if CONFIG_DENOISING
   vp9_denoiser_alloc(&(cpi->denoiser), cm->width, cm->height,
+                     // TODO(tkopp) An unrelated bug causes
+                     // cm->subsampling_{x,y} to be uninitialized at this point
+                     // in execution. For now we assume YUV-420, which is x/y
+                     // subsampling of 1.
+                     1, 1,
+                     // cm->subsampling_x, cm->subsampling_y,
                      VP9_ENC_BORDER_IN_PIXELS);
 #endif
 }
@@ -1558,6 +1564,7 @@ void vp9_update_reference_frames(VP9_COMP *cpi) {
   }
 #if CONFIG_DENOISING
   vp9_denoiser_update_frame_info(&cpi->denoiser,
+                                *cpi->Source,
                                 cpi->common.frame_type,
                                 cpi->refresh_alt_ref_frame,
                                 cpi->refresh_golden_frame,