diff --git a/vp9/encoder/vp9_denoiser.c b/vp9/encoder/vp9_denoiser.c
index f40d8ab8f..dbca633c7 100644
--- a/vp9/encoder/vp9_denoiser.c
+++ b/vp9/encoder/vp9_denoiser.c
@@ -450,11 +450,12 @@ void vp9_denoiser_update_frame_info(VP9_DENOISER *denoiser,
                                     int resized) {
   // Copy source into denoised reference buffers on KEY_FRAME or
   // if the just encoded frame was resized.
-  if (frame_type == KEY_FRAME || resized != 0) {
+  if (frame_type == KEY_FRAME || resized != 0 || denoiser->reset) {
     int i;
     // Start at 1 so as not to overwrite the INTRA_FRAME
     for (i = 1; i < MAX_REF_FRAMES; ++i)
       copy_frame(&denoiser->running_avg_y[i], &src);
+    denoiser->reset = 0;
     return;
   }
 
@@ -567,6 +568,8 @@ int vp9_denoiser_alloc(VP9_DENOISER *denoiser, int width, int height,
   denoiser->increase_denoising = 0;
   denoiser->frame_buffer_initialized = 1;
   denoiser->denoising_level = kDenLow;
+  denoiser->prev_denoising_level = kDenLow;
+  denoiser->reset = 0;
   return 0;
 }
 
@@ -586,6 +589,12 @@ void vp9_denoiser_free(VP9_DENOISER *denoiser) {
 void vp9_denoiser_set_noise_level(VP9_DENOISER *denoiser,
                                   int noise_level) {
   denoiser->denoising_level = noise_level;
+  if (denoiser->denoising_level > kDenLowLow &&
+      denoiser->prev_denoising_level == kDenLowLow)
+    denoiser->reset = 1;
+  else
+    denoiser->reset = 0;
+  denoiser->prev_denoising_level = denoiser->denoising_level;
 }
 
 #ifdef OUTPUT_YUV_DENOISED
diff --git a/vp9/encoder/vp9_denoiser.h b/vp9/encoder/vp9_denoiser.h
index a0e201781..84189b4fd 100644
--- a/vp9/encoder/vp9_denoiser.h
+++ b/vp9/encoder/vp9_denoiser.h
@@ -40,7 +40,9 @@ typedef struct vp9_denoiser {
   YV12_BUFFER_CONFIG last_source;
   int increase_denoising;
   int frame_buffer_initialized;
+  int reset;
   VP9_DENOISER_LEVEL denoising_level;
+  VP9_DENOISER_LEVEL prev_denoising_level;
 } VP9_DENOISER;
 
 struct VP9_COMP;
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index 547c1628d..69445af64 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -2839,7 +2839,8 @@ void vp9_update_reference_frames(VP9_COMP *cpi) {
              sizeof(cpi->interp_filter_selected[0]));
   }
 #if CONFIG_VP9_TEMPORAL_DENOISING
-  if (cpi->oxcf.noise_sensitivity > 0) {
+  if (cpi->oxcf.noise_sensitivity > 0 &&
+      cpi->denoiser.denoising_level > kDenLowLow) {
     vp9_denoiser_update_frame_info(&cpi->denoiser,
                                    *cpi->Source,
                                    cpi->common.frame_type,
diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 2057dca3a..b0aaa2762 100644
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -1270,7 +1270,10 @@ void vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   }
 
 #if CONFIG_VP9_TEMPORAL_DENOISING
-  vp9_denoiser_reset_frame_stats(ctx);
+  if (cpi->oxcf.noise_sensitivity > 0 &&
+      cpi->denoiser.denoising_level > kDenLowLow) {
+    vp9_denoiser_reset_frame_stats(ctx);
+  }
 #endif
 
   if (cpi->rc.frames_since_golden == 0 && !cpi->use_svc) {
@@ -1642,7 +1645,8 @@ void vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
     }
 
 #if CONFIG_VP9_TEMPORAL_DENOISING
-    if (cpi->oxcf.noise_sensitivity > 0) {
+    if (cpi->oxcf.noise_sensitivity > 0 &&
+        cpi->denoiser.denoising_level > kDenLowLow) {
       vp9_denoiser_update_frame_stats(mi, sse_y, this_mode, ctx);
       // Keep track of zero_last cost.
       if (ref_frame == LAST_FRAME && frame_mv[this_mode][ref_frame].as_int == 0)
@@ -1823,7 +1827,9 @@ void vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
 
 #if CONFIG_VP9_TEMPORAL_DENOISING
   if (cpi->oxcf.noise_sensitivity > 0 &&
-      cpi->resize_pending == 0) {
+      cpi->resize_pending == 0 &&
+      cpi->denoiser.denoising_level > kDenLowLow &&
+      cpi->denoiser.reset == 0) {
     VP9_DENOISER_DECISION decision = COPY_BLOCK;
     vp9_denoiser_denoise(cpi, x, mi_row, mi_col, VPXMAX(BLOCK_8X8, bsize),
                          ctx, &decision);