From 3ca565624fdd2d71011bfc3670fd663e242cfc92 Mon Sep 17 00:00:00 2001
From: Joe Rowley <jrowley@vertigo.com>
Date: Thu, 8 Oct 2015 08:59:56 -0600
Subject: [PATCH] Check for a shutdown signal in the webjob program before
 initializing data

We have seen intermittent cases where a couple of devices would be
bootstrapped, then about 7 seconds later four more devices. After some
investigation we found that on every fresh cloud deploy the web job would
go through two starts -- we suspect this is as it goes through the
staging, warmup and swap to production.

To avoid letting this happen we will watch the location of the shutdown
file that is created when shutdown is scheduled. If we see that before we
enter the initialization method then we will skip initialization since it
will likely be aborted mid-process.

In Debugging we have sometimes seen cases where initialization is started,
then the shutdown signal is immediatly received. However, in these cases
initialization has been able to complete successfully.

It may not be a perfect solution, but for now I think it's pretty solid.
---
 DeviceAdministration/WebJob/Program.cs | 49 ++++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/DeviceAdministration/WebJob/Program.cs b/DeviceAdministration/WebJob/Program.cs
index 001eb897..e58ccc60 100644
--- a/DeviceAdministration/WebJob/Program.cs
+++ b/DeviceAdministration/WebJob/Program.cs
@@ -21,6 +21,7 @@ using Microsoft.Azure.Devices.Applications.RemoteMonitoring.Simulator.WorkerRole
 using Microsoft.Azure.Devices.Applications.RemoteMonitoring.Common.Configurations;
 using Microsoft.Azure.Devices.Applications.RemoteMonitoring.EventProcessor.WorkerRole.DataInitialization;
 using Microsoft.Azure.IoT.Samples.EventProcessor.WorkerRole.Processors;
+using System.IO;
 
 namespace DeviceAdministration.WebJob
 {
@@ -31,10 +32,40 @@ namespace DeviceAdministration.WebJob
         //static ManualResetEvent runCompleteEvent = new ManualResetEvent(false);
         static IContainer eventProcessorContainer;
 
+        private static readonly string SHUTDOWN_FILE_ENV_VAR = "WEBJOBS_SHUTDOWN_FILE";
+        private static string _shutdownFile;
+        private static bool _shutdownSignalReceived = false;
+
         static void Main(string[] args)
         {
             try
             {
+                //Cloud deploys often get staged and started to warm them up, then get a shutdown
+                //signal from the framework before being moved to the production slot. We don't want 
+                //to start initializing data if we have already gotten the shutdown message, so we'll 
+                //monitor it. This environment variable is reliable
+                //http://blog.amitapple.com/post/2014/05/webjobs-graceful-shutdown/#.VhVYO6L8-B4
+                _shutdownFile = Environment.GetEnvironmentVariable(SHUTDOWN_FILE_ENV_VAR);
+
+                // Setup a file system watcher on that file's directory to know when the file is created
+                //First check for null, though. This does not exist on a localhost deploy, only cloud
+                if (!string.IsNullOrWhiteSpace(_shutdownFile))
+                {
+                    var fileSystemWatcher = new FileSystemWatcher(Path.GetDirectoryName(_shutdownFile));
+                    fileSystemWatcher.Created += OnShutdownFileChanged;
+                    fileSystemWatcher.Changed += OnShutdownFileChanged;
+                    fileSystemWatcher.NotifyFilter = NotifyFilters.CreationTime | NotifyFilters.FileName | NotifyFilters.LastWrite;
+                    fileSystemWatcher.IncludeSubdirectories = false;
+                    fileSystemWatcher.EnableRaisingEvents = true;
+
+                    //In case the file had already been created before we started watching it.
+                    if (System.IO.File.Exists(_shutdownFile))
+                    {
+                        _shutdownSignalReceived = true;
+
+                    }
+                }
+
                 BuildContainer();
 
                 CreateInitialDataAsNeeded();
@@ -53,6 +84,14 @@ namespace DeviceAdministration.WebJob
             }
         }
 
+        private static void OnShutdownFileChanged(object sender, FileSystemEventArgs e)
+        {
+            if (e.FullPath.IndexOf(Path.GetFileName(_shutdownFile), StringComparison.OrdinalIgnoreCase) >= 0)
+            {
+                _shutdownSignalReceived = true;
+            }
+        }
+
         static void BuildContainer()
         {
             var builder = new ContainerBuilder();
@@ -62,9 +101,13 @@ namespace DeviceAdministration.WebJob
 
         static void CreateInitialDataAsNeeded()
         {
-            Trace.TraceInformation("Preparing to add initial data");
-            var creator = eventProcessorContainer.Resolve<IDataInitializer>();
-            creator.CreateInitialDataIfNeeded();
+            if (!_shutdownSignalReceived && !cancellationTokenSource.Token.IsCancellationRequested)
+            {
+
+                Trace.TraceInformation("Preparing to add initial data");
+                var creator = eventProcessorContainer.Resolve<IDataInitializer>();
+                creator.CreateInitialDataIfNeeded();
+            }
         }
 
         static void StartEventProcessorHost()