Added wait and retries before attempting to kill process

misc/custom-script-shim

@@ -52,12 +52,27 @@ write_status() {
 
 check_binary_write_lock() {
     set +e # disable exit on non-zero return code
+    local retry_attempts=0
+    while (( retry_attempts < 3 )); do
+        lsof_output="$(lsof ${bin})"
+        if [ "$?" -eq 0 ]; then
+            echo "${HANDLER_BIN} is open by the following processes: "
+            echo "${lsof_output}"
+            ((++retry_attempts))
+            echo "sleeping for 30 seconds before retry, attempt ${retry_attempts} of 3"
+            sleep 30
+        else
+            set -e
+            return 0
+        fi
+    done
+
+    # retries over, kill process(es) if needed
     lsof_output="$(lsof ${bin})"
     if [ "$?" -eq 0 ]; then
-        echo "${HANDLER_BIN} is open by the following processes: "
-        echo "${lsof_output}"
+    # If the wait timed out we need to write status file
+        echo "Timed out waiting for lock on ${HANDLER_BIN}"
         echo "attempting to kill processes with open file handles to ${HANDLER_BIN}"
-        # suppress output and errors in case process with file-handle is already dead and kill gets called without a process id
         lsof -t ${bin} | xargs kill -9 > /dev/null 2>&1
     fi
     set -e # re-enable exit on non-zero return code