From b1d99b08c909d5914cb0ae23c5d1caf673594c76 Mon Sep 17 00:00:00 2001
From: Bruno Moura <brunotm@gmail.com>
Date: Sat, 22 Oct 2016 20:04:48 +0100
Subject: [PATCH] Handle busy error when opening metada in KvESX This is a
 ***workaround*** to the timing/locking with metadata files. issue #626

---
 esx_service/utils/kvESX.py | 48 ++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 12 deletions(-)

diff --git a/esx_service/utils/kvESX.py b/esx_service/utils/kvESX.py
index 521861612..8358233bf 100644
--- a/esx_service/utils/kvESX.py
+++ b/esx_service/utils/kvESX.py
@@ -23,6 +23,8 @@
 import json
 import logging
 import sys
+import errno
+import time
 
 # Python version 3.5.1
 PYTHON64_VERSION = 50659824
@@ -60,6 +62,10 @@
 # Flag to track the version of Python on the platform
 is_64bits = False
 
+# Number of times and sleep time to retry on IOError EBUSY
+EBUSY_RETRY_COUNT = 8
+EBUSY_RETRY_SLEEP = 0.5
+
 class disk_info(Structure):
    _fields_ = [('size', c_uint64),
                ('allocated', c_uint64),
@@ -238,12 +244,21 @@ def load(volpath):
     meta_file = lib.DiskLib_SidecarMakeFileName(volpath.encode(),
                                                 DVOL_KEY.encode())
 
-    try:
-       with open(meta_file, "r") as fh:
-          kv_str = fh.read()
-    except:
-        logging.exception("Failed to access %s", meta_file)
-        return None
+    retry_count = 0
+    while True:
+        try:
+            with open(meta_file, "r") as fh:
+                kv_str = fh.read()
+            break
+        except Exception as open_error:
+            # This is a workaround to the timing/locking with metadata files issue #626
+            if open_error.errno == errno.EBUSY and retry_count <= EBUSY_RETRY_COUNT:
+                logging.warning("Meta file %s busy for load(), retrying...", meta_file)
+                retry_count += 1
+                time.sleep(EBUSY_RETRY_TIME)
+            else:
+                logging.exception("Failed to access %s", meta_file)
+                return None
 
     try:
        return json.loads(kv_str)
@@ -259,12 +274,21 @@ def save(volpath, kv_dict):
 
     kv_str = json.dumps(kv_dict)
 
-    try:
-       with open(meta_file, "w") as fh:
-          fh.write(align_str(kv_str, KV_ALIGN))
-    except:
-        logging.exception("Failed to save meta-data for %s", volpath);
-        return False
+    retry_count = 0
+    while True:
+        try:
+            with open(meta_file, "w") as fh:
+                fh.write(align_str(kv_str, KV_ALIGN))
+            break
+        except Exception as open_error:
+            # This is a workaround to the timing/locking with metadata files issue #626
+            if open_error.errno == errno.EBUSY and retry_count <= EBUSY_RETRY_COUNT:
+                logging.warning("Meta file %s busy for save(), retrying...", meta_file)
+                retry_count += 1
+                time.sleep(EBUSY_RETRY_SLEEP)
+            else:
+                logging.exception("Failed to save meta-data for %s", volpath);
+                return False
 
     return True