Version 6.2.0.32

Note: This is a pre-release version, future versions of VDO may not support VDO devices created with this version. - Add a library, libuser.a, to provide easy access to VDO user space code for other projects. - Fixed a bug in vdo script when /dev/disk/by-id does not exist. - Fixed an internationalization bug in the vdo script's --indexMemory option. - Changed vdo script to not accept --vdoSlabSize=0 as a way of specifying the default since it was confusing. The default can be obtained by merely omitting the parameter entirely.
dm-vdo · Apr 24, 2018 · 798a827 · 798a827
1 parent 9f5af91
commit 798a827
Show file tree

Hide file tree

Showing 64 changed files with 737 additions and 1,085 deletions.
diff --git a/utils/uds/Makefile b/utils/uds/Makefile
@@ -22,7 +22,7 @@
 # UDS interfaces exported from libuds.so.  DISTRIBUTION_VERSION is the long
 # version name used in distribution builds.  We extract these values from the
 # traditional location.
-UDS_VERSION      = 6.2.0.3
+UDS_VERSION      = 6.2.0.18
 BUILD_VERSION    = $(UDS_VERSION)
 DISTRO_CODENAME := $(shell lsb_release -s -c)
 

diff --git a/utils/uds/cachedChapterIndex.c b/utils/uds/cachedChapterIndex.c
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/cachedChapterIndex.c#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/cachedChapterIndex.c#2 $
  */
 
 #include "cachedChapterIndex.h"
@@ -70,7 +70,6 @@ int cacheChapterIndex(CachedChapterIndex *chapter,
 
   // Mark the entry as valid--it's now in the cache.
   chapter->virtualChapter = virtualChapter;
-  chapter->invalid        = false;
   chapter->skipSearch     = false;
 
   return UDS_SUCCESS;

diff --git a/utils/uds/cachedChapterIndex.h b/utils/uds/cachedChapterIndex.h
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/cachedChapterIndex.h#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/cachedChapterIndex.h#2 $
  */
 
 #ifndef CACHED_CHAPTER_INDEX_H
@@ -64,9 +64,6 @@ struct __attribute__((aligned(CACHE_LINE_BYTES))) cachedChapterIndex {
   // These flags are mutable between cache updates, but they rarely change and
   // are frequently accessed, so they are grouped with the immutable fields.
 
-  /** if set, the virtual chapter is cached, but must not be searched */
-  volatile bool     invalid;
-
   /** if set, skip the chapter when searching the entire cache */
   volatile bool     skipSearch;
 
@@ -127,18 +124,21 @@ static INLINE void setSkipSearch(CachedChapterIndex *chapter, bool skipSearch)
  * for a chunk name. Filters out unused, invalid, disabled, and irrelevant
  * cache entries.
  *
+ * @param zone            the zone doing the check
  * @param chapter         the cache entry search candidate
  * @param virtualChapter  the virtualChapter containing a hook, or UINT64_MAX
  *                        if searching the whole cache for a non-hook
  *
  * @return <code>true</code> if the provided chapter index should be skipped
  **/
-static INLINE bool shouldSkipChapterIndex(CachedChapterIndex *chapter,
-                                          uint64_t            virtualChapter)
+static INLINE bool shouldSkipChapterIndex(const IndexZone *zone,
+                                          const CachedChapterIndex *chapter,
+                                          uint64_t virtualChapter)
 {
   // Don't search unused entries (contents undefined) or invalid entries
-  // (the chapter is no longer in the volume).
-  if ((chapter->virtualChapter == UINT64_MAX) || chapter->invalid) {
+  // (the chapter is no longer the zone's view of the volume).
+  if ((chapter->virtualChapter == UINT64_MAX)
+      || (chapter->virtualChapter < zone->oldestVirtualChapter)) {
     return true;
   }
 

diff --git a/utils/uds/chapterWriter.c b/utils/uds/chapterWriter.c
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/chapterWriter.c#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/chapterWriter.c#2 $
  */
 
 #include "chapterWriter.h"
@@ -97,7 +97,7 @@ static void closeChapters(void *arg)
                                               &OPEN_CHAPTER_INFO);
       int result = discardIndexComponent(oc);
       if (result == UDS_SUCCESS) {
-        logNotice("Discarding saved open chapter");
+        logDebug("Discarding saved open chapter");
       }
     }
 

diff --git a/utils/uds/context.c b/utils/uds/context.c
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/context.c#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/context.c#2 $
  */
 
 #include "context.h"
@@ -312,22 +312,22 @@ int getContextStats(unsigned int contextId, UdsContextStats *stats)
 /**********************************************************************/
 static void collectStats(const UdsContext *context, UdsContextStats *stats)
 {
-  const StatCounters *counters = &context->stats.counters;
+  const SessionStats *sessionStats = &context->indexSession->stats;
 
   stats->currentTime = asTimeT(currentTime(CT_REALTIME));
 
-  stats->postsFound         = counters->postsFound;
-  stats->inMemoryPostsFound = counters->postsFoundOpenChapter;
-  stats->densePostsFound    = counters->postsFoundDense;
-  stats->sparsePostsFound   = counters->postsFoundSparse;
-  stats->postsNotFound      = counters->postsNotFound;
-  stats->updatesFound       = counters->updatesFound;
-  stats->updatesNotFound    = counters->updatesNotFound;
-  stats->deletionsFound     = counters->deletionsFound;
-  stats->deletionsNotFound  = counters->deletionsNotFound;
-  stats->queriesFound       = counters->queriesFound;
-  stats->queriesNotFound    = counters->queriesNotFound;
-  stats->requests           = counters->requests;
+  stats->postsFound         = sessionStats->postsFound;
+  stats->inMemoryPostsFound = sessionStats->postsFoundOpenChapter;
+  stats->densePostsFound    = sessionStats->postsFoundDense;
+  stats->sparsePostsFound   = sessionStats->postsFoundSparse;
+  stats->postsNotFound      = sessionStats->postsNotFound;
+  stats->updatesFound       = sessionStats->updatesFound;
+  stats->updatesNotFound    = sessionStats->updatesNotFound;
+  stats->deletionsFound     = sessionStats->deletionsFound;
+  stats->deletionsNotFound  = sessionStats->deletionsNotFound;
+  stats->queriesFound       = sessionStats->queriesFound;
+  stats->queriesNotFound    = sessionStats->queriesNotFound;
+  stats->requests           = sessionStats->requests;
 }
 
 /**********************************************************************/

diff --git a/utils/uds/context.h b/utils/uds/context.h
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/context.h#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/context.h#2 $
  */
 
 #ifndef CONTEXT_H
@@ -37,38 +37,17 @@ typedef enum udsContextState {
   UDS_CS_DISABLED  = 2
 } UdsContextState;
 
-typedef struct statCounters {
-  uint64_t postsFound;          /* Post calls that found an entry */
-  uint64_t postsFoundOpenChapter; /* Post calls found in the open chapter */
-  uint64_t postsFoundDense;     /* Post calls found in the dense index */
-  uint64_t postsFoundSparse;    /* Post calls found in the sparse index */
-  uint64_t postsNotFound;       /* Post calls that did not find an entry */
-  uint64_t updatesFound;        /* Update calls that found an entry */
-  uint64_t updatesNotFound;     /* Update calls that did not find an entry */
-  uint64_t deletionsFound;      /* Delete calls that found an entry */
-  uint64_t deletionsNotFound;   /* Delete calls that did not find an entry */
-  uint64_t queriesFound;        /* Query calls that found an entry */
-  uint64_t queriesNotFound;     /* Query calls that did not find an entry */
-  uint64_t requests;            /* Total number of requests */
-} StatCounters;
-
-typedef struct __attribute__((aligned(CACHE_LINE_BYTES))) contextStats {
-  StatCounters    counters;
-} ContextStats;
-
 /**
  * Context for uds client index requests.
  **/
-typedef struct __attribute__((aligned(CACHE_LINE_BYTES))) udsContext {
+typedef struct udsContext {
   /* The id of this context */
   unsigned int     id;
   /* The state of this context (whether or not it may be used) */
   UdsContextState  contextState;
   /* The index and session which own this context */
   IndexSession    *indexSession;
   Session          session;
-  /** request statistics for this context, all owned by the callback thread */
-  ContextStats     stats;
 } UdsContext;
 
 /**

diff --git a/utils/uds/deltaIndex.c b/utils/uds/deltaIndex.c
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/deltaIndex.c#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/deltaIndex.c#2 $
  */
 #include "deltaIndex.h"
 
@@ -140,13 +140,13 @@ enum { MAGIC_SIZE = 8 };
 static const char MAGIC_DI_START[] = "DI-00002";
 
 struct di_header {
-  char magic[MAGIC_SIZE];       // MAGIC_DI_START
-  unsigned int zoneNumber;
-  unsigned int numZones;
-  unsigned int firstList;
-  unsigned int numLists;
-  long recordCount;
-  long collisionCount;
+  char     magic[MAGIC_SIZE];   // MAGIC_DI_START
+  uint32_t zoneNumber;
+  uint32_t numZones;
+  uint32_t firstList;
+  uint32_t numLists;
+  uint64_t recordCount;
+  uint64_t collisionCount;
 };
 
 //**********************************************************************
@@ -810,8 +810,8 @@ int startRestoringDeltaIndex(const DeltaIndex *deltaIndex,
                                      "No delta index files");
   }
 
-  long recordCount = 0;
-  long collisionCount = 0;
+  unsigned long recordCount = 0;
+  unsigned long collisionCount = 0;
   unsigned int numZones = numReaders;
   unsigned int firstList[numZones], numLists[numZones];
   BufferedReader *reader[numZones];

diff --git a/utils/uds/index.c b/utils/uds/index.c
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/index.c#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/index.c#3 $
  */
 
 #include "index.h"
@@ -387,8 +387,7 @@ static int searchIndexZone(IndexZone *zone, Request *request)
     if (!isMasterIndexSample(zone->index->masterIndex, &request->hash)
         && isSparse(zone->index->volume->geometry)) {
       // Passing UINT64_MAX triggers a search of the entire sparse cache.
-      result = searchVolumeSparseCache(zone->index->volume, request,
-                                       UINT64_MAX, &found);
+      result = searchSparseCacheInZone(zone, request, UINT64_MAX, &found);
       if (result != UDS_SUCCESS) {
         return result;
       }
@@ -702,12 +701,10 @@ static int replayRecord(Index              *index,
        * already find an entry in the master index that has a different chapter.
        * In this case, we need to search that chapter to determine if the
        * master index entry was for the same record or a different one.
-       *
-       * We should treat all searches as dense since we already know that we've
-       * found the record we are looking for in the master index.
        */
-      result = searchVolume(index->volume, NULL, name, record.virtualChapter,
-                            false, NULL, &updateRecord);
+      result = searchVolumePageCache(index->volume, NULL, name,
+                                     record.virtualChapter, NULL,
+                                     &updateRecord);
       if (result != UDS_SUCCESS) {
         return result;
       }

diff --git a/utils/uds/indexSession.c b/utils/uds/indexSession.c
@@ -16,7 +16,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/indexSession.c#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/indexSession.c#3 $
  */
 
 #include "indexSession.h"
@@ -201,3 +201,50 @@ int udsSetCheckpointFrequency(UdsIndexSession session, unsigned int frequency)
   releaseIndexSession(indexSession);
   return result;
 }
+
+/**********************************************************************/
+int udsGetIndexConfiguration(UdsIndexSession session, UdsConfiguration *conf)
+{
+  if (conf == NULL) {
+    return logErrorWithStringError(UDS_CONF_PTR_REQUIRED,
+                                   "received a NULL config pointer");
+  }
+  IndexSession *indexSession;
+  int result = getIndexSession(session.id, &indexSession);
+  if (result != UDS_SUCCESS) {
+    return result;
+  }
+  result = ALLOCATE(1, struct udsConfiguration, __func__, conf);
+  if (result == UDS_SUCCESS) {
+    **conf = indexSession->grid->userConfig;
+  }
+  releaseIndexSession(indexSession);
+  return result;
+}
+
+/**********************************************************************/
+int udsGetIndexStats(UdsIndexSession session, UdsIndexStats *stats)
+{
+  if (stats == NULL) {
+    return logErrorWithStringError(UDS_INDEX_STATS_PTR_REQUIRED,
+                                   "received a NULL index stats pointer");
+  }
+  IndexSession *indexSession;
+  int result = getIndexSession(session.id, &indexSession);
+  if (result != UDS_SUCCESS) {
+    return result;
+  }
+  IndexRouterStatCounters routerStats;
+  result = getGridStatistics(indexSession->grid, &routerStats);
+  releaseIndexSession(indexSession);
+  if (result != UDS_SUCCESS) {
+    return logErrorWithStringError(result, "%s failed", __func__);
+  }
+  stats->entriesIndexed   = routerStats.entriesIndexed;
+  stats->memoryUsed       = routerStats.memoryUsed;
+  stats->diskUsed         = routerStats.diskUsed;
+  stats->collisions       = routerStats.collisions;
+  stats->entriesDiscarded = routerStats.entriesDiscarded;
+  stats->checkpoints      = routerStats.checkpoints;
+  return UDS_SUCCESS;
+}
diff --git a/utils/uds/indexSession.h b/utils/uds/indexSession.h
@@ -16,13 +16,14 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA. 
  *
- * $Id: //eng/uds-releases/gloria/src/uds/indexSession.h#1 $
+ * $Id: //eng/uds-releases/gloria/src/uds/indexSession.h#2 $
  */
 
 #ifndef INDEX_SESSION_H
 #define INDEX_SESSION_H
 
 #include "atomicDefs.h"
+#include "cpu.h"
 #include "opaqueTypes.h"
 #include "session.h"
 #include "uds.h"
@@ -33,6 +34,21 @@ typedef enum {
   IS_DISABLED = 3
 } IndexSessionState;
 
+typedef struct __attribute__((aligned(CACHE_LINE_BYTES))) sessionStats {
+  uint64_t postsFound;            /* Post calls that found an entry */
+  uint64_t postsFoundOpenChapter; /* Post calls found in the open chapter */
+  uint64_t postsFoundDense;       /* Post calls found in the dense index */
+  uint64_t postsFoundSparse;      /* Post calls found in the sparse index */
+  uint64_t postsNotFound;         /* Post calls that did not find an entry */
+  uint64_t updatesFound;          /* Update calls that found an entry */
+  uint64_t updatesNotFound;       /* Update calls that did not find an entry */
+  uint64_t deletionsFound;        /* Delete calls that found an entry */
+  uint64_t deletionsNotFound;     /* Delete calls that did not find an entry */
+  uint64_t queriesFound;          /* Query calls that found an entry */
+  uint64_t queriesNotFound;       /* Query calls that did not find an entry */
+  uint64_t requests;              /* Total number of requests */
+} SessionStats;
+
 /**
  * Structure corresponding to a UdsIndexSession
  **/
@@ -41,6 +57,8 @@ struct indexSession {
   atomic_t      state;         // atomically updated IndexSessionState
   Grid         *grid;
   RequestQueue *callbackQueue;
+  // Request statistics, all owned by the callback thread
+  SessionStats  stats;
 };
 
 /**