Unidata · edhartnett · Aug 20, 2017 · Oct 30, 2017 · Oct 30, 2017 · Oct 30, 2017
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -267,8 +267,6 @@ SET(CMAKE_REQUIRED_INCLUDES ${CMAKE_SOURCE_DIR}/libsrc)
 # End Compiler Configuration
 ################################
 
-
-
 ##
 # Configuration for post-install RPath
 # Adapted from http://www.cmake.org/Wiki/CMake_RPATH_handling

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -12,6 +12,7 @@ reading and writing. See the file docs/filters.md.
 ## 4.5.1 - TBD
 
 * [Enhancement] Added an option to enable strict null-byte padding for headers; this padding was specified in the spec but was not enforced.  Enabling this option will allow you to check your files, as it will return an E_NULLPAD error.  It is possible for these files to have been written by older versions of libnetcdf.  There is no effective problem caused by this lack of null padding, so enabling these options is informational only.  The options for `configure` and `cmake` are `--enable-strict-null-byte-header-padding` and `-DENABLE_STRICT_NULL_BYTE_HEADER_PADDING`, respectively.  See [Github #657](https://github.com/Unidata/netcdf-c/issues/657) for more information.
+* Corrected a number of memory issues identified in `ncgen`.  See [GitHub #558 for more information](https://github.com/Unidata/netcdf-c/pull/558).
 * [Enhancement] Reverted behavior/handling of out-of-range attribute values to pre-4.5.0 default. See [Github #512](https://github.com/Unidata/netcdf-c/issues/512) for more information.
 * [Bug] Fixed error in tst_parallel2.c. See [Github #545](https://github.com/Unidata/netcdf-c/issues/545) for more information.
 * [Bug] Fixed handling of corrupt files + proper offset handling for hdf5 files. See [Github #552](https://github.com/Unidata/netcdf-c/issues/552) for more information.

diff --git a/cf b/cf
@@ -3,6 +3,7 @@
 DB=1
 #X=-x
 FAST=1
+#MEM=1
 
 HDF5=1
 DAP=1
@@ -22,8 +23,9 @@ fi
 CFLAGS=""
 #CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wconversion ${CFLAGS}"
 CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign -Wno-format ${CFLAGS}"
-#CFLAGS="-Wall ${CFLAGS}"
-#CFLAGS="-Wconversion"
+if test "x$MEM" = x1 ; then
+CFLAGS="-fsanitize=address -fno-omit-frame-pointer ${CFLAGS}"
+fi
 
 stddir="/usr/local"
 PREFIX=/usr/local

diff --git a/docs/OPeNDAP.dox b/docs/OPeNDAP.dox
@@ -1,4 +1,5 @@
-/*! \page dap2 DAP2 Support
+/*!
+\page dap2 DAP2 Support
 
 \tableofcontents
 
@@ -35,7 +36,7 @@ See the following pages for more information.
 - \subpage var_dim_trans
 - \subpage var_name_trans
 
-\page dap_accessing_data Accessing OPeNDAP Data
+\section dap_accessing_data Accessing OPeNDAP Data
 
 In order to access an OPeNDAP data source through the netCDF API, the
 file name normally used is replaced with a URL with a specific
@@ -177,13 +178,13 @@ variables:
 }
 \endcode
 
-\page dap_to_netcdf DAP to NetCDF Translation Rules
+\section dap_to_netcdf DAP to NetCDF Translation Rules
 
 Currently only one translation available: DAP 2 Protocol to netCDF-3.
 There used to be a DAP 2 Protocol to netCDF-4 translation
 but that has been removed until the DAP4 protocol is available.
 
-\section nc3_trans_rules netCDF-3 Translation Rules
+\subsection nc3_trans_rules netCDF-3 Translation Rules
 
 The current default translation code translates the OPeNDAP protocol
 to netCDF-3 (classic). This netCDF-3 translation converts an OPeNDAP
@@ -225,7 +226,7 @@ Dataset {
 } D1;
 \endcode
 
-\section var_def Variable Definition
+\subsection var_def Variable Definition
 
 The set of netCDF variables is derived from the fields with primitive
 base types as they occur in Sequences, Grids, and Structures. The
@@ -244,7 +245,7 @@ within grids are left out in order to mimic the behavior of libnc-dap.
     lon
 \endcode
 
-\page dap2_reserved_keywords DAP2 Reserved Keywords
+\section dap2_reserved_keywords DAP2 Reserved Keywords
 
 In the OPeNDAP DAP2 protocol, there are a number of reserved keywords.  These keywords are case insensitive and if you use one as a netCDF variable name, you may encounter odd behavior such as case changes (depending on the client DDS/DAS parser).  The list of reserved keywords as used by the netCDF-C library parser are as follows:
 
@@ -272,7 +273,7 @@ In the OPeNDAP DAP2 protocol, there are a number of reserved keywords.  These ke
 - program
 
 
-\page var_dim_trans Variable Dimension Translation
+\section var_dim_trans Variable Dimension Translation
 
 A variable's rank is determined from three sources.
 - The variable has the dimensions associated with the field it
@@ -287,7 +288,7 @@ containers is a DAP DDS Sequence. This is discussed more fully below.
 If the type of the netCDF variable is char, then an extra string
 dimension is added as the last dimension.
 
-\section dim_trans Dimension translation
+\subsection dim_trans Dimension translation
 
 For dimensions, the rules are as follows.
 
@@ -366,7 +367,7 @@ dimensions:
   S1.FS2.f2_0 = 2 ;
 \endcode
 
-\page var_name_trans Variable Name Translation
+\section var_name_trans Variable Name Translation
 
 The steps for variable name translation are as follows.
 
@@ -435,7 +436,7 @@ int S1.FS2.f1(lat, lat) ;
 Note that this is incorrect because it dimensions S1.FS2.f1(2,2)
 rather than S1.FS2.f1(2,3).
 
-\section dap_translation Translating DAP DDS Sequences
+\subsection dap_translation Translating DAP DDS Sequences
 
 Any variable (as determined above) that is contained directly or
 indirectly by a Sequence is subject to revision of its rank using the
@@ -552,7 +553,7 @@ performance.
 
 Currently, a limited set of client parameters is
 recognized. Parameters not listed here are ignored, but no error is
-signalled.
+signalled. All names are case insensitive.
 
 Parameter Name Legal Values Semantics
 - "log" | "log=<file>" - Turn on logging and send the log output to
@@ -568,10 +569,11 @@ Parameter Name Legal Values Semantics
   enabled, then this can be helpful in checking to see the access
   behavior of the netCDF code.
 - "stringlength=NN" - Specify the default string length to use for
-  string dimensions. The default is 64.
+  string dimensions. The default is 64. The name "maxstrlen" is an
+  alias for "stringlength".
 - "stringlength_<var>=NN" - Specify the default string length to use
   for a string dimension for the specified variable. The default is
-  64.
+  64. The name "maxstrlen_<var>" is an alias for "stringlength_<var>".
 - "cache" - This enables caching.
 - "cachelimit=NN" - Specify the maximum amount of space allowed for
   the cache.

diff --git a/include/nc4internal.h b/include/nc4internal.h
@@ -102,6 +102,12 @@ typedef enum {VAR, DIM, ATT} NC_OBJ_T;
  * as the netCDF dimid. */
 #define NC_DIMID_ATT_NAME "_Netcdf4Dimid"
 
+/** This is the name of the class HDF5 dimension scale attribute. */
+#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS"
+
+/** This is the name of the name HDF5 dimension scale attribute. */
+#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
+
 /* Boolean type, to make the code easier to read */
 typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
 
@@ -126,7 +132,7 @@ typedef struct NC_DIM_INFO
    nc_bool_t unlimited;         /* True if the dimension is unlimited */
    nc_bool_t extended;          /* True if the dimension needs to be extended */
    nc_bool_t too_long;          /* True if len is too big to fit in local size_t. */
-   hid_t hdf_dimscaleid;
+   hid_t hdf_dimscaleid;        /* Non-zero if a DIM_WITHOUT_VARIABLE dataset is in use (no coord var). */
    HDF5_OBJID_T hdf5_objid;
    struct NC_VAR_INFO *coord_var; /* The coord var, if it exists. */
 } NC_DIM_INFO_T;
@@ -344,6 +350,7 @@ int nc4_convert_type(const void *src, void *dest,
 /* These functions do HDF5 things. */
 int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
 int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
+int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim);
 int nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset);
 int nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
 		 const size_t *countp, nc_type xtype, int is_long, void *op);

diff --git a/include/nc_tests.h b/include/nc_tests.h
@@ -1,11 +1,12 @@
-/** \internal
-\file
-Common includes, defines, etc., for test code in the libsrc4 and
-nc_test4 directories.
-
-This is part of the netCDF package. Copyright 2005 University
-Corporation for Atmospheric Research/Unidata. See \ref copyright file
-for conditions of use.
+/* This is part of the netCDF package. Copyright 2005 University
+ * Corporation for Atmospheric Research/Unidata. See \ref copyright
+ * file for conditions of use. */
+/** @internal @file
+ *
+ * Common includes, defines, etc., for test code in the libsrc4 and
+ * nc_test4 directories.
+ *
+ * @author Ed Hartnett, Denis Heimbigner, Ward Fisher
 */
 
 #ifndef _NC_TESTS_H
@@ -22,8 +23,10 @@ for conditions of use.
 #include "netcdf.h"
 //#include "err_macros.h"
 
+/** NC_MAX_DIMS for tests.  Allows different NC_MAX_DIMS values
+ * without breaking this test with a heap or stack overflow. */
+#define NC_TESTS_MAX_DIMS 1024
 
-#define NC_TESTS_MAX_DIMS 1024 /**< NC_MAX_DIMS for tests.  Allows different NC_MAX_DIMS values without breaking this test with a heap or stack overflow. */
 #define MAX_NUM_FORMATS 5 /**< Max number of available binary formats. */
 
 /** Useful define for tests. */
@@ -38,6 +41,7 @@ for conditions of use.
 #define ONE_TWENTY_EIGHT_MEG (SIXTEEN_MEG * 8)
 #define TEST_VAL_42 42
 #define BAD_NAME "dd//d/  "
+#define NUM_NETCDF_TYPES 12
 /** \} */
 
 #ifdef USE_PNETCDF

diff --git a/include/netcdf.h b/include/netcdf.h
@@ -254,11 +254,14 @@ These maximums are not used for netCDF-4/HDF5 files unless they were
 created with the ::NC_CLASSIC_MODEL flag.
 
 As a rule, NC_MAX_VAR_DIMS <= NC_MAX_DIMS.
+
+NOTE: The NC_MAX_DIMS, NC_MAX_ATTRS, and NC_MAX_VARS limits 
+      are *not* enforced after version 4.5.0
 */
 /**@{*/
-#define NC_MAX_DIMS     1024
-#define NC_MAX_ATTRS    8192
-#define NC_MAX_VARS     8192
+#define NC_MAX_DIMS     1024 /* not enforced after 4.5.0 */
+#define NC_MAX_ATTRS    8192 /* not enforced after 4.5.0 */
+#define NC_MAX_VARS     8192 /* not enforced after 4.5.0 */
 #define NC_MAX_NAME     256
 #define NC_MAX_VAR_DIMS 1024 /**< max per variable dimensions */
 /**@}*/

diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c
@@ -201,6 +201,7 @@ NCD2_initialize(void)
 int
 NCD2_finalize(void)
 {
+    curl_global_cleanup();
     return NC_NOERR;
 }
 
@@ -1193,6 +1194,7 @@ constrainable(NCURI* durl)
    return 0;
 }
 
+/* Lookup a parameter key; case insensitive */
 static const char*
 paramlookup(NCDAPCOMMON* state, const char* key)
 {
@@ -1261,21 +1263,29 @@ applyclientparams(NCDAPCOMMON* nccomm)
 
     /* allow embedded _ */
     value = paramlookup(nccomm,"stringlength");
+    if(value == NULL) 
+        value = paramlookup(nccomm,"maxstrlen");
     if(value != NULL && strlen(value) != 0) {
         if(sscanf(value,"%d",&len) && len > 0) dfaltstrlen = len;
-    }
+    } 
     nccomm->cdf.defaultstringlength = dfaltstrlen;
 
     /* String dimension limits apply to variables */
     for(i=0;i<nclistlength(nccomm->cdf.ddsroot->tree->varnodes);i++) {
 	CDFnode* var = (CDFnode*)nclistget(nccomm->cdf.ddsroot->tree->varnodes,i);
-	/* Define the client param stringlength for this variable*/
-	var->maxstringlength = 0; /* => use global dfalt */
-	strncpy(tmpname,"stringlength_",sizeof(tmpname));
+	/* Define the client param stringlength/maxstrlen for this variable*/
+	/* create the variable path name */
 	pathstr = makeocpathstring(conn,var->ocnode,".");
-	strlcat(tmpname,pathstr,sizeof(tmpname));
-	nullfree(pathstr);
+	var->maxstringlength = 0; /* => use global dfalt */
+	strcpy(tmpname,"stringlength_");
+	strncat(tmpname,pathstr,NC_MAX_NAME);
 	value = paramlookup(nccomm,tmpname);
+	if(value == NULL) {
+	    strcpy(tmpname,"maxstrlen_");
+	    strncat(tmpname,pathstr,NC_MAX_NAME);
+	    value = paramlookup(nccomm,tmpname);
+        }
+	nullfree(pathstr);
         if(value != NULL && strlen(value) != 0) {
             if(sscanf(value,"%d",&len) && len > 0) var->maxstringlength = len;
 	}

diff --git a/libdap4/d4file.c b/libdap4/d4file.c
@@ -90,6 +90,9 @@ NCD4_open(const char * path, int mode,
     {
 	char tmpname[NC_MAX_NAME];
 
+	/* Guarantee tmpname is a legitimate string */
+	tmpname[0] = '\0';
+
         /* Create fake file name: exact name must be unique,
            but is otherwise irrelevant because we are using NC_DISKLESS
         */
@@ -122,7 +125,7 @@ NCD4_open(const char * path, int mode,
 	    nc_set_default_format(old,&new); /* restore */
 	    d4info->substrate.realfile = ((ncflags & NC_DISKLESS) == 0);
 	    d4info->substrate.filename = strdup(tmpname);
-	    if(tmpname == NULL) ret = NC_ENOMEM;
+	    if(d4info->substrate.filename == NULL) ret = NC_ENOMEM;
 	    d4info->substrate.nc4id = ncid;
 	}
         if(ret != NC_NOERR) goto done;

diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h
@@ -41,11 +41,6 @@ defined here, including function-like #defines.
 /* Clear empty structure alignment space; Potentially costly, but probably less than  CLEARMEM */
 #define CLEARSTRUCT
 
-/**************************************************/
-/* Global state */
-
-extern NCD4globalstate* NCD4_globalstate;
-
 /**************************************************/
 
 #undef nullfree

diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c
@@ -34,9 +34,6 @@ static NC_Dispatch NCD4_dispatch_base;
 
 NC_Dispatch* NCD4_dispatch_table = NULL; /* moved here from ddispatch.c */
 
-/* Collect global state info in one place */
-NCD4globalstate* NCD4_globalstate = NULL;
-
 /* Forward */
 static int globalinit(void);
 
@@ -63,6 +60,7 @@ NCD4_initialize(void)
 int
 NCD4_finalize(void)
 {
+    curl_global_cleanup();
     return THROW(NC_NOERR);
 }
 

diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h
@@ -42,7 +42,6 @@ typedef enum NCD4mode NCD4mode;
 typedef enum NCD4translation NCD4translation;
 typedef struct NCD4curl NCD4curl;
 typedef struct NCD4meta NCD4meta;
-typedef struct NCD4globalstate NCD4globalstate;
 typedef struct NCD4node NCD4node;
 typedef struct NCD4params NCD4params;
 

diff --git a/libdispatch/dtype.c b/libdispatch/dtype.c
@@ -41,7 +41,12 @@ type). Read attributes of the new type with nc_get_att (see
 
 /** 
 \ingroup user_types
-Learn if two types are equal
+Learn if two types are equal.
+
+\note User-defined types in netCDF-4/HDF5 files must be committed to
+the file before nc_inq_type_equal() will work on the type. For
+uncommitted user-defined types, nc_inq_type_equal() will return
+::NC_EHDFERR. Commit types to the file with a call to nc_enddef().
 
 \param ncid1 \ref ncid of first typeid.
 \param typeid1 First typeid.
@@ -54,8 +59,11 @@ the two types are equal, a zero if they are not equal.
 \returns ::NC_EBADID Bad \ref ncid.
 \returns ::NC_EBADTYPE Bad type id.
 \returns ::NC_ENOTNC4 Not an netCDF-4 file, or classic model enabled.
-\returns ::NC_EHDFERR An error was reported by the HDF5 layer.
-\author Dennis Heimbigner, Ward Fisher
+\returns ::NC_EHDFERR An error was reported by the HDF5 layer. This
+will occur if either of the types have not been committed to the file
+(with an nc_enddef()).
+
+\author Dennis Heimbigner, Ward Fisher, Ed Hartnett
  */
 int
 nc_inq_type_equal(int ncid1, nc_type typeid1, int ncid2,

diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c
@@ -209,7 +209,7 @@ NC_mktmp(const char* base)
     cvtpath = NCpathcvt(base);
     strncpy(tmp,cvtpath,sizeof(tmp));
     nullfree(cvtpath);
-	strncat(tmp, "XXXXXX", sizeof(tmp));
+	strncat(tmp, "XXXXXX", sizeof(tmp) - strlen(tmp) - 1);
 
 #ifdef HAVE_MKSTEMP
     /* Note Potential problem: old versions of this function