Read + Write Zarr with consolidated metadata

CMakeLists.txt

@@ -1737,6 +1737,7 @@ endif()
 
 # The Unidata testing S3 bucket
 # WARNING: this must match the value in configure.ac
+set(S3ENDPOINT "s3.us-east-1.amazonaws.com" CACHE STRING "S3 endpoint")
 set(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket")
 
 # The working S3 path tree within the Unidata bucket.

config.h.cmake.in

@@ -181,6 +181,9 @@ are set when opening a binary file on Windows. */
 /* if true, enable S3 testing*/
 #cmakedefine WITH_S3_TESTING "${WITH_S3_TESTING}"
 
+/* S3 Test endpoint */
+#define S3ENDPOINT "${S3ENDPOINT}"
+
 /* S3 Test Bucket */
 #define S3TESTBUCKET "${S3TESTBUCKET}"
 

configure.ac

@@ -2114,6 +2114,11 @@ if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno; then
 AC_SUBST(WHICH_S3_SDK,[none])
 fi
 
+# The Unidata testing S3 bucket
+# WARNING: this must match the value in CMakeLists.txt
+AC_DEFINE([S3ENDPOINT], ["s3.us-east-1.amazonaws.com"], [S3 test endpoint])
+AC_SUBST([S3ENDPOINT],["s3.us-east-1.amazonaws.com"])
+
 # The Unidata testing S3 bucket
 # WARNING: this must match the value in CMakeLists.txt
 AC_DEFINE([S3TESTBUCKET], ["unidata-zarr-test-data"], [S3 test bucket])

include/netcdf.h

@@ -531,8 +531,9 @@ by the desired type. */
 #define NC_EOBJECT       (-140)    /**< Some object exists when it should not */
 #define NC_ENOOBJECT     (-141)    /**< Some object not found */
 #define NC_EPLUGIN       (-142)    /**< Unclassified failure in accessing a dynamically loaded plugin> */
+#define NC_ENOTZARR      (-143)    /**< Malformed (NC)Zarr file */
 
-#define NC4_LAST_ERROR   (-142)    /**< @internal All netCDF errors > this. */
+#define NC4_LAST_ERROR   (-143)    /**< @internal All netCDF errors > this. */
 
 /* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/
 #define NC_EURL         (NC_EDAPURL)   /**< Malformed URL */

libdispatch/ncjson.c

@@ -898,15 +898,26 @@ NCJaddstring(NCjson* json, int sort, const char* s)
 OPTSTATIC int
 NCJinsert(NCjson* object, const char* key, NCjson* jvalue)
 {
-    int stat = NCJ_OK;
-    NCjson* jkey = NULL;
-    if(object == NULL || object->sort != NCJ_DICT || key == NULL || jvalue == NULL)
-	{stat = NCJTHROW(NCJ_ERR); goto done;}
-    if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done;
-    if((stat = NCJappend(object,jkey))==NCJ_ERR) goto done;
-    if((stat = NCJappend(object,jvalue))==NCJ_ERR) goto done;
+	int stat = NCJ_OK;
+	NCjson *jkey = NULL;
+	if (object == NULL || object->sort != NCJ_DICT || key == NULL || jvalue == NULL) {
+		stat = NCJTHROW(NCJ_ERR);
+		goto done;
+	}
+	for (size_t i = 0; i < NCJlength(object); i += 2) {
+		jkey = NCJith(object, i);
+		if (jkey->string != NULL && strcmp(jkey->string, key) == 0) {
+			// replace existing values for new key
+			NCJreclaim(object->list.contents[i + 1]); // free old value
+			object->list.contents[i + 1] = jvalue;
+			goto done;
+		}
+	}
+	if ((stat = NCJnewstring(NCJ_STRING, key, &jkey)) == NCJ_ERR) goto done;
+	if ((stat = NCJappend(object, jkey)) == NCJ_ERR) goto done;
+	if ((stat = NCJappend(object, jvalue)) == NCJ_ERR) goto done;
 done:
-    return NCJTHROW(stat);
+	return NCJTHROW(stat);
 }
 
 /* Insert key-value pair as strings into a dict object.

libnczarr/CMakeLists.txt

@@ -23,6 +23,8 @@ zgrp.c
 zinternal.c
 zmap.c
 zmap_file.c
+zmetadata2.c
+zmetadata.c
 zodom.c
 zopen.c
 zprov.c
@@ -39,6 +41,7 @@ zdispatch.h
 zincludes.h
 zinternal.h
 zmap.h
+zmetadata.h
 zodom.h
 zprovenance.h
 zplugins.h

libnczarr/Makefile.am

@@ -43,6 +43,8 @@ zgrp.c \
 zinternal.c \
 zmap.c \
 zmap_file.c \
+zmetadata2.c \
+zmetadata.c \
 zodom.c \
 zopen.c \
 zprov.c \
@@ -59,6 +61,7 @@ zdispatch.h \
 zincludes.h \
 zinternal.h \
 zmap.h \
+zmetadata.h \
 zodom.h \
 zprovenance.h \
 zplugins.h \

libnczarr/zarr.c

@@ -79,6 +79,10 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls)
     if((stat = nczmap_create(zinfo->controls.mapimpl,nc->path,nc->mode,zinfo->controls.flags,NULL,&zinfo->map)))
 	goto done;
 
+    /* Initialize metadata handle */
+    assert(zinfo->map != NULL);
+    if((stat = NCZMD_set_metadata_handler(zinfo,(const NCZ_Metadata**)&zinfo->metadata_handler))) goto done;
+
 done:
     ncurifree(uri);
     NCJreclaim(json);
@@ -143,6 +147,10 @@ ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls)
     if((stat = nczmap_open(zinfo->controls.mapimpl,nc->path,mode,zinfo->controls.flags,NULL,&zinfo->map)))
 	goto done;
 
+    /* Initialize metadata handle */
+    assert(zinfo->map != NULL);
+    if((stat = NCZMD_set_metadata_handler(zinfo,(const NCZ_Metadata**)&zinfo->metadata_handler))) goto done;
+
     /* Ok, try to read superblock */
     if((stat = ncz_read_superblock(file,&nczarr_version,&zarr_format))) goto done;
 

libnczarr/zarr.h

@@ -69,7 +69,7 @@ EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* ty
 EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
 EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
 EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
-EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
+EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json);
 EXTERNL int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp);
 EXTERNL int NCZ_isLittleEndian(void);
 EXTERNL int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist);

libnczarr/zclose.c

@@ -51,6 +51,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort)
 	goto done;
     nclistfreeall(zinfo->controllist);
     NC_authfree(zinfo->auth);
+    NCZMD_free_metadata_handler(zinfo->metadata_handler);
     nullfree(zinfo);
 
 done:

libnczarr/zincludes.h

@@ -48,6 +48,7 @@ extern "C" {
 #include "ncjson.h"
 
 #include "zmap.h"
+#include "zmetadata.h"
 #include "zinternal.h"
 #include "zdispatch.h"
 #include "zprovenance.h"

libnczarr/zinternal.h

@@ -13,6 +13,7 @@
 #define ZINTERNAL_H
 
 #define ZARRVERSION "2"
+#define ZARRFORMAT2 2
 
 /* NCZARRVERSION is independent of Zarr version,
    but NCZARRVERSION => ZARRVERSION */
@@ -38,11 +39,12 @@
 #  endif
 #endif
 
-#define ZMETAROOT "/.zgroup"
-#define ZMETAATTR "/.zattrs"
-#define ZGROUP ".zgroup"
-#define ZATTRS ".zattrs"
-#define ZARRAY ".zarray"
+/* V2 Reserved Objects */
+#define Z2METAROOT "/.zgroup"
+#define Z2ATTSROOT "/.zattrs"
+#define Z2GROUP ".zgroup"
+#define Z2ATTRS ".zattrs"
+#define Z2ARRAY ".zarray"
 
 /* V2 Reserved Attributes */
 /*
@@ -143,6 +145,7 @@ typedef struct NCZ_FILE_INFO {
 #		define FLAG_NCZARR_KEY  16 /* _nczarr_xxx keys are stored in object and not in _nczarr_attrs */
 	NCZM_IMPL mapimpl;
     } controls;
+    struct NCZ_Metadata * metadata_handler;
     int default_maxstrlen; /* default max str size for variables of type string */
 } NCZ_FILE_INFO_T;
 

libnczarr/zmap_s3sdk.c

@@ -223,14 +223,14 @@ zs3open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
     z3map->s3client = NC_s3sdkcreateclient(&z3map->s3);
 
     /* Search the root for content */
-    content = nclistnew();
-    if((stat = NC_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
-	goto done;
-    if(nkeys == 0) {
-	/* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */
-	stat = NC_ENOOBJECT;
-	goto done;
-    }
+    // content = nclistnew();
+    // if((stat = NC_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
+	// goto done;
+    // if(nkeys == 0) {
+	// /* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */
+	// stat = NC_ENOOBJECT;
+	// goto done;
+    // }
     if(mapp) *mapp = (NCZMAP*)z3map;    
 
 done: