From 0260f9ce0cfe355bc6b5ca3ecd3244bc5b628e84 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Mon, 18 Nov 2024 09:40:41 +0100
Subject: [PATCH 01/24] Add dispatch layer for consolidated ZARR access

---
 libnczarr/CMakeLists.txt |   4 +
 libnczarr/zincludes.h    |   1 +
 libnczarr/zinternal.h    |  16 ++
 libnczarr/zmetadata.c    | 206 +++++++++++++++++++++++
 libnczarr/zmetadata.h    | 104 ++++++++++++
 libnczarr/zmetadata2.c   | 309 ++++++++++++++++++++++++++++++++++
 libnczarr/zmetadata3.c   | 349 +++++++++++++++++++++++++++++++++++++++
 7 files changed, 989 insertions(+)
 create mode 100644 libnczarr/zmetadata.c
 create mode 100644 libnczarr/zmetadata.h
 create mode 100644 libnczarr/zmetadata2.c
 create mode 100644 libnczarr/zmetadata3.c

diff --git a/libnczarr/CMakeLists.txt b/libnczarr/CMakeLists.txt
index c460e1d76e..f49dc1e736 100644
--- a/libnczarr/CMakeLists.txt
+++ b/libnczarr/CMakeLists.txt
@@ -23,6 +23,9 @@ zgrp.c
 zinternal.c
 zmap.c
 zmap_file.c
+zmetadata3.c
+zmetadata2.c
+zmetadata.c
 zodom.c
 zopen.c
 zprov.c
@@ -39,6 +42,7 @@ zdispatch.h
 zincludes.h
 zinternal.h
 zmap.h
+zmetadata.h
 zodom.h
 zprovenance.h
 zplugins.h
diff --git a/libnczarr/zincludes.h b/libnczarr/zincludes.h
index 3fdae6c6fd..4280a2ff8a 100644
--- a/libnczarr/zincludes.h
+++ b/libnczarr/zincludes.h
@@ -48,6 +48,7 @@ extern "C" {
 #include "ncjson.h"
 
 #include "zmap.h"
+#include "zmetadata.h"
 #include "zinternal.h"
 #include "zdispatch.h"
 #include "zprovenance.h"
diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h
index 2548ad54ba..fba4b4e1cc 100644
--- a/libnczarr/zinternal.h
+++ b/libnczarr/zinternal.h
@@ -43,6 +43,22 @@
 #define ZGROUP ".zgroup"
 #define ZATTRS ".zattrs"
 #define ZARRAY ".zarray"
+/* V2 Reserved Objects */
+#define Z2METAROOT "/.zgroup"
+#define Z2ATTSROOT "/.zattrs"
+#define Z2GROUP ".zgroup"
+#define Z2ATTRS ".zattrs"
+#define Z2ARRAY ".zarray"
+
+/* V3 Reserved Objects */
+#define Z3METAROOT "/zarr.json"
+#define Z3OBJECT "zarr.json"
+#define Z3GROUP Z3OBJECT
+#define Z3ATTRS Z3OBJECT
+#define Z3ARRAY Z3OBJECT
+
+/* Bytes codec name */
+#define ZBYTES3 "bytes"
 
 /* V2 Reserved Attributes */
 /*
diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
new file mode 100644
index 0000000000..070e170a8f
--- /dev/null
+++ b/libnczarr/zmetadata.c
@@ -0,0 +1,206 @@
+/*********************************************************************
+ *   Copyright 2018, UCAR/Unidata
+ *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ *********************************************************************/
+
+#include "zmetadata.h"
+
+/**************************************************/
+
+extern int NCZF2_initialize(void);
+extern int NCZF2_finalize(void);
+extern int NCZF3_initialize(void);
+extern int NCZF3_finalize(void);
+
+/**************************************************/
+////////////////////////////////////////////////////
+
+int NCZMD_initialize(void)
+{	
+    int stat = NC_NOERR;
+    if((stat=NCZMD2_initialize())) goto done;
+    if((stat=NCZMD3_initialize())) goto done;
+done:
+    return THROW(stat);
+}
+
+int NCZMD_finalize(void)
+{
+
+    int stat = NC_NOERR;
+    if((stat=NCZMD2_finalize())) goto done;
+    if((stat=NCZMD3_finalize())) goto done;
+done:
+    return THROW(stat);
+}
+
+// Returns the list of subgroups from *grp
+int NCZMD_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames)
+{
+	return zfile->metadata_handler->dispatcher->list_groups(zfile, grp, subgrpnames);
+}
+
+// Returns the list of variables from grp
+int NCZMD_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
+{
+	return zfile->metadata_handler->dispatcher->list_variables(zfile, grp, varnames);
+}
+
+
+/////////////////////////////////////////////////////////////////////
+//     Fetch JSON content from .zmetadata or storage
+/////////////////////////////////////////////////////////////////////
+
+int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jgroup)
+{
+	int stat = NC_NOERR;
+	char *key = NULL;	
+
+	if (grp && ((stat = NCZ_grpkey(grp, &key)) != NC_NOERR)) 
+		goto done;
+	if (name && (stat = nczm_concat(key, name, &key)))
+		goto done;
+
+	stat = zfile->metadata_handler->dispatcher->fetch_json_content(zfile, NCZMD_GROUP, key, jgroup);
+done:	
+	nullfree(key);
+	return stat;
+}
+
+int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jattrs)
+{
+	int stat = NC_NOERR;
+	char *key = NULL;	
+
+	if (grp && ((stat = NCZ_grpkey(grp, &key)) != NC_NOERR)) 
+		goto done;
+	if (name && (stat = nczm_concat(key, name, &key)))
+		goto done;
+
+	stat = zfile->metadata_handler->dispatcher->fetch_json_content(zfile, NCZMD_ATTRS, key, jattrs);
+done:	
+	nullfree(key);
+	return stat;
+}
+
+int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jarray)
+{
+	int stat = NC_NOERR;
+	char *key = NULL;	
+
+	if (grp && ((stat = NCZ_grpkey(grp, &key)) != NC_NOERR)) 
+		goto done;
+	if (name && (stat = nczm_concat(key, name, &key)))
+		goto done;
+
+	stat = zfile->metadata_handler->dispatcher->fetch_json_content(zfile, NCZMD_ARRAY, key, jarray);
+done:	
+	nullfree(key);
+	return stat;
+}
+
+////////////////////////////////////////////////////////////////////////////
+
+int NCZMD_is_metadata_consolidated(NCZ_FILE_INFO_T *zfile)
+{
+	NCZ_Metadata *zmd = NULL;
+	zmd = zfile->metadata_handler;
+	if (zmd == NULL ||
+	    zmd->jcsl == NULL ||
+	    NCJsort(zmd->jcsl) != NCJ_DICT ||
+	    !(zmd->dispatcher->flags & ZARR_CONSOLIDATED))
+	{
+		return NC_ENOOBJECT;
+	}
+	return NC_NOERR;
+}
+
+int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat)
+{ // Only pure Zarr is determined
+
+	NCZ_Metadata *zmd = zfile->metadata_handler;
+	if ( !zmd || !zmd->dispatcher ) {
+		return NC_EFILEMETA;
+	}
+
+
+	if (zmd->dispatcher->zarr_format >= ZARRFORMAT2)
+	{
+		*zarrformat = zmd->dispatcher->zarr_format;
+		return NC_NOERR;
+	}
+
+	// At this point, we know that version 3 cannot be
+	// otherwise the zarr.json would have been found
+	// when setting the handler.
+	// Last thing to do is to look for:
+	//      .zattrs, .zgroup or .zarray
+
+	if (!nczmap_exists(zfile->map, "/" Z2ATTRS) && !nczmap_exists(zfile->map, "/" Z2GROUP) && !nczmap_exists(zfile->map, "/" Z2ARRAY))
+	{
+		return NC_ENOTZARR;
+	}
+
+	*zarrformat = ZARRFORMAT2;
+	return NC_NOERR;
+}
+
+int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdhandlerp)
+{
+	int stat = NC_NOERR;
+	const NCZ_Metadata_Dispatcher *zmd_dispatcher = NULL;
+	NCjson *jcsl = NULL;
+
+	if (zfile->metadata_handler != NULL)
+	{
+		stat = NC_EOBJECT;
+		goto done;
+	}
+
+	zmd_dispatcher = NCZ_metadata_handler2;
+
+	if ((stat = NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl)) != NC_NOERR || jcsl == NULL)
+	{
+		if ((stat = NCZ_downloadjson(zfile->map, Z3METADATA, &jcsl)) != NC_NOERR || jcsl == NULL)
+		{
+			/* We've tried to get any json 
+			objects for consolidated access
+			but they seem to be absent */
+			goto done;
+		}
+		NCjson * jtmp = NULL;
+		if (NCJsort(jcsl) != NCJ_DICT || NCJdictget(jcsl,"consolidated_metadata", &jtmp) || NCJsort(jtmp) != NCJ_DICT)
+		{
+			zmd_dispatcher = NCZ_metadata_handler3;
+		}
+		else
+		{
+			zmd_dispatcher = NCZ_csl_metadata_handler3;
+		}
+	}
+	else
+	{
+		zmd_dispatcher = NCZ_csl_metadata_handler2;
+	}
+
+	if (jcsl != NULL && NCJsort(jcsl) == NCJ_DICT)
+	{
+		NCZ_Metadata *zmdh = NULL;
+		if ((zmdh = (NCZ_Metadata *)calloc(1, sizeof(NCZ_Metadata))) == NULL)
+		{
+			stat = NC_ENOMEM;
+			goto done;
+		}
+		zmdh->jcsl = jcsl;
+		zmdh->dispatcher = zmd_dispatcher;
+
+		*mdhandlerp = (const NCZ_Metadata *)zmdh;
+	}
+	else
+	{
+		NCJreclaim(jcsl); // Free jcsl if not assigned
+	}
+
+done:
+	return stat;
+}
diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h
new file mode 100644
index 0000000000..4b1a6bfe79
--- /dev/null
+++ b/libnczarr/zmetadata.h
@@ -0,0 +1,104 @@
+/* Copyright 2018-2018 University Corporation for Atmospheric
+   Research/Unidata. */
+
+
+#ifndef ZMETADATA_H
+#define ZMETADATA_H
+#include "zincludes.h"
+#include "ncjson.h"
+#include "zinternal.h"
+
+/*
+Notes on internal architecture.
+
+Encapsulating Zarr metadata operations across versions.
+Such allows to use the same interface for both consolidated
+access as well as fetching each and every object on the filesystem
+
+The dispatcher is defined by the type NCZ_Metadata_Dispatcher.
+That dispatcher allows the Zarr independent code to be 
+isolated from the code handling the json files.
+The table has the following groups of entries:
+1. List variables within a group
+2. List subgroups withing a group
+3. Retrieve JSON representation of (sub)groups, arrays and attributes
+    depending on the implementation it might require fetch the content
+    of the json file or process the consolidated json to retrive
+    the corrent part
+   Note: This is also the case of v3, the elements will be extracted from zarr.json
+
+*/
+
+/* This is the version of the metadata table. It should be changed
+ * when new functions are added to the metadata table. */
+#ifndef NCZ_METADATA_VERSION
+#define NCZ_METADATA_VERSION 1
+#endif /*NCZ_METADATA_VERSION*/
+
+#define Z2METADATA "/.zmetadata"
+#define Z3METADATA "/zarr.json"
+
+typedef enum
+{
+	NCZMD_NULL,
+	NCZMD_GROUP,
+	NCZMD_ATTRS,
+	NCZMD_ARRAY
+} NCZMD_MetadataType;
+
+typedef struct NCZ_Metadata_Dispatcher
+{
+	int zarr_format;
+	int dispatch_version; /* Version of the dispatch table */
+	size64_t flags;
+#define ZARR_NOT_CONSOLIDATED 0
+#define ZARR_CONSOLIDATED 1
+
+	int (*list_groups)(NCZ_FILE_INFO_T *, NC_GRP_INFO_T *, NClist *subgrpnames);
+	int (*list_variables)(NCZ_FILE_INFO_T *, NC_GRP_INFO_T *, NClist *varnames);
+
+	int (*fetch_json_content)(NCZ_FILE_INFO_T *, NCZMD_MetadataType, const char *name, NCjson **jobj);
+} NCZ_Metadata_Dispatcher;
+
+typedef struct NCZ_Metadata
+{
+	NCjson *jcsl; // Internal JSON configuration
+	const NCZ_Metadata_Dispatcher *dispatcher;
+} NCZ_Metadata;
+
+// "normal" handlers
+extern const NCZ_Metadata_Dispatcher *NCZ_metadata_handler2;
+extern const NCZ_Metadata_Dispatcher *NCZ_metadata_handler3;
+
+// "consolidated" metadata handlers
+extern const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler2;
+extern const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler3;
+
+#if defined(__cplusplus)
+extern "C"
+{
+#endif
+
+/* Called by nc_initialize and nc_finalize respectively */
+extern int NCZMD_initialize(void);
+extern int NCZMD_finalize(void);
+
+extern int NCZMD_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+extern int NCZMD_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames);
+
+extern int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jgroup);
+extern int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jattrs);
+extern int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jarrays);
+
+/**************************************************/
+
+/* Inference for the Metadata handler */
+extern int NCZMD_is_metadata_consolidated(NCZ_FILE_INFO_T *zfile);
+extern int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat); // Only pure Zarr is determined
+extern int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdhandlerp);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* ZMETADATA_H */
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
new file mode 100644
index 0000000000..016fabd232
--- /dev/null
+++ b/libnczarr/zmetadata2.c
@@ -0,0 +1,309 @@
+/*********************************************************************
+ *   Copyright 2018, UCAR/Unidata
+ *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ *********************************************************************/
+
+#include "zmetadata.h"
+
+/**************************************************/
+
+extern int NCZF2_initialize(void);
+extern int NCZF2_finalize(void);
+
+int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+
+int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+int NCZMD_v2_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+
+
+int v2_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
+int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
+
+/**************************************************/
+
+static const NCZ_Metadata_Dispatcher NCZ_md2_table = {
+    ZARRFORMAT2,
+    NCZ_METADATA_VERSION,  /* Version of the dispatch table */
+    ZARR_NOT_CONSOLIDATED, /* Flags*/
+
+    .list_groups = NCZMD_v2_list_groups,
+    .list_variables = NCZMD_v2_list_variables,
+
+    .fetch_json_content = v2_json_content,
+};
+
+static const NCZ_Metadata_Dispatcher NCZ_csl_md2_table = {
+    ZARRFORMAT2,
+    NCZ_METADATA_VERSION, /* Version of the dispatch table */
+    ZARR_CONSOLIDATED,	  /* Flags*/
+
+    .list_groups = NCZMD_v2_csl_list_groups,
+    .list_variables = NCZMD_v2_csl_list_variables,
+
+    .fetch_json_content = v2_csl_json_content,
+};
+
+const NCZ_Metadata_Dispatcher *NCZ_metadata_handler2 = &NCZ_md2_table;
+const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler2 = &NCZ_csl_md2_table;
+
+/******************************************************/
+
+int
+NCZMD2_initialize(void)
+{
+    return NC_NOERR;
+}
+
+int
+NCZMD2_finalize(void)
+{
+    return NC_NOERR;
+}
+
+////////////////////////////////////////////////////
+
+int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames)
+{
+	size_t i;
+	int stat = NC_NOERR;
+	char *grpkey = NULL;
+	char *subkey = NULL;
+	char *zgroup = NULL;
+	NClist *matches = nclistnew();
+
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+		goto done;
+	for (i = 0; i < nclistlength(matches); i++)
+	{
+		const char *name = nclistget(matches, i);
+		if (name[0] == NCZM_DOT)
+			continue; /* zarr/nczarr specific */
+		/* See if name/.zgroup exists */
+		if ((stat = nczm_concat(grpkey, name, &subkey)))
+			goto done;
+		if ((stat = nczm_concat(subkey, Z2GROUP, &zgroup)))
+			goto done;
+		if ((stat = nczmap_exists(zfile->map, zgroup)) == NC_NOERR)
+			nclistpush(subgrpnames, strdup(name));
+		stat = NC_NOERR;
+		nullfree(subkey);
+		subkey = NULL;
+		nullfree(zgroup);
+		zgroup = NULL;
+	}
+done:
+	nullfree(grpkey);
+	nullfree(subkey);
+	nullfree(zgroup);
+	nclistfreeall(matches);
+	return stat;
+}
+
+int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames)
+{
+	size_t i;
+	int stat = NC_NOERR;
+	char *grpkey = NULL;
+	char *subkey = NULL;
+	char *zgroup = NULL;
+	NClist *matches = nclistnew();
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	const char *group = grpkey + (grpkey[0] == '/');
+	size_t lgroup = strlen(group);
+
+	const NCjson *jmetadata = NULL;
+	NCJdictget(zfile->metadata_handler->jcsl, "metadata", &jmetadata);
+	for (i = 0; i < NCJarraylength(jmetadata); i += 2)
+	{
+		NCjson *jname = NCJith(jmetadata, i);
+		const char *fullname = NCJstring(jname);
+		size_t lfullname = strlen(fullname);
+
+		if (lfullname < lgroup ||
+		    strncmp(fullname, group, lgroup) ||
+		    (lgroup > 0 && fullname[lgroup] != NCZM_SEP[0]))
+		{
+			continue;
+		}
+		char *start = fullname + lgroup + (lgroup > 0);
+		char *end = strchr(start, NCZM_SEP[0]);
+		if (end == NULL)
+			continue;
+		size_t lname = end - start;
+		// Ends with "/.zgroup
+		if (strncmp(Z2METAROOT, end, sizeof(Z2METAROOT)) == 0)
+		{
+			nclistpush(subgrpnames, strndup(start, lname));
+		}
+	}
+done:
+	nullfree(grpkey);
+	nullfree(subkey);
+	nullfree(zgroup);
+	nclistfreeall(matches);
+	return stat;
+}
+
+int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
+{
+	int stat = NC_NOERR;
+	size_t i;
+	char *grpkey = NULL;
+	char *varkey = NULL;
+	char *zarray = NULL;
+	NClist *matches = nclistnew();
+
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	/* Get the map and search group */
+	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+		goto done;
+	for (i = 0; i < nclistlength(matches); i++)
+	{
+		const char *name = nclistget(matches, i);
+		if (name[0] == NCZM_DOT)
+			continue; /* zarr/nczarr specific */
+		/* See if name/.zarray exists */
+		if ((stat = nczm_concat(grpkey, name, &varkey)))
+			goto done;
+		if ((stat = nczm_concat(varkey, Z2ARRAY, &zarray)))
+			goto done;
+		switch (stat = nczmap_exists(zfile->map, zarray))
+		{
+		case NC_NOERR:
+			nclistpush(varnames, strdup(name));
+			break;
+		case NC_ENOOBJECT:
+			stat = NC_NOERR;
+			break; /* ignore */
+		default:
+			goto done;
+		}
+		nullfree(varkey);
+		varkey = NULL;
+		nullfree(zarray);
+		zarray = NULL;
+	}
+
+done:
+	nullfree(grpkey);
+	nullfree(varkey);
+	nullfree(zarray);
+	nclistfreeall(matches);
+	return stat;
+}
+
+int NCZMD_v2_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
+{
+	size_t i;
+	int stat = NC_NOERR;
+	char *grpkey = NULL;
+	char *varkey = NULL;
+	char *zarray = NULL;
+	NClist *matches = nclistnew();
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	const char *group = grpkey + (grpkey[0] == '/');
+	size_t lgroup = strlen(group);
+
+	const NCjson *jmetadata = NULL;
+	NCJdictget(zfile->metadata_handler->jcsl, "metadata", &jmetadata);
+	for (i = 0; i < NCJarraylength(jmetadata); i += 2)
+	{
+		NCjson *jname = NCJith(jmetadata, i);
+		const char *fullname = NCJstring(jname);
+		size_t lfullname = strlen(fullname);
+		if (lfullname < lgroup ||
+		    strncmp(fullname, group, lgroup) ||
+		    (lgroup > 0 && fullname[lgroup] != NCZM_SEP[0]))
+		{
+			continue;
+		}
+		char *start = fullname + lgroup + (lgroup > 0);
+		char *end = strchr(start, NCZM_SEP[0]);
+		if (end == NULL)
+			continue;
+		size_t lname = end - start;
+		// Ends with ".zarray"
+		if (strncmp("/" Z2ARRAY, end, sizeof("/" Z2ARRAY)) == 0)
+		{
+			nclistpush(varnames, strndup(start, lname));
+		}
+	}
+done:
+	nullfree(grpkey);
+	nullfree(varkey);
+	nullfree(zarray);
+	nclistfreeall(matches);
+	return stat;
+}
+
+// Static function only valid for V2!
+static int zarr_obj_type2suffix(NCZMD_MetadataType zarr_obj_type, const char **suffix){
+	switch (zarr_obj_type)
+	{
+		case NCZMD_GROUP:
+			*suffix = Z2GROUP;
+			break;
+		case NCZMD_ATTRS:
+			*suffix = Z2ATTRS;
+			break;
+		case NCZMD_ARRAY:
+			*suffix = Z2ARRAY;
+			break;
+		default:
+			return NC_EINVAL; // 	Invalid type
+	}
+	return NC_NOERR;
+}
+
+int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
+{
+	int stat = NC_NOERR;
+	NCjson *jtmp = NULL;
+	const char *suffix;
+	char * key = NULL;
+	if ( (stat = zarr_obj_type2suffix(zobj_t, &suffix))
+	   ||(stat = nczm_concat(prefix, suffix, &key))){
+		return stat;
+	}
+	
+	if (NCJdictget(zfile->metadata_handler->jcsl, "metadata", &jtmp) == 0 
+	&& jtmp && NCJsort(jtmp) == NCJ_DICT)
+	{
+		NCjson *tmp = NULL;
+		if ((stat = NCJdictget(jtmp, key + (key[0] == '/'), &tmp)))
+			goto done;
+		if (tmp)
+			NCJclone(tmp, jobj);
+	}
+done:
+	nullfree(key);
+	return stat;
+
+}
+
+int v2_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
+{
+	int stat = NC_NOERR;
+	const char *suffix;
+	char * key = NULL;
+	if ((stat = zarr_obj_type2suffix(zobj_t, &suffix))
+	    || (stat = nczm_concat(prefix, suffix, &key))){
+		goto done;
+	}
+
+	stat = NCZ_downloadjson(zfile->map, key, jobj);
+done:
+	nullfree(key);
+	return stat;
+}
+////////////////////////////////////////////////////////////////////////////
\ No newline at end of file
diff --git a/libnczarr/zmetadata3.c b/libnczarr/zmetadata3.c
new file mode 100644
index 0000000000..fd120c125a
--- /dev/null
+++ b/libnczarr/zmetadata3.c
@@ -0,0 +1,349 @@
+/*********************************************************************
+ *   Copyright 2018, UCAR/Unidata
+ *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ *********************************************************************/
+
+#include "zmetadata.h"
+
+/**************************************************/
+
+extern int NCZF3_initialize(void);
+extern int NCZF3_finalize(void);
+
+int NCZMD_v3_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+int NCZMD_v3_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+
+int NCZMD_v3_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+int NCZMD_v3_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
+
+
+int v3_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
+int v3_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
+
+/**************************************************/
+
+static const NCZ_Metadata_Dispatcher NCZ_md3_table = {
+    ZARRFORMAT3,
+    NCZ_METADATA_VERSION,  /* Version of the dispatch table */
+    ZARR_NOT_CONSOLIDATED, /* Flags*/
+
+    NCZMD_v3_list_groups,
+    NCZMD_v3_list_variables,
+
+    .fetch_json_content = v3_json_content,
+};
+
+static const NCZ_Metadata_Dispatcher NCZ_csl_md3_table = {
+    ZARRFORMAT3,
+    NCZ_METADATA_VERSION, /* Version of the dispatch table */
+    ZARR_CONSOLIDATED,	  /* Flags*/
+
+    NCZMD_v3_csl_list_groups,
+    NCZMD_v3_csl_list_variables,
+
+    .fetch_json_content = v3_csl_json_content,
+};
+
+const NCZ_Metadata_Dispatcher *NCZ_metadata_handler3 = &NCZ_md3_table;
+const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler3 = &NCZ_csl_md3_table;
+int
+NCZMD3_initialize(void)
+{
+    return NC_NOERR;
+}
+
+int
+NCZMD3_finalize(void)
+{
+    return NC_NOERR;
+}
+
+////////////////////////////////////////////////////
+
+int NCZMD_v3_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames)
+{
+	size_t i;
+	int stat = NC_NOERR;
+	char *grpkey = NULL;
+	char *subkey = NULL;
+	char *zgroup = NULL;
+	NClist *matches = nclistnew();
+
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+		goto done;
+	for (i = 0; i < nclistlength(matches); i++)
+	{
+		const char *name = nclistget(matches, i);
+		if (name[0] == NCZM_DOT)
+			continue; /* zarr/nczarr specific */
+		/* See if name/.zgroup exists */
+		if ((stat = nczm_concat(grpkey, name, &subkey)))
+			goto done;
+		if ((stat = nczm_concat(subkey, Z3GROUP, &zgroup)))
+			goto done;
+		if ((stat = nczmap_exists(zfile->map, zgroup)) == NC_NOERR)
+			nclistpush(subgrpnames, strdup(name));
+		stat = NC_NOERR;
+		nullfree(subkey);
+		subkey = NULL;
+		nullfree(zgroup);
+		zgroup = NULL;
+	}
+done:
+	nullfree(grpkey);
+	nullfree(subkey);
+	nullfree(zgroup);
+	nclistfreeall(matches);
+	return stat;
+}
+
+int extract_node_types(const NCjson *jobj, const char * prefix, const char *node_type, NClist **nodenames){
+	size_t lprefix = strlen(prefix);
+	for (int i = 0; i < NCJarraylength(jobj); i += 2)	{
+		const NCjson *jname = NCJith(jobj, i);
+		const NCjson *jvalue = NCJith(jobj, i+1);
+
+		//check if json types are the expected string for key and dict for value
+		if (NCJsort(jname) != NCJ_STRING || NCJsort(jvalue) != NCJ_DICT){
+			continue;
+		}
+		// check if it value dict has key "node_type" and matches "array"
+		if (NCJdictget(jvalue, "node_type", &jvalue) || NCJsort(jvalue) != NCJ_STRING 
+		|| strncmp(node_type, NCJstring(jvalue), strlen(node_type + 1))){
+			continue;
+		}
+		// check if prefix matches item's name but ensure it's not looking at itself
+		if(strncmp(prefix, NCJstring(jname),lprefix) || lprefix == strlen(NCJstring(jname))){
+			continue;
+		}
+	
+		// if we've reach here it means it's the correct type for sure
+		// but we must check if it's directly under our current group
+
+		const char *start = NCJstring(jname) + lprefix + (lprefix > 0);
+		if (strchr(start, NCZM_SEP[0])) // Zarr separator found => not a direct group under grpkey
+			continue;
+
+		// It's a subgroup
+		const size_t lname = strlen(start);
+		nclistpush(nodenames, strndup(start, lname));
+	}
+	return NC_NOERR;
+}
+
+int NCZMD_v3_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames)
+{
+	int stat = NC_NOERR;
+	char *grpkey = NULL;
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	// In the consolidated representation '/' is never observed as the key's first char
+	const char *group = grpkey + (grpkey[0] == '/'); 
+	const size_t lgroup = strlen(group);
+
+	const NCjson *jmetadata = NULL;
+	if (( stat = NCJdictget(zfile->metadata_handler->jcsl, "consolidated_metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
+		goto done;
+	}
+
+	if (( stat = NCJdictget(jmetadata, "metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
+		goto done;
+	}
+
+
+	// for each key, check if is a dict and has node_type ="group"
+	extract_node_types(jmetadata, group, "group", subgrpnames);
+	
+done:
+	nullfree(grpkey);
+	return stat;
+}
+
+
+int NCZMD_v3_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
+{
+	int stat = NC_NOERR;
+	size_t i;
+	char *grpkey = NULL;
+	char *varkey = NULL;
+	char *zarray = NULL;
+	NClist *matches = nclistnew();
+
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	/* Get the map and search group */
+	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+		goto done;
+	for (i = 0; i < nclistlength(matches); i++)
+	{
+		const char *name = nclistget(matches, i);
+		if (name[0] == NCZM_DOT)
+			continue; /* zarr/nczarr specific */
+		/* See if name/.zarray exists */
+		if ((stat = nczm_concat(grpkey, name, &varkey)))
+			goto done;
+		if ((stat = nczm_concat(varkey, Z3ARRAY, &zarray)))
+			goto done;
+		switch (stat = nczmap_exists(zfile->map, zarray))
+		{
+		case NC_NOERR:
+			nclistpush(varnames, strdup(name));
+			break;
+		case NC_ENOOBJECT:
+			stat = NC_NOERR;
+			break; /* ignore */
+		default:
+			goto done;
+		}
+		nullfree(varkey);
+		varkey = NULL;
+		nullfree(zarray);
+		zarray = NULL;
+	}
+
+done:
+	nullfree(grpkey);
+	nullfree(varkey);
+	nullfree(zarray);
+	nclistfreeall(matches);
+	return stat;
+}
+
+int NCZMD_v3_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
+{
+	size_t i;
+	int stat = NC_NOERR;
+	char *grpkey = NULL;
+	/* Compute the key for the grp */
+	if ((stat = NCZ_grpkey(grp, &grpkey)))
+		goto done;
+	// In the consolidated representation '/' is never observed as the key's first char
+	const char *group = grpkey + (grpkey[0] == '/'); 
+	const size_t lgroup = strlen(group);
+
+	const NCjson *jmetadata = NULL;
+	if (( stat = NCJdictget(zfile->metadata_handler->jcsl, "consolidated_metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
+		goto done;
+	}
+
+	if (( stat = NCJdictget(jmetadata, "metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
+		goto done;
+	}
+
+	// for each key, check if is a dict and has node_type ="array"
+	extract_node_types(jmetadata, group, "array", varnames);
+
+done:
+	nullfree(grpkey);
+	return stat;
+}
+
+// Returns error if consolidated metadata is not found 
+// node_name should match item name on the consolidated representation, meaning no '/' prefix
+static int get_consolidated_json_node(const NCjson * zobj, const char* c_node_name ,const char * node_type, const NCjson ** jitem){
+	size_t i;
+	int stat = NC_NOERR;
+
+	// In the consolidated representation '/' is never observed as the key's first char
+	const char *node_name = c_node_name + (c_node_name[0] == '/'); 
+	const size_t lnode_name = strlen(node_name);
+
+	const NCjson *jmetadata = NULL;
+	if (( stat = NCJdictget(zobj, "consolidated_metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
+		goto done;
+	}
+
+	if (( stat = NCJdictget(jmetadata, "metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
+		goto done;
+	}
+
+	// for each key, check if is a dict and has matching node_type 
+	for (i = 0; i < NCJarraylength(jmetadata); i += 2)	{
+		const NCjson *jname = NCJith(jmetadata, i);
+		if (NCJsort(jname) != NCJ_STRING || strncmp(NCJstring(jname),node_name, lnode_name+1)){
+			continue;
+		}
+
+		const NCjson *jvalue = NCJith(jmetadata, i+1);
+		//check if json types are the expected string for key and dict for value
+		if (NCJsort(jname) != NCJ_STRING ||NCJsort(jvalue) != NCJ_DICT){
+			continue;
+		}
+
+		// check if it value dict has key "node_type" and matches node_type ["array","group"]
+		if (NCJsort(jvalue) != NCJ_DICT	|| NCJdictget(jvalue, "node_type", &jname) || jname == NULL 
+		 || NCJsort(jname) != NCJ_STRING || strncmp(node_type, NCJstring(jname),strlen(node_type))){
+			continue;
+		}
+		
+		*jitem = jvalue;
+		break; //object found!		
+	}
+done:
+	return stat;
+}
+
+// Static function only valid for v3!
+static int zarr_obj_type2suffix(NCZMD_MetadataType zarr_obj_type, const char **suffix){
+	switch (zarr_obj_type)
+	{
+		case NCZMD_GROUP:
+			*suffix = Z3GROUP;
+			break;
+		case NCZMD_ATTRS:
+			*suffix = Z3ATTRS;
+			break;
+		case NCZMD_ARRAY:
+			*suffix = Z3ARRAY;
+			break;
+		default:
+			return NC_EINVAL; // 	Invalid type
+	}
+	return NC_NOERR;
+}
+
+int v3_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
+{
+	int stat = NC_NOERR;
+	const char *suffix;
+	char * key = NULL;
+	if ( (stat = zarr_obj_type2suffix(zobj_t, &suffix))
+	   ||(stat = nczm_concat(prefix, suffix, &key))){
+		goto done;
+	}
+	
+	if (prefix == NULL || (*prefix == '/' && strlen(prefix)==1) ){
+		stat = NCJclone(zfile->metadata_handler->jcsl, jobj);
+		goto done;		
+	}
+
+
+	stat = get_consolidated_json_node(zfile->metadata_handler->jcsl, prefix, "group", jobj);
+
+done:
+	nullfree(key);
+	return stat;
+
+}
+
+int v3_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
+{
+	int stat = NC_NOERR;
+	const char *suffix;
+	char * key = NULL;
+	if ((stat = zarr_obj_type2suffix(zobj_t, &suffix))
+	    || (stat = nczm_concat(prefix, suffix, &key))){
+		goto done;
+	}
+
+	stat = NCZ_downloadjson(zfile->map, key, jobj);
+done:
+	nullfree(key);
+	return stat;
+}
+////////////////////////////////////////////////////////////////////////////
\ No newline at end of file

From 0832d450d207223fe43a9ee619bb722f9a29bff8 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Fri, 22 Nov 2024 11:51:20 +0100
Subject: [PATCH 02/24] Skip s3 listing until actually necessary

---
 libnczarr/zmap_s3sdk.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libnczarr/zmap_s3sdk.c b/libnczarr/zmap_s3sdk.c
index 552a73473d..4f4fd7a35a 100644
--- a/libnczarr/zmap_s3sdk.c
+++ b/libnczarr/zmap_s3sdk.c
@@ -223,14 +223,14 @@ zs3open(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m
     z3map->s3client = NC_s3sdkcreateclient(&z3map->s3);
 
     /* Search the root for content */
-    content = nclistnew();
-    if((stat = NC_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
-	goto done;
-    if(nkeys == 0) {
-	/* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */
-	stat = NC_ENOOBJECT;
-	goto done;
-    }
+    // content = nclistnew();
+    // if((stat = NC_s3sdkgetkeys(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
+	// goto done;
+    // if(nkeys == 0) {
+	// /* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */
+	// stat = NC_ENOOBJECT;
+	// goto done;
+    // }
     if(mapp) *mapp = (NCZMAP*)z3map;    
 
 done:

From 5061334096c23254e028242a0238c91015b73c0a Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Fri, 22 Nov 2024 13:50:55 +0100
Subject: [PATCH 03/24] Resolving rebase issues

---
 libnczarr/zmetadata2.c | 8 ++++----
 libnczarr/zmetadata3.c | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index 016fabd232..10a1fcc97d 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -75,7 +75,7 @@ int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *sub
 	/* Compute the key for the grp */
 	if ((stat = NCZ_grpkey(grp, &grpkey)))
 		goto done;
-	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+	if ((stat = nczmap_search(zfile->map, grpkey, matches)))
 		goto done;
 	for (i = 0; i < nclistlength(matches); i++)
 	{
@@ -119,7 +119,7 @@ int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist
 
 	const NCjson *jmetadata = NULL;
 	NCJdictget(zfile->metadata_handler->jcsl, "metadata", &jmetadata);
-	for (i = 0; i < NCJarraylength(jmetadata); i += 2)
+	for (i = 0; i < NCJlength(jmetadata); i += 2)
 	{
 		NCjson *jname = NCJith(jmetadata, i);
 		const char *fullname = NCJstring(jname);
@@ -163,7 +163,7 @@ int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *
 	if ((stat = NCZ_grpkey(grp, &grpkey)))
 		goto done;
 	/* Get the map and search group */
-	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+	if ((stat = nczmap_search(zfile->map, grpkey, matches)))
 		goto done;
 	for (i = 0; i < nclistlength(matches); i++)
 	{
@@ -216,7 +216,7 @@ int NCZMD_v2_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NCli
 
 	const NCjson *jmetadata = NULL;
 	NCJdictget(zfile->metadata_handler->jcsl, "metadata", &jmetadata);
-	for (i = 0; i < NCJarraylength(jmetadata); i += 2)
+	for (i = 0; i < NCJlength(jmetadata); i += 2)
 	{
 		NCjson *jname = NCJith(jmetadata, i);
 		const char *fullname = NCJstring(jname);
diff --git a/libnczarr/zmetadata3.c b/libnczarr/zmetadata3.c
index fd120c125a..92741e1d6e 100644
--- a/libnczarr/zmetadata3.c
+++ b/libnczarr/zmetadata3.c
@@ -72,7 +72,7 @@ int NCZMD_v3_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *sub
 	/* Compute the key for the grp */
 	if ((stat = NCZ_grpkey(grp, &grpkey)))
 		goto done;
-	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+	if ((stat = nczmap_search(zfile->map, grpkey, matches)))
 		goto done;
 	for (i = 0; i < nclistlength(matches); i++)
 	{
@@ -102,7 +102,7 @@ int NCZMD_v3_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *sub
 
 int extract_node_types(const NCjson *jobj, const char * prefix, const char *node_type, NClist **nodenames){
 	size_t lprefix = strlen(prefix);
-	for (int i = 0; i < NCJarraylength(jobj); i += 2)	{
+	for (int i = 0; i < NCJlength(jobj); i += 2)	{
 		const NCjson *jname = NCJith(jobj, i);
 		const NCjson *jvalue = NCJith(jobj, i+1);
 
@@ -177,7 +177,7 @@ int NCZMD_v3_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *
 	if ((stat = NCZ_grpkey(grp, &grpkey)))
 		goto done;
 	/* Get the map and search group */
-	if ((stat = nczmap_list(zfile->map, grpkey, matches)))
+	if ((stat = nczmap_search(zfile->map, grpkey, matches)))
 		goto done;
 	for (i = 0; i < nclistlength(matches); i++)
 	{
@@ -263,7 +263,7 @@ static int get_consolidated_json_node(const NCjson * zobj, const char* c_node_na
 	}
 
 	// for each key, check if is a dict and has matching node_type 
-	for (i = 0; i < NCJarraylength(jmetadata); i += 2)	{
+	for (i = 0; i < NCJlength(jmetadata); i += 2)	{
 		const NCjson *jname = NCJith(jmetadata, i);
 		if (NCJsort(jname) != NCJ_STRING || strncmp(NCJstring(jname),node_name, lnode_name+1)){
 			continue;

From 19046fc7d86240d2bf74aba2bb7bde01ae1461a5 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Fri, 22 Nov 2024 13:54:48 +0100
Subject: [PATCH 04/24] Use zmetadata layer

---
 include/netcdf.h      |  3 ++-
 libnczarr/zarr.c      |  4 ++++
 libnczarr/zinternal.h |  3 +++
 libnczarr/zsync.c     | 16 +++++++++++-----
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/include/netcdf.h b/include/netcdf.h
index 234a7e4ce4..2b100563a7 100644
--- a/include/netcdf.h
+++ b/include/netcdf.h
@@ -531,8 +531,9 @@ by the desired type. */
 #define NC_EOBJECT       (-140)    /**< Some object exists when it should not */
 #define NC_ENOOBJECT     (-141)    /**< Some object not found */
 #define NC_EPLUGIN       (-142)    /**< Unclassified failure in accessing a dynamically loaded plugin> */
+#define NC_ENOTZARR      (-143)    /**< Malformed (NC)Zarr file */
 
-#define NC4_LAST_ERROR   (-142)    /**< @internal All netCDF errors > this. */
+#define NC4_LAST_ERROR   (-143)    /**< @internal All netCDF errors > this. */
 
 /* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/
 #define NC_EURL         (NC_EDAPURL)   /**< Malformed URL */
diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c
index 9ff7893a7f..438ad71713 100644
--- a/libnczarr/zarr.c
+++ b/libnczarr/zarr.c
@@ -143,6 +143,10 @@ ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls)
     if((stat = nczmap_open(zinfo->controls.mapimpl,nc->path,mode,zinfo->controls.flags,NULL,&zinfo->map)))
 	goto done;
 
+    /* Initialize metadata handle */
+    assert(zinfo->map != NULL);
+    if((stat = NCZMD_set_metadata_handler(zinfo,&zinfo->metadata_handler))) goto done;
+
     /* Ok, try to read superblock */
     if((stat = ncz_read_superblock(file,&nczarr_version,&zarr_format))) goto done;
 
diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h
index fba4b4e1cc..2d78b1faf4 100644
--- a/libnczarr/zinternal.h
+++ b/libnczarr/zinternal.h
@@ -13,6 +13,8 @@
 #define ZINTERNAL_H
 
 #define ZARRVERSION "2"
+#define ZARRFORMAT2 2
+#define ZARRFORMAT3 3
 
 /* NCZARRVERSION is independent of Zarr version,
    but NCZARRVERSION => ZARRVERSION */
@@ -159,6 +161,7 @@ typedef struct NCZ_FILE_INFO {
 #		define FLAG_NCZARR_KEY  16 /* _nczarr_xxx keys are stored in object and not in _nczarr_attrs */
 	NCZM_IMPL mapimpl;
     } controls;
+    struct NCZ_Metadata * metadata_handler;
     int default_maxstrlen; /* default max str size for variables of type string */
 } NCZ_FILE_INFO_T;
 
diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index c9d55ee751..7c244598be 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -1141,8 +1141,9 @@ define_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
     /* Construct grp path */
     if((stat = NCZ_grpkey(grp,&fullpath))) goto done;
 
+    stat = NCZMD_fetch_json_group(zinfo, grp, fullpath, &zgrp->zgroup.obj);
+    stat = NCZMD_fetch_json_attrs(zinfo, grp, fullpath, &zgrp->zgroup.atts);
     /* Download .zgroup and .zattrs */
-    if((stat = downloadzarrobj(file,&zgrp->zgroup,fullpath,ZGROUP))) goto done;
     jgroup = zgrp->zgroup.obj;
     jattrs = zgrp->zgroup.atts;
 
@@ -1448,7 +1449,9 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname)
 	goto done;
 
     /* Download */
-    if((stat = downloadzarrobj(file,&zvar->zarray,varpath,ZARRAY))) goto done;
+    if(stat = NCZMD_fetch_json_array(zinfo, grp, varpath, &zvar->zarray.obj) 
+    || NCZMD_fetch_json_attrs(zinfo, grp, varpath, &zvar->zarray.atts)) goto done;
+    
     jvar = zvar->zarray.obj;
     jatts = zvar->zarray.atts;
     assert(jvar == NULL || NCJsort(jvar) == NCJ_DICT);
@@ -1819,7 +1822,10 @@ ncz_read_superblock(NC_FILE_INFO_T* file, char** nczarrvp, char** zarrfp)
     if((stat = NCZ_grpkey(root,&fullpath))) goto done;
 
     /* Download the root group .zgroup and associated .zattrs */
-    if((stat = downloadzarrobj(file, &zroot->zgroup, fullpath, ZGROUP))) goto done;
+        /* Download */
+    if(stat = NCZMD_fetch_json_group(zinfo, root, fullpath, &zroot->zgroup.obj) 
+    || NCZMD_fetch_json_attrs(zinfo, root, fullpath, &zroot->zgroup.atts)) goto done;
+    
     jzgroup = zroot->zgroup.obj;    
 
     /* Look for superblock; first in .zattrs and then in .zgroup */
@@ -1957,9 +1963,9 @@ parse_group_content_pure(NCZ_FILE_INFO_T*  zinfo, NC_GRP_INFO_T* grp, NClist* va
     ZTRACE(3,"zinfo=%s grp=%s |varnames|=%u |subgrps|=%u",zinfo->common.file->controller->path,grp->hdr.name,(unsigned)nclistlength(varnames),(unsigned)nclistlength(subgrps));
 
     nclistclear(varnames);
-    if((stat = searchvars(zinfo,grp,varnames))) goto done;
+    if((stat = NCZMD_list_variables(zinfo, grp,varnames))) goto done;
     nclistclear(subgrps);
-    if((stat = searchsubgrps(zinfo,grp,subgrps))) goto done;
+    if((stat = NCZMD_list_groups(zinfo, grp,varnames))) goto done;
 
 done:
     return ZUNTRACE(THROW(stat));

From 32a98a16345d74f8353d0389c1685b66a5cb839b Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Fri, 22 Nov 2024 13:56:21 +0100
Subject: [PATCH 05/24] Ensure no s3 listing with using consolidated metadata

---
 libnczarr/zsync.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 7c244598be..28c791bbef 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -1843,12 +1843,17 @@ ncz_read_superblock(NC_FILE_INFO_T* file, char** nczarrvp, char** zarrfp)
     if(jsuper == NULL) {
 	/* See if this is looks like a NCZarr/Zarr dataset at all
            by looking for anything here of the form ".z*" */
-        if((stat = ncz_validate(file))) goto done;
+        if(!NCZMD_is_metadata_consolidated(zinfo) || (stat = ncz_validate(file))) goto done;
 	/* ok, assume pure zarr with no groups */
 	zinfo->controls.flags |= FLAG_PUREZARR;	
 	if(zarr_format == NULL) zarr_format = strdup("2");
     }
 
+    int tformat = 0;
+    if(!NCZMD_get_metadata_format(zinfo, &tformat)){
+        sprintf(zarr_format, "%d",tformat);
+    }
+
     /* Look for _nczarr_group */
     if((stat = getnczarrkey((NC_OBJ*)root,NCZ_V2_GROUP,&jnczgroup))) goto done;
 

From 802cb06f61c98e57fd9d0e20272fbdaa52ac656f Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Mon, 25 Nov 2024 16:51:57 +0100
Subject: [PATCH 06/24] Remove draft consolidated zarr3

---
 libnczarr/CMakeLists.txt |   1 -
 libnczarr/zmetadata.c    |  39 +----
 libnczarr/zmetadata3.c   | 349 ---------------------------------------
 3 files changed, 6 insertions(+), 383 deletions(-)
 delete mode 100644 libnczarr/zmetadata3.c

diff --git a/libnczarr/CMakeLists.txt b/libnczarr/CMakeLists.txt
index f49dc1e736..4e2f572002 100644
--- a/libnczarr/CMakeLists.txt
+++ b/libnczarr/CMakeLists.txt
@@ -23,7 +23,6 @@ zgrp.c
 zinternal.c
 zmap.c
 zmap_file.c
-zmetadata3.c
 zmetadata2.c
 zmetadata.c
 zodom.c
diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index 070e170a8f..0af7e2e949 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -9,8 +9,6 @@
 
 extern int NCZF2_initialize(void);
 extern int NCZF2_finalize(void);
-extern int NCZF3_initialize(void);
-extern int NCZF3_finalize(void);
 
 /**************************************************/
 ////////////////////////////////////////////////////
@@ -19,7 +17,6 @@ int NCZMD_initialize(void)
 {	
     int stat = NC_NOERR;
     if((stat=NCZMD2_initialize())) goto done;
-    if((stat=NCZMD3_initialize())) goto done;
 done:
     return THROW(stat);
 }
@@ -29,7 +26,6 @@ int NCZMD_finalize(void)
 
     int stat = NC_NOERR;
     if((stat=NCZMD2_finalize())) goto done;
-    if((stat=NCZMD3_finalize())) goto done;
 done:
     return THROW(stat);
 }
@@ -157,34 +153,17 @@ int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdha
 		goto done;
 	}
 
-	zmd_dispatcher = NCZ_metadata_handler2;
 
-	if ((stat = NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl)) != NC_NOERR || jcsl == NULL)
-	{
-		if ((stat = NCZ_downloadjson(zfile->map, Z3METADATA, &jcsl)) != NC_NOERR || jcsl == NULL)
-		{
-			/* We've tried to get any json 
-			objects for consolidated access
-			but they seem to be absent */
-			goto done;
-		}
-		NCjson * jtmp = NULL;
-		if (NCJsort(jcsl) != NCJ_DICT || NCJdictget(jcsl,"consolidated_metadata", &jtmp) || NCJsort(jtmp) != NCJ_DICT)
-		{
-			zmd_dispatcher = NCZ_metadata_handler3;
-		}
-		else
-		{
-			zmd_dispatcher = NCZ_csl_metadata_handler3;
-		}
-	}
-	else
+	if ((stat = NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl)) == NC_NOERR
+		&& jcsl != NULL && NCJsort(jcsl) == NCJ_DICT)
 	{
 		zmd_dispatcher = NCZ_csl_metadata_handler2;
+	}else{
+		zmd_dispatcher = NCZ_metadata_handler2;
+		NCJreclaim(jcsl);
+		jcsl = NULL;
 	}
 
-	if (jcsl != NULL && NCJsort(jcsl) == NCJ_DICT)
-	{
 		NCZ_Metadata *zmdh = NULL;
 		if ((zmdh = (NCZ_Metadata *)calloc(1, sizeof(NCZ_Metadata))) == NULL)
 		{
@@ -195,12 +174,6 @@ int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdha
 		zmdh->dispatcher = zmd_dispatcher;
 
 		*mdhandlerp = (const NCZ_Metadata *)zmdh;
-	}
-	else
-	{
-		NCJreclaim(jcsl); // Free jcsl if not assigned
-	}
-
 done:
 	return stat;
 }
diff --git a/libnczarr/zmetadata3.c b/libnczarr/zmetadata3.c
deleted file mode 100644
index 92741e1d6e..0000000000
--- a/libnczarr/zmetadata3.c
+++ /dev/null
@@ -1,349 +0,0 @@
-/*********************************************************************
- *   Copyright 2018, UCAR/Unidata
- *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
- *********************************************************************/
-
-#include "zmetadata.h"
-
-/**************************************************/
-
-extern int NCZF3_initialize(void);
-extern int NCZF3_finalize(void);
-
-int NCZMD_v3_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
-int NCZMD_v3_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
-
-int NCZMD_v3_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
-int NCZMD_v3_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
-
-
-int v3_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
-int v3_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
-
-/**************************************************/
-
-static const NCZ_Metadata_Dispatcher NCZ_md3_table = {
-    ZARRFORMAT3,
-    NCZ_METADATA_VERSION,  /* Version of the dispatch table */
-    ZARR_NOT_CONSOLIDATED, /* Flags*/
-
-    NCZMD_v3_list_groups,
-    NCZMD_v3_list_variables,
-
-    .fetch_json_content = v3_json_content,
-};
-
-static const NCZ_Metadata_Dispatcher NCZ_csl_md3_table = {
-    ZARRFORMAT3,
-    NCZ_METADATA_VERSION, /* Version of the dispatch table */
-    ZARR_CONSOLIDATED,	  /* Flags*/
-
-    NCZMD_v3_csl_list_groups,
-    NCZMD_v3_csl_list_variables,
-
-    .fetch_json_content = v3_csl_json_content,
-};
-
-const NCZ_Metadata_Dispatcher *NCZ_metadata_handler3 = &NCZ_md3_table;
-const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler3 = &NCZ_csl_md3_table;
-int
-NCZMD3_initialize(void)
-{
-    return NC_NOERR;
-}
-
-int
-NCZMD3_finalize(void)
-{
-    return NC_NOERR;
-}
-
-////////////////////////////////////////////////////
-
-int NCZMD_v3_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames)
-{
-	size_t i;
-	int stat = NC_NOERR;
-	char *grpkey = NULL;
-	char *subkey = NULL;
-	char *zgroup = NULL;
-	NClist *matches = nclistnew();
-
-	/* Compute the key for the grp */
-	if ((stat = NCZ_grpkey(grp, &grpkey)))
-		goto done;
-	if ((stat = nczmap_search(zfile->map, grpkey, matches)))
-		goto done;
-	for (i = 0; i < nclistlength(matches); i++)
-	{
-		const char *name = nclistget(matches, i);
-		if (name[0] == NCZM_DOT)
-			continue; /* zarr/nczarr specific */
-		/* See if name/.zgroup exists */
-		if ((stat = nczm_concat(grpkey, name, &subkey)))
-			goto done;
-		if ((stat = nczm_concat(subkey, Z3GROUP, &zgroup)))
-			goto done;
-		if ((stat = nczmap_exists(zfile->map, zgroup)) == NC_NOERR)
-			nclistpush(subgrpnames, strdup(name));
-		stat = NC_NOERR;
-		nullfree(subkey);
-		subkey = NULL;
-		nullfree(zgroup);
-		zgroup = NULL;
-	}
-done:
-	nullfree(grpkey);
-	nullfree(subkey);
-	nullfree(zgroup);
-	nclistfreeall(matches);
-	return stat;
-}
-
-int extract_node_types(const NCjson *jobj, const char * prefix, const char *node_type, NClist **nodenames){
-	size_t lprefix = strlen(prefix);
-	for (int i = 0; i < NCJlength(jobj); i += 2)	{
-		const NCjson *jname = NCJith(jobj, i);
-		const NCjson *jvalue = NCJith(jobj, i+1);
-
-		//check if json types are the expected string for key and dict for value
-		if (NCJsort(jname) != NCJ_STRING || NCJsort(jvalue) != NCJ_DICT){
-			continue;
-		}
-		// check if it value dict has key "node_type" and matches "array"
-		if (NCJdictget(jvalue, "node_type", &jvalue) || NCJsort(jvalue) != NCJ_STRING 
-		|| strncmp(node_type, NCJstring(jvalue), strlen(node_type + 1))){
-			continue;
-		}
-		// check if prefix matches item's name but ensure it's not looking at itself
-		if(strncmp(prefix, NCJstring(jname),lprefix) || lprefix == strlen(NCJstring(jname))){
-			continue;
-		}
-	
-		// if we've reach here it means it's the correct type for sure
-		// but we must check if it's directly under our current group
-
-		const char *start = NCJstring(jname) + lprefix + (lprefix > 0);
-		if (strchr(start, NCZM_SEP[0])) // Zarr separator found => not a direct group under grpkey
-			continue;
-
-		// It's a subgroup
-		const size_t lname = strlen(start);
-		nclistpush(nodenames, strndup(start, lname));
-	}
-	return NC_NOERR;
-}
-
-int NCZMD_v3_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames)
-{
-	int stat = NC_NOERR;
-	char *grpkey = NULL;
-	/* Compute the key for the grp */
-	if ((stat = NCZ_grpkey(grp, &grpkey)))
-		goto done;
-	// In the consolidated representation '/' is never observed as the key's first char
-	const char *group = grpkey + (grpkey[0] == '/'); 
-	const size_t lgroup = strlen(group);
-
-	const NCjson *jmetadata = NULL;
-	if (( stat = NCJdictget(zfile->metadata_handler->jcsl, "consolidated_metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
-		goto done;
-	}
-
-	if (( stat = NCJdictget(jmetadata, "metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
-		goto done;
-	}
-
-
-	// for each key, check if is a dict and has node_type ="group"
-	extract_node_types(jmetadata, group, "group", subgrpnames);
-	
-done:
-	nullfree(grpkey);
-	return stat;
-}
-
-
-int NCZMD_v3_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
-{
-	int stat = NC_NOERR;
-	size_t i;
-	char *grpkey = NULL;
-	char *varkey = NULL;
-	char *zarray = NULL;
-	NClist *matches = nclistnew();
-
-	/* Compute the key for the grp */
-	if ((stat = NCZ_grpkey(grp, &grpkey)))
-		goto done;
-	/* Get the map and search group */
-	if ((stat = nczmap_search(zfile->map, grpkey, matches)))
-		goto done;
-	for (i = 0; i < nclistlength(matches); i++)
-	{
-		const char *name = nclistget(matches, i);
-		if (name[0] == NCZM_DOT)
-			continue; /* zarr/nczarr specific */
-		/* See if name/.zarray exists */
-		if ((stat = nczm_concat(grpkey, name, &varkey)))
-			goto done;
-		if ((stat = nczm_concat(varkey, Z3ARRAY, &zarray)))
-			goto done;
-		switch (stat = nczmap_exists(zfile->map, zarray))
-		{
-		case NC_NOERR:
-			nclistpush(varnames, strdup(name));
-			break;
-		case NC_ENOOBJECT:
-			stat = NC_NOERR;
-			break; /* ignore */
-		default:
-			goto done;
-		}
-		nullfree(varkey);
-		varkey = NULL;
-		nullfree(zarray);
-		zarray = NULL;
-	}
-
-done:
-	nullfree(grpkey);
-	nullfree(varkey);
-	nullfree(zarray);
-	nclistfreeall(matches);
-	return stat;
-}
-
-int NCZMD_v3_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
-{
-	size_t i;
-	int stat = NC_NOERR;
-	char *grpkey = NULL;
-	/* Compute the key for the grp */
-	if ((stat = NCZ_grpkey(grp, &grpkey)))
-		goto done;
-	// In the consolidated representation '/' is never observed as the key's first char
-	const char *group = grpkey + (grpkey[0] == '/'); 
-	const size_t lgroup = strlen(group);
-
-	const NCjson *jmetadata = NULL;
-	if (( stat = NCJdictget(zfile->metadata_handler->jcsl, "consolidated_metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
-		goto done;
-	}
-
-	if (( stat = NCJdictget(jmetadata, "metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
-		goto done;
-	}
-
-	// for each key, check if is a dict and has node_type ="array"
-	extract_node_types(jmetadata, group, "array", varnames);
-
-done:
-	nullfree(grpkey);
-	return stat;
-}
-
-// Returns error if consolidated metadata is not found 
-// node_name should match item name on the consolidated representation, meaning no '/' prefix
-static int get_consolidated_json_node(const NCjson * zobj, const char* c_node_name ,const char * node_type, const NCjson ** jitem){
-	size_t i;
-	int stat = NC_NOERR;
-
-	// In the consolidated representation '/' is never observed as the key's first char
-	const char *node_name = c_node_name + (c_node_name[0] == '/'); 
-	const size_t lnode_name = strlen(node_name);
-
-	const NCjson *jmetadata = NULL;
-	if (( stat = NCJdictget(zobj, "consolidated_metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
-		goto done;
-	}
-
-	if (( stat = NCJdictget(jmetadata, "metadata", &jmetadata) ) || jmetadata == NULL || NCJsort(jmetadata) != NCJ_DICT){
-		goto done;
-	}
-
-	// for each key, check if is a dict and has matching node_type 
-	for (i = 0; i < NCJlength(jmetadata); i += 2)	{
-		const NCjson *jname = NCJith(jmetadata, i);
-		if (NCJsort(jname) != NCJ_STRING || strncmp(NCJstring(jname),node_name, lnode_name+1)){
-			continue;
-		}
-
-		const NCjson *jvalue = NCJith(jmetadata, i+1);
-		//check if json types are the expected string for key and dict for value
-		if (NCJsort(jname) != NCJ_STRING ||NCJsort(jvalue) != NCJ_DICT){
-			continue;
-		}
-
-		// check if it value dict has key "node_type" and matches node_type ["array","group"]
-		if (NCJsort(jvalue) != NCJ_DICT	|| NCJdictget(jvalue, "node_type", &jname) || jname == NULL 
-		 || NCJsort(jname) != NCJ_STRING || strncmp(node_type, NCJstring(jname),strlen(node_type))){
-			continue;
-		}
-		
-		*jitem = jvalue;
-		break; //object found!		
-	}
-done:
-	return stat;
-}
-
-// Static function only valid for v3!
-static int zarr_obj_type2suffix(NCZMD_MetadataType zarr_obj_type, const char **suffix){
-	switch (zarr_obj_type)
-	{
-		case NCZMD_GROUP:
-			*suffix = Z3GROUP;
-			break;
-		case NCZMD_ATTRS:
-			*suffix = Z3ATTRS;
-			break;
-		case NCZMD_ARRAY:
-			*suffix = Z3ARRAY;
-			break;
-		default:
-			return NC_EINVAL; // 	Invalid type
-	}
-	return NC_NOERR;
-}
-
-int v3_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
-{
-	int stat = NC_NOERR;
-	const char *suffix;
-	char * key = NULL;
-	if ( (stat = zarr_obj_type2suffix(zobj_t, &suffix))
-	   ||(stat = nczm_concat(prefix, suffix, &key))){
-		goto done;
-	}
-	
-	if (prefix == NULL || (*prefix == '/' && strlen(prefix)==1) ){
-		stat = NCJclone(zfile->metadata_handler->jcsl, jobj);
-		goto done;		
-	}
-
-
-	stat = get_consolidated_json_node(zfile->metadata_handler->jcsl, prefix, "group", jobj);
-
-done:
-	nullfree(key);
-	return stat;
-
-}
-
-int v3_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
-{
-	int stat = NC_NOERR;
-	const char *suffix;
-	char * key = NULL;
-	if ((stat = zarr_obj_type2suffix(zobj_t, &suffix))
-	    || (stat = nczm_concat(prefix, suffix, &key))){
-		goto done;
-	}
-
-	stat = NCZ_downloadjson(zfile->map, key, jobj);
-done:
-	nullfree(key);
-	return stat;
-}
-////////////////////////////////////////////////////////////////////////////
\ No newline at end of file

From 29fae5d70d6a341feb275a72e58f90aa32b09639 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Mon, 25 Nov 2024 16:52:31 +0100
Subject: [PATCH 07/24] Free zarr metadata pointer

---
 libnczarr/zclose.c    | 1 +
 libnczarr/zmetadata.c | 6 ++++++
 libnczarr/zmetadata.h | 1 +
 3 files changed, 8 insertions(+)

diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c
index 3dbba0d6be..f1bbb725d6 100644
--- a/libnczarr/zclose.c
+++ b/libnczarr/zclose.c
@@ -51,6 +51,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort)
 	goto done;
     nclistfreeall(zinfo->controllist);
     NC_authfree(zinfo->auth);
+    NCZMD_free_metadata_handler(zinfo->metadata_handler);
     nullfree(zinfo);
 
 done:
diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index 0af7e2e949..862a5a0d9a 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -177,3 +177,9 @@ int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdha
 done:
 	return stat;
 }
+
+void NCZMD_free_metadata_handler(NCZ_Metadata * zmd){
+	if (zmd == NULL) return;
+	NCJreclaim(zmd->jcsl);
+	nullfree(zmd);
+}
\ No newline at end of file
diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h
index 4b1a6bfe79..c2869e51c0 100644
--- a/libnczarr/zmetadata.h
+++ b/libnczarr/zmetadata.h
@@ -96,6 +96,7 @@ extern int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, co
 extern int NCZMD_is_metadata_consolidated(NCZ_FILE_INFO_T *zfile);
 extern int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat); // Only pure Zarr is determined
 extern int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdhandlerp);
+extern void NCZMD_free_metadata_handler(NCZ_Metadata * zmd);
 
 #if defined(__cplusplus)
 }

From a2d9fb18fb658665e0035838b7cc6924310924c1 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Tue, 10 Dec 2024 11:49:30 +0100
Subject: [PATCH 08/24] Fix memory leaks

---
 libnczarr/zmetadata.c  | 24 ++++++++++++++++--------
 libnczarr/zmetadata2.c | 16 +++++-----------
 libnczarr/zsync.c      | 31 +++++++++++++------------------
 3 files changed, 34 insertions(+), 37 deletions(-)

diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index 862a5a0d9a..137148ef08 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -50,15 +50,17 @@ int NCZMD_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *var
 int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jgroup)
 {
 	int stat = NC_NOERR;
+	char *group= NULL;
 	char *key = NULL;	
 
-	if (grp && ((stat = NCZ_grpkey(grp, &key)) != NC_NOERR)) 
+	if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) 
 		goto done;
-	if (name && (stat = nczm_concat(key, name, &key)))
+	if ((stat = nczm_concat(group, name, &key)))
 		goto done;
 
 	stat = zfile->metadata_handler->dispatcher->fetch_json_content(zfile, NCZMD_GROUP, key, jgroup);
 done:	
+	nullfree(group);
 	nullfree(key);
 	return stat;
 }
@@ -66,15 +68,17 @@ int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const cha
 int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jattrs)
 {
 	int stat = NC_NOERR;
+	char *group= NULL;
 	char *key = NULL;	
 
-	if (grp && ((stat = NCZ_grpkey(grp, &key)) != NC_NOERR)) 
+	if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) 
 		goto done;
-	if (name && (stat = nczm_concat(key, name, &key)))
+	if ((stat = nczm_concat(group, name, &key)))
 		goto done;
 
-	stat = zfile->metadata_handler->dispatcher->fetch_json_content(zfile, NCZMD_ATTRS, key, jattrs);
+	stat = zfile->metadata_handler->dispatcher->fetch_json_content(zfile, NCZMD_ATTRS, key , jattrs);
 done:	
+	nullfree(group);
 	nullfree(key);
 	return stat;
 }
@@ -82,15 +86,19 @@ int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const cha
 int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jarray)
 {
 	int stat = NC_NOERR;
+	char *group= NULL;
 	char *key = NULL;	
 
-	if (grp && ((stat = NCZ_grpkey(grp, &key)) != NC_NOERR)) 
+	if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) 
 		goto done;
-	if (name && (stat = nczm_concat(key, name, &key)))
+
+	if ((stat = nczm_concat(group, name, &key)))
 		goto done;
+	
 
 	stat = zfile->metadata_handler->dispatcher->fetch_json_content(zfile, NCZMD_ARRAY, key, jarray);
-done:	
+done:
+	nullfree(group);
 	nullfree(key);
 	return stat;
 }
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index 10a1fcc97d..bf7e3109f7 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -75,6 +75,7 @@ int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *sub
 	/* Compute the key for the grp */
 	if ((stat = NCZ_grpkey(grp, &grpkey)))
 		goto done;
+	/* Get the map and search group */
 	if ((stat = nczmap_search(zfile->map, grpkey, matches)))
 		goto done;
 	for (i = 0; i < nclistlength(matches); i++)
@@ -95,6 +96,7 @@ int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *sub
 		nullfree(zgroup);
 		zgroup = NULL;
 	}
+
 done:
 	nullfree(grpkey);
 	nullfree(subkey);
@@ -152,8 +154,8 @@ int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist
 
 int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *varnames)
 {
-	int stat = NC_NOERR;
 	size_t i;
+	int stat = NC_NOERR;
 	char *grpkey = NULL;
 	char *varkey = NULL;
 	char *zarray = NULL;
@@ -175,17 +177,9 @@ int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *
 			goto done;
 		if ((stat = nczm_concat(varkey, Z2ARRAY, &zarray)))
 			goto done;
-		switch (stat = nczmap_exists(zfile->map, zarray))
-		{
-		case NC_NOERR:
+		if ((stat = nczmap_exists(zfile->map, zarray)) == NC_NOERR)
 			nclistpush(varnames, strdup(name));
-			break;
-		case NC_ENOOBJECT:
-			stat = NC_NOERR;
-			break; /* ignore */
-		default:
-			goto done;
-		}
+		stat = NC_NOERR;
 		nullfree(varkey);
 		varkey = NULL;
 		nullfree(zarray);
diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 28c791bbef..7fea851b2b 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -1140,10 +1140,10 @@ define_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
 
     /* Construct grp path */
     if((stat = NCZ_grpkey(grp,&fullpath))) goto done;
-
-    stat = NCZMD_fetch_json_group(zinfo, grp, fullpath, &zgrp->zgroup.obj);
-    stat = NCZMD_fetch_json_attrs(zinfo, grp, fullpath, &zgrp->zgroup.atts);
+    
     /* Download .zgroup and .zattrs */
+    stat = NCZMD_fetch_json_group(zinfo, grp, NULL, &zgrp->zgroup.obj);
+    stat = NCZMD_fetch_json_attrs(zinfo, grp, NULL, &zgrp->zgroup.atts);
     jgroup = zgrp->zgroup.obj;
     jattrs = zgrp->zgroup.atts;
 
@@ -1405,7 +1405,6 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname)
     const NCjson* jncvar = NULL;
     const NCjson* jdimrefs = NULL;
     const NCjson* jvalue = NULL;
-    char* varpath = NULL;
     char* key = NULL;
     size64_t* shapes = NULL;
     NClist* dimnames = NULL;
@@ -1444,13 +1443,9 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname)
     /* Indicate we do not have quantizer yet */
     var->quantize_mode = -1;
 
-    /* Construct var path */
-    if((stat = NCZ_varkey(var,&varpath)))
-	goto done;
-
     /* Download */
-    if(stat = NCZMD_fetch_json_array(zinfo, grp, varpath, &zvar->zarray.obj) 
-    || NCZMD_fetch_json_attrs(zinfo, grp, varpath, &zvar->zarray.atts)) goto done;
+    if(stat = NCZMD_fetch_json_array(zinfo, grp, varname, &zvar->zarray.obj) 
+    || NCZMD_fetch_json_attrs(zinfo, grp, varname, &zvar->zarray.atts)) goto done;
     
     jvar = zvar->zarray.obj;
     jatts = zvar->zarray.atts;
@@ -1714,7 +1709,6 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname)
 
 done:
     nclistfreeall(dimnames); dimnames = NULL;
-    nullfree(varpath); varpath = NULL;
     nullfree(shapes); shapes = NULL;
     nullfree(key); key = NULL;
     return THROW(stat);
@@ -1818,13 +1812,10 @@ ncz_read_superblock(NC_FILE_INFO_T* file, char** nczarrvp, char** zarrfp)
     zinfo = (NCZ_FILE_INFO_T*)file->format_file_info;    
     zroot = (NCZ_GRP_INFO_T*)root->format_grp_info;    
 
-    /* Construct grp key */
-    if((stat = NCZ_grpkey(root,&fullpath))) goto done;
-
     /* Download the root group .zgroup and associated .zattrs */
         /* Download */
-    if(stat = NCZMD_fetch_json_group(zinfo, root, fullpath, &zroot->zgroup.obj) 
-    || NCZMD_fetch_json_attrs(zinfo, root, fullpath, &zroot->zgroup.atts)) goto done;
+    if(stat = NCZMD_fetch_json_group(zinfo, root, NULL, &zroot->zgroup.obj) 
+    || NCZMD_fetch_json_attrs(zinfo, root, NULL, &zroot->zgroup.atts)) goto done;
     
     jzgroup = zroot->zgroup.obj;    
 
@@ -1851,6 +1842,9 @@ ncz_read_superblock(NC_FILE_INFO_T* file, char** nczarrvp, char** zarrfp)
 
     int tformat = 0;
     if(!NCZMD_get_metadata_format(zinfo, &tformat)){
+		if (zarr_format == NULL) {
+			zarr_format = strdup("0");
+		}
         sprintf(zarr_format, "%d",tformat);
     }
 
@@ -1879,7 +1873,8 @@ ncz_read_superblock(NC_FILE_INFO_T* file, char** nczarrvp, char** zarrfp)
     if(nczarrvp) {*nczarrvp = nczarr_version; nczarr_version = NULL;}
     if(zarrfp) {*zarrfp = zarr_format; zarr_format = NULL;}
 done:
-    nullfree(fullpath);
+	NCJreclaim(zroot->zgroup.obj);
+	NCJreclaim(zroot->zgroup.atts);
     nullfree(zarr_format);
     nullfree(nczarr_version);
     return ZUNTRACE(THROW(stat));
@@ -1970,7 +1965,7 @@ parse_group_content_pure(NCZ_FILE_INFO_T*  zinfo, NC_GRP_INFO_T* grp, NClist* va
     nclistclear(varnames);
     if((stat = NCZMD_list_variables(zinfo, grp,varnames))) goto done;
     nclistclear(subgrps);
-    if((stat = NCZMD_list_groups(zinfo, grp,varnames))) goto done;
+    if((stat = NCZMD_list_groups(zinfo, grp,subgrps))) goto done;
 
 done:
     return ZUNTRACE(THROW(stat));

From 50e4fa01d62fdce2d748ebb4a4c21c02d5711731 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Tue, 10 Dec 2024 11:50:02 +0100
Subject: [PATCH 09/24] Housekeeping

---
 libnczarr/zmetadata.c  | 26 +++++++--------
 libnczarr/zmetadata.h  | 72 ++++++++++++++++++++----------------------
 libnczarr/zmetadata2.c | 39 +++++++++++------------
 3 files changed, 65 insertions(+), 72 deletions(-)

diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index 137148ef08..ec8d10baf5 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -14,20 +14,20 @@ extern int NCZF2_finalize(void);
 ////////////////////////////////////////////////////
 
 int NCZMD_initialize(void)
-{	
-    int stat = NC_NOERR;
-    if((stat=NCZMD2_initialize())) goto done;
+{
+	int stat = NC_NOERR;
+	if((stat=NCZMD2_initialize())) goto done;
 done:
-    return THROW(stat);
+	return THROW(stat);
 }
 
 int NCZMD_finalize(void)
 {
 
-    int stat = NC_NOERR;
-    if((stat=NCZMD2_finalize())) goto done;
+	int stat = NC_NOERR;
+	if((stat=NCZMD2_finalize())) goto done;
 done:
-    return THROW(stat);
+	return THROW(stat);
 }
 
 // Returns the list of subgroups from *grp
@@ -69,7 +69,7 @@ int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const cha
 {
 	int stat = NC_NOERR;
 	char *group= NULL;
-	char *key = NULL;	
+	char *key = NULL;
 
 	if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) 
 		goto done;
@@ -110,9 +110,9 @@ int NCZMD_is_metadata_consolidated(NCZ_FILE_INFO_T *zfile)
 	NCZ_Metadata *zmd = NULL;
 	zmd = zfile->metadata_handler;
 	if (zmd == NULL ||
-	    zmd->jcsl == NULL ||
-	    NCJsort(zmd->jcsl) != NCJ_DICT ||
-	    !(zmd->dispatcher->flags & ZARR_CONSOLIDATED))
+		zmd->jcsl == NULL ||
+		NCJsort(zmd->jcsl) != NCJ_DICT ||
+		!(zmd->dispatcher->flags & ZARR_CONSOLIDATED))
 	{
 		return NC_ENOOBJECT;
 	}
@@ -134,9 +134,6 @@ int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat)
 		return NC_NOERR;
 	}
 
-	// At this point, we know that version 3 cannot be
-	// otherwise the zarr.json would have been found
-	// when setting the handler.
 	// Last thing to do is to look for:
 	//      .zattrs, .zgroup or .zarray
 
@@ -149,6 +146,7 @@ int NCZMD_get_metadata_format(NCZ_FILE_INFO_T *zfile, int *zarrformat)
 	return NC_NOERR;
 }
 
+//Inference of the metadata handler
 int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdhandlerp)
 {
 	int stat = NC_NOERR;
diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h
index c2869e51c0..d18b37a410 100644
--- a/libnczarr/zmetadata.h
+++ b/libnczarr/zmetadata.h
@@ -1,6 +1,26 @@
 /* Copyright 2018-2018 University Corporation for Atmospheric
    Research/Unidata. */
 
+/*
+Zarr Metadata Handling
+
+Encapsulates Zarr metadata operations across versions, supporting both 
+consolidated access and per-file access. Provides a common interface 
+for metadata operations.
+ 
+The dispatcher is defined by the type NCZ_Metadata_Dispatcher.
+It offers 2 types of operations that allow decoupling/abstract
+filesystem access, content reading of the JSON metadata files
+1. Listings: (involves either listing or parsing consolidated view)
+ - variables within a group
+ - groups withing a group
+2. Retrieve JSON representation of (sub)groups, arrays and attributes.
+	Directly read from filesystem/objectstore or retrieve the JSON 
+	object from the consolidated view respective to the group or variable
+
+Note: This will also be the case of zarr v3
+(the elements will be extracted from zarr.json instead)
+*/
 
 #ifndef ZMETADATA_H
 #define ZMETADATA_H
@@ -8,27 +28,11 @@
 #include "ncjson.h"
 #include "zinternal.h"
 
-/*
-Notes on internal architecture.
-
-Encapsulating Zarr metadata operations across versions.
-Such allows to use the same interface for both consolidated
-access as well as fetching each and every object on the filesystem
-
-The dispatcher is defined by the type NCZ_Metadata_Dispatcher.
-That dispatcher allows the Zarr independent code to be 
-isolated from the code handling the json files.
-The table has the following groups of entries:
-1. List variables within a group
-2. List subgroups withing a group
-3. Retrieve JSON representation of (sub)groups, arrays and attributes
-    depending on the implementation it might require fetch the content
-    of the json file or process the consolidated json to retrive
-    the corrent part
-   Note: This is also the case of v3, the elements will be extracted from zarr.json
-
-*/
 
+#if defined(__cplusplus)
+extern "C"
+{
+#endif
 /* This is the version of the metadata table. It should be changed
  * when new functions are added to the metadata table. */
 #ifndef NCZ_METADATA_VERSION
@@ -38,8 +42,10 @@ The table has the following groups of entries:
 #define Z2METADATA "/.zmetadata"
 #define Z3METADATA "/zarr.json"
 
-typedef enum
-{
+#define ZARR_NOT_CONSOLIDATED 0
+#define ZARR_CONSOLIDATED 1
+
+typedef enum {
 	NCZMD_NULL,
 	NCZMD_GROUP,
 	NCZMD_ATTRS,
@@ -48,36 +54,26 @@ typedef enum
 
 typedef struct NCZ_Metadata_Dispatcher
 {
-	int zarr_format;
-	int dispatch_version; /* Version of the dispatch table */
-	size64_t flags;
-#define ZARR_NOT_CONSOLIDATED 0
-#define ZARR_CONSOLIDATED 1
+	int zarr_format;		/* Zarr format version */
+	int dispatch_version;   /* Dispatch table version*/
+	size64_t flags;			/* Metadata handling flags */
 
 	int (*list_groups)(NCZ_FILE_INFO_T *, NC_GRP_INFO_T *, NClist *subgrpnames);
 	int (*list_variables)(NCZ_FILE_INFO_T *, NC_GRP_INFO_T *, NClist *varnames);
-
 	int (*fetch_json_content)(NCZ_FILE_INFO_T *, NCZMD_MetadataType, const char *name, NCjson **jobj);
 } NCZ_Metadata_Dispatcher;
 
 typedef struct NCZ_Metadata
 {
-	NCjson *jcsl; // Internal JSON configuration
+	NCjson *jcsl; // Consolidated JSON view or NULL
 	const NCZ_Metadata_Dispatcher *dispatcher;
 } NCZ_Metadata;
 
-// "normal" handlers
+// regular handler
 extern const NCZ_Metadata_Dispatcher *NCZ_metadata_handler2;
-extern const NCZ_Metadata_Dispatcher *NCZ_metadata_handler3;
-
-// "consolidated" metadata handlers
+// consolidated metadata handler
 extern const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler2;
-extern const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler3;
 
-#if defined(__cplusplus)
-extern "C"
-{
-#endif
 
 /* Called by nc_initialize and nc_finalize respectively */
 extern int NCZMD_initialize(void);
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index bf7e3109f7..1213ce0f13 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -16,32 +16,31 @@ int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist
 int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
 int NCZMD_v2_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
 
-
 int v2_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
 int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
 
 /**************************************************/
 
 static const NCZ_Metadata_Dispatcher NCZ_md2_table = {
-    ZARRFORMAT2,
-    NCZ_METADATA_VERSION,  /* Version of the dispatch table */
-    ZARR_NOT_CONSOLIDATED, /* Flags*/
+	ZARRFORMAT2,
+	NCZ_METADATA_VERSION,  /* Version of the dispatch table */
+	ZARR_NOT_CONSOLIDATED, /* Flags*/
 
-    .list_groups = NCZMD_v2_list_groups,
-    .list_variables = NCZMD_v2_list_variables,
+	.list_groups = NCZMD_v2_list_groups,
+	.list_variables = NCZMD_v2_list_variables,
 
-    .fetch_json_content = v2_json_content,
+	.fetch_json_content = v2_json_content,
 };
 
 static const NCZ_Metadata_Dispatcher NCZ_csl_md2_table = {
-    ZARRFORMAT2,
-    NCZ_METADATA_VERSION, /* Version of the dispatch table */
-    ZARR_CONSOLIDATED,	  /* Flags*/
+	ZARRFORMAT2,
+	NCZ_METADATA_VERSION, /* Version of the dispatch table */
+	ZARR_CONSOLIDATED,	  /* Flags*/
 
-    .list_groups = NCZMD_v2_csl_list_groups,
-    .list_variables = NCZMD_v2_csl_list_variables,
+	.list_groups = NCZMD_v2_csl_list_groups,
+	.list_variables = NCZMD_v2_csl_list_variables,
 
-    .fetch_json_content = v2_csl_json_content,
+	.fetch_json_content = v2_csl_json_content,
 };
 
 const NCZ_Metadata_Dispatcher *NCZ_metadata_handler2 = &NCZ_md2_table;
@@ -52,13 +51,13 @@ const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler2 = &NCZ_csl_md2_table;
 int
 NCZMD2_initialize(void)
 {
-    return NC_NOERR;
+	return NC_NOERR;
 }
 
 int
 NCZMD2_finalize(void)
 {
-    return NC_NOERR;
+	return NC_NOERR;
 }
 
 ////////////////////////////////////////////////////
@@ -128,8 +127,8 @@ int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist
 		size_t lfullname = strlen(fullname);
 
 		if (lfullname < lgroup ||
-		    strncmp(fullname, group, lgroup) ||
-		    (lgroup > 0 && fullname[lgroup] != NCZM_SEP[0]))
+			strncmp(fullname, group, lgroup) ||
+			(lgroup > 0 && fullname[lgroup] != NCZM_SEP[0]))
 		{
 			continue;
 		}
@@ -216,8 +215,8 @@ int NCZMD_v2_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NCli
 		const char *fullname = NCJstring(jname);
 		size_t lfullname = strlen(fullname);
 		if (lfullname < lgroup ||
-		    strncmp(fullname, group, lgroup) ||
-		    (lgroup > 0 && fullname[lgroup] != NCZM_SEP[0]))
+			strncmp(fullname, group, lgroup) ||
+			(lgroup > 0 && fullname[lgroup] != NCZM_SEP[0]))
 		{
 			continue;
 		}
@@ -291,7 +290,7 @@ int v2_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const cha
 	const char *suffix;
 	char * key = NULL;
 	if ((stat = zarr_obj_type2suffix(zobj_t, &suffix))
-	    || (stat = nczm_concat(prefix, suffix, &key))){
+		|| (stat = nczm_concat(prefix, suffix, &key))){
 		goto done;
 	}
 

From 6346e9134c15a2b80463e66d4030aca0294cac1a Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Thu, 12 Dec 2024 17:30:41 +0100
Subject: [PATCH 10/24] Add consolidated [nc]zarr tests with mode=file|zip

---
 .../ref_oldformat_only_consolidated.zip       | Bin 0 -> 1103 bytes
 nczarr_test/run_newformat.sh                  |  31 ++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 nczarr_test/ref_oldformat_only_consolidated.zip

diff --git a/nczarr_test/ref_oldformat_only_consolidated.zip b/nczarr_test/ref_oldformat_only_consolidated.zip
new file mode 100644
index 0000000000000000000000000000000000000000..29550a5481b6d116448ad075445736eee9492d52
GIT binary patch
literal 1103
zcmWIWW@Zs#U}E54P-^Un^w}7@T%3V{K>>(Wfw(9&Ej~XdB`v=wH?br>KQE^;J~=<H
zI6o&dC9xzmML#F8MBgAeB_Sb!DJe}MjWH=qA&ohyOyL<r5<>ur$&X{Ee061FU=ZhH
zV9+AMWWB1~)RIJ?Es1ZZ#b@6(5IAuEvu4x$*yE?HZcR-SxW#J~VJ0OZohos5-;QOQ
zQdKM`l>B@*b*fsa&dK1(eTO3EHNEru{=NKK?Rx$CI6fxzh=liR>eu^S$=q<n$zWgc
z3YLpc9%(ODHJIqA`|a6*3b*cy7xW#DoXjba*~5KLM%94b>9h>jL-CJ7#%5A%i2_x1
zulH>=Samn^-jue8DF=F$iXGlBU-9(FvopW{YW|s=bt)s&>eVN&wvXKxe&i)Dc)7mJ
zFKk*;&K#*#ibpE~pI0XLwI(dA{=1U>o$AatzIS)7yVb`XrBf93@rBXp#lb5pUs<nQ
z6!2<M!1?<Eou{@bFBOqADq8i}&za|F&78ZtqF+CJGqd{Yj`&H5@A{V8oYOd+%6{&b
zbN5HK?T#Bl)?RtvnZNpx-_*rd&GeU5o@^+N;?EBhyfjJOR4ePtvV8wr3(gujEwWUb
zdda|pRpTD-_n>JFkDe~dJ(j=m(~8ztK|Lb8Z?c~~*!rP7!2ja6W1rvr5dER2dB6RA
z-n(6yRi`B9-jJ?j5WN()?5R`Bw3bBKt=rDs6S(^KZPE*ly#~(u6YhLomFxI^@tIGv
zbx-Ps9}9cy7k4Cqji*b(imhrvZAy)~tH`d#1FWeQp}L{XES=9;UjAFXJksH?L}d2M
zO-p3H&3yANY;7HnZLz26*{9i$tS8EQJXY|0Sa;^1agE%WN6DPJ3VrL>AGcq0di8mE
zwacsZC$BW*_&f2M<ki)e|IUv*z4@;|+U#ea>eW`UNL~0QW%hb!&(qxvlYI8MC{{`Z
zU5WlGhn5(>N&Br}2PQ%XM&c4<x}knSezCrRp26?`^&$aGK;syh?3r=r8=%=>u&oh9
z5tWP3jYQ8d5Ca()wlzi(VI(5^pqq>CcZ9jmf!ULgQxPE%;LXYg(#s5jz;HBU2JryP
C^t*ci

literal 0
HcmV?d00001

diff --git a/nczarr_test/run_newformat.sh b/nczarr_test/run_newformat.sh
index fde66c9314..195a3e227b 100755
--- a/nczarr_test/run_newformat.sh
+++ b/nczarr_test/run_newformat.sh
@@ -22,6 +22,14 @@ ${NCDUMP} -n ref_oldformat "$fileurl" > ./tmp_oldformat.cdl
 diff -w ${srcdir}/ref_oldformat.cdl ./tmp_oldformat.cdl
 }
 
+testcaseoldconsolidated() {
+echo "*** Test old format support on consolidated zarr"
+zext=$1
+fileargs ${srcdir}/ref_oldformat_only_consolidated
+${NCDUMP} -n ref_oldformat "$fileurl" > ./tmp_oldformat_consolidated.cdl
+diff -w ${srcdir}/ref_oldformat.cdl ./tmp_oldformat_consolidated.cdl
+}
+
 testcasecvt() {
 echo "*** Test old format to new format nczarr copy"
 zext=$1
@@ -40,9 +48,32 @@ ${NCDUMP} -n ref_oldformat "file://tmp_newformat.file#mode=zarr,file" > ./tmp_ne
 diff -w ${srcdir}/ref_newformatpure.cdl ./tmp_newpure.cdl
 }
 
+testcaseconsolidated() {
+echo "*** Test old format to new format consolidated nczarr copy"
+zext=$1
+fileargs ${srcdir}/ref_oldformat_only_consolidated
+${NCCOPY} "$fileurl" "file://tmp_newformat_consolidated.file#mode=nczarr,file"
+${NCDUMP} -n ref_oldformat "file://tmp_newformat_consolidated.file#mode=nczarr,file" > ./tmp_oldformat_consolidated.cdl
+diff -w ${srcdir}/ref_oldformat.cdl ./tmp_oldformat_consolidated.cdl
+}
+
+testcasepureconsolidated() {
+echo "*** Test old format to new format consolidated pure zarr copy"
+zext=$1
+fileargs ${srcdir}/ref_oldformat_only_consolidated
+# For the moment is not possible to write consolidated datasets
+#${NCCOPY} "$fileurl" "file://tmp_newformat_consolidated.file#mode=nczarr,file"
+# so unziping will act like nccopy:
+unzip -qq ${srcdir}/ref_oldformat_only_consolidated.zip -d tmp_newformat_consolidated.file
+${NCDUMP} -n ref_oldformat "file://tmp_newformat_consolidated.file#mode=zarr,file" > ./tmp_new_consolidated.cdl
+diff -w ${srcdir}/ref_newformatpure.cdl ./tmp_newpure.cdl
+}
 # Do zip tests only
 if test "x$FEATURE_NCZARR_ZIP" = xyes ; then
     testcaseold zip
+    testcaseoldconsolidated zip
     testcasecvt zip
     testcasepure zip
+    testcaseconsolidated zip
+    testcasepureconsolidated zip
 fi

From 3ac802a481eb324c003df77c85eb394429d78ca0 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Thu, 12 Dec 2024 18:46:24 +0100
Subject: [PATCH 11/24] Allow to define S3 endpoint for tests during build

---
 CMakeLists.txt         | 1 +
 config.h.cmake.in      | 3 +++
 configure.ac           | 5 +++++
 test_common.in         | 3 +++
 unit_test/run_s3sdk.sh | 2 +-
 5 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bf0b960188..7bf8d78fa8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1737,6 +1737,7 @@ endif()
 
 # The Unidata testing S3 bucket
 # WARNING: this must match the value in configure.ac
+set(S3ENDPOINT "s3.us-east-1.amazonaws.com" CACHE STRING "S3 endpoint")
 set(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket")
 
 # The working S3 path tree within the Unidata bucket.
diff --git a/config.h.cmake.in b/config.h.cmake.in
index 26a5b4773b..ae45fcdb57 100644
--- a/config.h.cmake.in
+++ b/config.h.cmake.in
@@ -181,6 +181,9 @@ are set when opening a binary file on Windows. */
 /* if true, enable S3 testing*/
 #cmakedefine WITH_S3_TESTING "${WITH_S3_TESTING}"
 
+/* S3 Test endpoint */
+#define S3ENDPOINT "${S3ENDPOINT}"
+
 /* S3 Test Bucket */
 #define S3TESTBUCKET "${S3TESTBUCKET}"
 
diff --git a/configure.ac b/configure.ac
index 89d04efebe..c7078aa40c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2114,6 +2114,11 @@ if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno; then
 AC_SUBST(WHICH_S3_SDK,[none])
 fi
 
+# The Unidata testing S3 bucket
+# WARNING: this must match the value in CMakeLists.txt
+AC_DEFINE([S3ENDPOINT], ["s3.us-east-1.amazonaws.com"], [S3 test endpoint])
+AC_SUBST([S3ENDPOINT],["s3.us-east-1.amazonaws.com"])
+
 # The Unidata testing S3 bucket
 # WARNING: this must match the value in CMakeLists.txt
 AC_DEFINE([S3TESTBUCKET], ["unidata-zarr-test-data"], [S3 test bucket])
diff --git a/test_common.in b/test_common.in
index 5a3d1c29e9..c6ff5662e1 100644
--- a/test_common.in
+++ b/test_common.in
@@ -52,6 +52,9 @@ FEATURE_PLUGIN_SEARCH_PATH="@NETCDF_PLUGIN_SEARCH_PATH@"
 # Thredds-test server is currently disabled
 #FEATURE_THREDDSTEST=1
 
+# This is the S3 endpoint to be used during tests
+S3ENDPOINT=@S3ENDPOINT@
+
 # This is the Unidata S3 test bucket
 # All S3 tests should use this to store intermediate results.
 S3TESTBUCKET=@S3TESTBUCKET@
diff --git a/unit_test/run_s3sdk.sh b/unit_test/run_s3sdk.sh
index 18073e4570..3d3f9eead0 100755
--- a/unit_test/run_s3sdk.sh
+++ b/unit_test/run_s3sdk.sh
@@ -7,7 +7,7 @@ set -e
 
 #CMD="valgrind --leak-check=full"
 
-URL="https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}"
+URL="https://${S3ENDPOINT}/${S3TESTBUCKET}"
 
 isolate "testdir_uts3sdk"
 

From 7ddd7c9a7474e67795483c326e778750ae045b5b Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Thu, 12 Dec 2024 18:47:31 +0100
Subject: [PATCH 12/24] Setup initial Zarr consolidated tests

---
 nczarr_test/CMakeLists.txt                 |  1 +
 nczarr_test/Makefile.am                    |  3 +-
 nczarr_test/ref_consolidated_zarr.cdl      | 12 ++++++
 nczarr_test/ref_consolidated_zarr_base.cdl |  7 ++++
 nczarr_test/run_consolidated_zarr.sh       | 48 ++++++++++++++++++++++
 nczarr_test/test_nczarr.sh                 |  2 +-
 6 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 nczarr_test/ref_consolidated_zarr.cdl
 create mode 100644 nczarr_test/ref_consolidated_zarr_base.cdl
 create mode 100755 nczarr_test/run_consolidated_zarr.sh

diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt
index ad6fde8c8b..e7ffbb19f4 100644
--- a/nczarr_test/CMakeLists.txt
+++ b/nczarr_test/CMakeLists.txt
@@ -181,6 +181,7 @@ IF(NETCDF_ENABLE_TESTS)
     ENDIF()
 
     add_sh_test(nczarr_test run_purezarr)
+    add_sh_test(nczarr_test run_consolidated_zarr)
     add_sh_test(nczarr_test run_misc)
     add_sh_test(nczarr_test run_nczarr_fill)
     add_sh_test(nczarr_test run_jsonconvention)
diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am
index 7054eeb42b..71aad41c38 100644
--- a/nczarr_test/Makefile.am
+++ b/nczarr_test/Makefile.am
@@ -68,6 +68,7 @@ TESTS += run_ncgen4.sh
 
 TESTS += run_quantize.sh
 TESTS += run_purezarr.sh
+TESTS += run_consolidated_zarr.sh
 TESTS += run_interop.sh
 TESTS += run_misc.sh
 TESTS += run_nczarr_fill.sh
@@ -205,7 +206,7 @@ endif
 EXTRA_DIST = CMakeLists.txt \
 run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ncgen4.sh \
 run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh \
-run_purezarr.sh run_interop.sh run_misc.sh \
+run_purezarr.sh run_consolidated_zarr.sh run_interop.sh run_misc.sh \
 run_filter.sh \
 run_newformat.sh run_nczarr_fill.sh run_quantize.sh \
 run_jsonconvention.sh run_nczfilter.sh run_unknown.sh \
diff --git a/nczarr_test/ref_consolidated_zarr.cdl b/nczarr_test/ref_consolidated_zarr.cdl
new file mode 100644
index 0000000000..f37bd3946f
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr.cdl
@@ -0,0 +1,12 @@
+netcdf tmp_consolidated_zarr {
+dimensions:
+	_Anonymous_Dim_2 = 2 ;
+	_Anonymous_Dim_5 = 5 ;
+variables:
+	int i(_Anonymous_Dim_2, _Anonymous_Dim_5) ;
+data:
+
+ i =
+  _, _, _, _, _,
+  _, _, _, _, _ ;
+}
diff --git a/nczarr_test/ref_consolidated_zarr_base.cdl b/nczarr_test/ref_consolidated_zarr_base.cdl
new file mode 100644
index 0000000000..34c0a525d8
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_base.cdl
@@ -0,0 +1,7 @@
+netcdf ref_purezarr {
+dimensions:
+    x = 2;
+    y = 5;
+variables:
+	int i(x,y) ;
+}
diff --git a/nczarr_test/run_consolidated_zarr.sh b/nczarr_test/run_consolidated_zarr.sh
new file mode 100755
index 0000000000..bea7c653d9
--- /dev/null
+++ b/nczarr_test/run_consolidated_zarr.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi 
+. ../test_common.sh
+
+. "$srcdir/test_nczarr.sh"
+
+set -e
+
+s3isolate "testdir_consolidated_zarr"
+THISDIR=`pwd`
+cd $ISOPATH
+
+# This shell script tests support for:
+# 1. consolidated zarr (noxarray) read-only
+# 2. xarray read
+
+testcase() {
+    zext=$1
+
+    echo "*** Test: consolidated zarr write then read; format=$zext"
+    fileargs tmp_consolidated_zarr "mode=zarr,noxarray,$zext"
+    deletemap $zext $file
+    ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_consolidated_zarr_base.cdl
+    echo "${NCDUMP} $fileurl > tmp_consolidated_zarr_${zext}.cdl"
+    ${NCDUMP} $fileurl > tmp_consolidated_zarr_${zext}.cdl
+    diff -b ${srcdir}/ref_consolidated_zarr.cdl tmp_consolidated_zarr_${zext}.cdl
+
+    echo "*** Test: xarray zarr write then read; format=$zext"
+    fileargs tmp_xarray "mode=zarr,$zext"
+    #deletemap $zext $file
+    ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_consolidated_zarr_base.cdl
+    ${NCDUMP} $fileurl > tmp_xarray_${zext}.cdl
+    diff -b ${srcdir}/ref_xarray.cdl tmp_xarray_${zext}.cdl
+
+    echo "*** Test: consolidated zarr reading nczarr; format=$zext"
+    fileargs tmp_nczarr "mode=nczarr,noxarray,$zext"
+    deletemap $zext $file
+    ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_whole.cdl
+    fileargs tmp_nczarr "mode=zarr,$zext"
+    ${NCDUMP} -n nczarr2zarr $fileurl > tmp_nczarr_${zext}.cdl
+    diff -b ${srcdir}/ref_nczarr2zarr.cdl tmp_nczarr_${zext}.cdl
+}
+
+testcase file
+if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
+if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi
+
diff --git a/nczarr_test/test_nczarr.sh b/nczarr_test/test_nczarr.sh
index 85e4794cd0..90faa7c89d 100755
--- a/nczarr_test/test_nczarr.sh
+++ b/nczarr_test/test_nczarr.sh
@@ -9,7 +9,7 @@ if test "x$SETX" != x; then set -x; fi
 # Figure out which cloud repo to use
 if test "x$NCZARR_S3_TEST_HOST" = x ; then
 #    export NCZARR_S3_TEST_HOST=stratus.ucar.edu
-    export NCZARR_S3_TEST_HOST=s3.us-east-1.amazonaws.com
+    export NCZARR_S3_TEST_HOST="${S3ENDPOINT:-s3.us-east-1.amazonaws.com}"
 fi
 if test "x$NCZARR_S3_TEST_BUCKET" = x ; then
     export NCZARR_S3_TEST_BUCKET="${S3TESTBUCKET}"

From c20aec10478b3cc713bc8255595433e5bc6b329b Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Tue, 17 Dec 2024 14:54:48 +0100
Subject: [PATCH 13/24] Add more tests

---
 libnczarr/Makefile.am                         |   3 +
 .../.zattrs                                   |   3 +
 .../.zgroup                                   |   3 +
 .../.zmetadata                                |  67 ++++++++++++++++++
 .../G1/.zattrs                                |   3 +
 .../G1/.zgroup                                |   3 +
 .../G1/subg1/.zgroup                          |   3 +
 .../G1/subg1/myarray/.zarray                  |  22 ++++++
 .../G1/subg1/myarray/0.0                      | Bin 0 -> 130 bytes
 .../G2/.zgroup                                |   3 +
 .../G2/other variable with spaces/.zarray     |  22 ++++++
 .../G2/other variable with spaces/0.0         | Bin 0 -> 136 bytes
 nczarr_test/run_consolidated_zarr.sh          |  21 +++++-
 13 files changed, 152 insertions(+), 1 deletion(-)
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zattrs
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zgroup
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zmetadata
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zattrs
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zgroup
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/.zgroup
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/myarray/.zarray
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/myarray/0.0
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/.zgroup
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/other variable with spaces/.zarray
 create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/other variable with spaces/0.0

diff --git a/libnczarr/Makefile.am b/libnczarr/Makefile.am
index 4227516267..f3e166de69 100644
--- a/libnczarr/Makefile.am
+++ b/libnczarr/Makefile.am
@@ -43,6 +43,8 @@ zgrp.c \
 zinternal.c \
 zmap.c \
 zmap_file.c \
+zmetadata2.c \
+zmetadata.c \
 zodom.c \
 zopen.c \
 zprov.c \
@@ -59,6 +61,7 @@ zdispatch.h \
 zincludes.h \
 zinternal.h \
 zmap.h \
+zmetadata.h \
 zodom.h \
 zprovenance.h \
 zplugins.h \
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zattrs b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zattrs
new file mode 100644
index 0000000000..89c77849b2
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zattrs
@@ -0,0 +1,3 @@
+{
+    "Description": "Consolidated zarr test"
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zgroup b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zgroup
new file mode 100644
index 0000000000..3b7daf227c
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zgroup
@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zmetadata b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zmetadata
new file mode 100644
index 0000000000..f77dab187e
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/.zmetadata
@@ -0,0 +1,67 @@
+{
+    "metadata": {
+        ".zattrs": {
+            "Description": "Consolidated zarr test"
+        },
+        ".zgroup": {
+            "zarr_format": 2
+        },
+        "G1/.zattrs": {
+            "Details": "Varibles are chunked"
+        },
+        "G1/.zgroup": {
+            "zarr_format": 2
+        },
+        "G1/subg1/.zgroup": {
+            "zarr_format": 2
+        },
+        "G1/subg1/myarray/.zarray": {
+            "chunks": [
+                6,
+                15
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<i4",
+            "fill_value": 0,
+            "filters": null,
+            "order": "C",
+            "shape": [
+                6,
+                15
+            ],
+            "zarr_format": 2
+        },
+        "G2/.zgroup": {
+            "zarr_format": 2
+        },
+        "G2/other variable with spaces/.zarray": {
+            "chunks": [
+                3,
+                5
+            ],
+            "compressor": {
+                "blocksize": 0,
+                "clevel": 5,
+                "cname": "lz4",
+                "id": "blosc",
+                "shuffle": 1
+            },
+            "dtype": "<i8",
+            "fill_value": 0,
+            "filters": null,
+            "order": "C",
+            "shape": [
+                3,
+                5
+            ],
+            "zarr_format": 2
+        }
+    },
+    "zarr_consolidated_format": 1
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zattrs b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zattrs
new file mode 100644
index 0000000000..85854ef85c
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zattrs
@@ -0,0 +1,3 @@
+{
+    "Details": "Varibles are chunked"
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zgroup b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zgroup
new file mode 100644
index 0000000000..3b7daf227c
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/.zgroup
@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/.zgroup b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/.zgroup
new file mode 100644
index 0000000000..3b7daf227c
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/.zgroup
@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/myarray/.zarray b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/myarray/.zarray
new file mode 100644
index 0000000000..669f0dc969
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/myarray/.zarray
@@ -0,0 +1,22 @@
+{
+    "chunks": [
+        6,
+        15
+    ],
+    "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "lz4",
+        "id": "blosc",
+        "shuffle": 1
+    },
+    "dtype": "<i4",
+    "fill_value": 0,
+    "filters": null,
+    "order": "C",
+    "shape": [
+        6,
+        15
+    ],
+    "zarr_format": 2
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/myarray/0.0 b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G1/subg1/myarray/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..f32ad194f2cf9892dc92758dd6ab1846ce05ab4a
GIT binary patch
literal 130
zcmZQ#G-SzOWMBZ{CI$uu5g^V2;{O2*j7-cdtZeKYoLt;IynOruf<nR~qGI9_l2Xz#
zvU2hYib~2Vs%q*Qnp)aAx_bHshDOFFre@|AmR8m_ws!Uoj!w=lu5Rugo?hNQzJC4z
XfkD9`p<&?>kwD`a7+4uT27nm=J#rL9

literal 0
HcmV?d00001

diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/.zgroup b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/.zgroup
new file mode 100644
index 0000000000..3b7daf227c
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/.zgroup
@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/other variable with spaces/.zarray b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/other variable with spaces/.zarray
new file mode 100644
index 0000000000..502c3bb39f
--- /dev/null
+++ b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/other variable with spaces/.zarray	
@@ -0,0 +1,22 @@
+{
+    "chunks": [
+        3,
+        5
+    ],
+    "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "lz4",
+        "id": "blosc",
+        "shuffle": 1
+    },
+    "dtype": "<i8",
+    "fill_value": 0,
+    "filters": null,
+    "order": "C",
+    "shape": [
+        3,
+        5
+    ],
+    "zarr_format": 2
+}
\ No newline at end of file
diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/other variable with spaces/0.0 b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr/G2/other variable with spaces/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..ae982e1aba463c458392974a267f65a8dab852ac
GIT binary patch
literal 136
zcmXxWi3tE85Cg&UKi_6-&JrwJg*C8DteLt*<h5^qhGWs8$AA$NDrPKLv0=x72kton
AAOHXW

literal 0
HcmV?d00001

diff --git a/nczarr_test/run_consolidated_zarr.sh b/nczarr_test/run_consolidated_zarr.sh
index bea7c653d9..721627220d 100755
--- a/nczarr_test/run_consolidated_zarr.sh
+++ b/nczarr_test/run_consolidated_zarr.sh
@@ -18,11 +18,11 @@ cd $ISOPATH
 testcase() {
     zext=$1
 
+    set -x
     echo "*** Test: consolidated zarr write then read; format=$zext"
     fileargs tmp_consolidated_zarr "mode=zarr,noxarray,$zext"
     deletemap $zext $file
     ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_consolidated_zarr_base.cdl
-    echo "${NCDUMP} $fileurl > tmp_consolidated_zarr_${zext}.cdl"
     ${NCDUMP} $fileurl > tmp_consolidated_zarr_${zext}.cdl
     diff -b ${srcdir}/ref_consolidated_zarr.cdl tmp_consolidated_zarr_${zext}.cdl
 
@@ -42,7 +42,26 @@ testcase() {
     diff -b ${srcdir}/ref_nczarr2zarr.cdl tmp_nczarr_${zext}.cdl
 }
 
+testcase_csl_vs_no(){
+    zext=$1
+
+    set -x
+    echo "*** Test: consolidated pure python zarr read; format=$zext"
+    deletemap $zext $file
+    cp -r $srcdir/ref_consolidated_zarr_2.18.2_python.zarr ref_consolidated_zarr_2.18.2_python.zarr.$zext
+    cp -r $srcdir/ref_consolidated_zarr_2.18.2_python.zarr ref_zarr_2.18.2_python.zarr.$zext
+    rm -f ref_zarr_2.18.2_python.zarr.$zext/.zmetadata
+    fileargs ref_consolidated_zarr_2.18.2_python.zarr "mode=zarr"
+    echo "${NCDUMP} $fileurl > tmp_consolidated_python_zarr_${zext}.cdl"
+    ${NCDUMP} -n same_name $fileurl > tmp_consolidated_python_zarr_${zext}.cdl
+    fileargs ref_zarr_2.18.2_python.zarr "mode=zarr,$zext"
+    ${NCDUMP} -n same_name $fileurl > tmp_python_zarr_${zext}.cdl
+    diff -b tmp_consolidated_python_zarr_${zext}.cdl tmp_python_zarr_${zext}.cdl 
+    set +x
+}
+
 testcase file
+testcase_csl_vs_no file
 if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi
 if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi
 

From 2b9f30541545ad016759229ba61058abb136036b Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Tue, 17 Dec 2024 10:23:48 +0100
Subject: [PATCH 14/24] Fix warnings

---
 libnczarr/zarr.c       |  2 +-
 libnczarr/zmetadata.c  |  4 ++--
 libnczarr/zmetadata2.c | 20 ++++++++++----------
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c
index 438ad71713..482c167ba3 100644
--- a/libnczarr/zarr.c
+++ b/libnczarr/zarr.c
@@ -145,7 +145,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls)
 
     /* Initialize metadata handle */
     assert(zinfo->map != NULL);
-    if((stat = NCZMD_set_metadata_handler(zinfo,&zinfo->metadata_handler))) goto done;
+    if((stat = NCZMD_set_metadata_handler(zinfo,(const NCZ_Metadata**)&zinfo->metadata_handler))) goto done;
 
     /* Ok, try to read superblock */
     if((stat = ncz_read_superblock(file,&nczarr_version,&zarr_format))) goto done;
diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index ec8d10baf5..267981f8f4 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -7,8 +7,8 @@
 
 /**************************************************/
 
-extern int NCZF2_initialize(void);
-extern int NCZF2_finalize(void);
+extern int NCZMD2_initialize(void);
+extern int NCZMD2_finalize(void);
 
 /**************************************************/
 ////////////////////////////////////////////////////
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index 1213ce0f13..e789c5576a 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -132,11 +132,11 @@ int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist
 		{
 			continue;
 		}
-		char *start = fullname + lgroup + (lgroup > 0);
-		char *end = strchr(start, NCZM_SEP[0]);
-		if (end == NULL)
+		const char *start = fullname + lgroup + (lgroup > 0);
+		const char *end = strchr(start, NCZM_SEP[0]);
+		if (end == NULL || end <= start)
 			continue;
-		size_t lname = end - start;
+		size_t lname = (size_t)(end - start);
 		// Ends with "/.zgroup
 		if (strncmp(Z2METAROOT, end, sizeof(Z2METAROOT)) == 0)
 		{
@@ -220,11 +220,11 @@ int NCZMD_v2_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NCli
 		{
 			continue;
 		}
-		char *start = fullname + lgroup + (lgroup > 0);
-		char *end = strchr(start, NCZM_SEP[0]);
-		if (end == NULL)
+		const char *start = fullname + lgroup + (lgroup > 0);
+		const char *end = strchr(start, NCZM_SEP[0]);
+		if (end == NULL || end <= start)
 			continue;
-		size_t lname = end - start;
+		size_t lname = (size_t)(end - start);
 		// Ends with ".zarray"
 		if (strncmp("/" Z2ARRAY, end, sizeof("/" Z2ARRAY)) == 0)
 		{
@@ -261,7 +261,7 @@ static int zarr_obj_type2suffix(NCZMD_MetadataType zarr_obj_type, const char **s
 int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
 {
 	int stat = NC_NOERR;
-	NCjson *jtmp = NULL;
+	const NCjson *jtmp = NULL;
 	const char *suffix;
 	char * key = NULL;
 	if ( (stat = zarr_obj_type2suffix(zobj_t, &suffix))
@@ -273,7 +273,7 @@ int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const
 	&& jtmp && NCJsort(jtmp) == NCJ_DICT)
 	{
 		NCjson *tmp = NULL;
-		if ((stat = NCJdictget(jtmp, key + (key[0] == '/'), &tmp)))
+		if ((stat = NCJdictget(jtmp, key + (key[0] == '/'), (const NCjson**)&tmp)))
 			goto done;
 		if (tmp)
 			NCJclone(tmp, jobj);

From 42ff0515378d3522ac3fff235ed142e3520418d5 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Wed, 18 Dec 2024 11:37:37 +0100
Subject: [PATCH 15/24] Adjusting DEFINEs

---
 libnczarr/zinternal.h   | 16 ----------------
 libnczarr/zsync.c       | 16 ++++++++--------
 nczarr_test/ut_map.c    | 10 +++++-----
 nczarr_test/ut_mapapi.c | 14 +++++++-------
 4 files changed, 20 insertions(+), 36 deletions(-)

diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h
index 2d78b1faf4..8acd6407f4 100644
--- a/libnczarr/zinternal.h
+++ b/libnczarr/zinternal.h
@@ -14,7 +14,6 @@
 
 #define ZARRVERSION "2"
 #define ZARRFORMAT2 2
-#define ZARRFORMAT3 3
 
 /* NCZARRVERSION is independent of Zarr version,
    but NCZARRVERSION => ZARRVERSION */
@@ -40,11 +39,6 @@
 #  endif
 #endif
 
-#define ZMETAROOT "/.zgroup"
-#define ZMETAATTR "/.zattrs"
-#define ZGROUP ".zgroup"
-#define ZATTRS ".zattrs"
-#define ZARRAY ".zarray"
 /* V2 Reserved Objects */
 #define Z2METAROOT "/.zgroup"
 #define Z2ATTSROOT "/.zattrs"
@@ -52,16 +46,6 @@
 #define Z2ATTRS ".zattrs"
 #define Z2ARRAY ".zarray"
 
-/* V3 Reserved Objects */
-#define Z3METAROOT "/zarr.json"
-#define Z3OBJECT "zarr.json"
-#define Z3GROUP Z3OBJECT
-#define Z3ATTRS Z3OBJECT
-#define Z3ARRAY Z3OBJECT
-
-/* Bytes codec name */
-#define ZBYTES3 "bytes"
-
 /* V2 Reserved Attributes */
 /*
 For nczarr version 2.x.x, the following (key,value)
diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 7fea851b2b..0a33d015fd 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -181,13 +181,13 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     if((stat = NCZ_grpkey(grp,&fullpath)))
         goto done;
 
-    /* build ZGROUP contents */
+    /* build group contents */
     NCJnew(NCJ_DICT,&jgroup);
     snprintf(version,sizeof(version),"%d",zinfo->zarr.zarr_version);
     if((stat = NCJaddstring(jgroup,NCJ_STRING,"zarr_format"))<0) {stat = NC_EINVAL; goto done;}
     if((stat = NCJaddstring(jgroup,NCJ_INT,version))<0) {stat = NC_EINVAL; goto done;}
-    /* build ZGROUP path */
-    if((stat = nczm_concat(fullpath,ZGROUP,&key)))
+    /* build group path */
+    if((stat = nczm_concat(fullpath,Z2GROUP,&key)))
 	goto done;
     /* Write to map */
     if((stat=NCZ_uploadjson(map,key,jgroup))) goto done;
@@ -486,7 +486,7 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
     }
 
     /* build .zarray path */
-    if((stat = nczm_concat(fullpath,ZARRAY,&key)))
+    if((stat = nczm_concat(fullpath,Z2ARRAY,&key)))
 	goto done;
 
     /* Write to map */
@@ -1991,7 +1991,7 @@ searchvars(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames)
 	if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */
 	/* See if name/.zarray exists */
 	if((stat = nczm_concat(grpkey,name,&varkey))) goto done;
-	if((stat = nczm_concat(varkey,ZARRAY,&zarray))) goto done;
+	if((stat = nczm_concat(varkey,Z2ARRAY,&zarray))) goto done;
 	if((stat = nczmap_exists(zfile->map,zarray)) == NC_NOERR)
 	    nclistpush(varnames,strdup(name));
 	stat = NC_NOERR;
@@ -2026,7 +2026,7 @@ searchsubgrps(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* subgrpnames)
 	if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */
 	/* See if name/.zgroup exists */
 	if((stat = nczm_concat(grpkey,name,&subkey))) goto done;
-	if((stat = nczm_concat(subkey,ZGROUP,&zgroup))) goto done;
+	if((stat = nczm_concat(subkey,Z2GROUP,&zgroup))) goto done;
 	if((stat = nczmap_exists(zfile->map,zgroup)) == NC_NOERR)
 	    nclistpush(subgrpnames,strdup(name));
 	stat = NC_NOERR;
@@ -2505,7 +2505,7 @@ upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts)
     if(stat) goto done;
 
     /* write .zattrs*/
-    if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done;
+    if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done;
     if((stat=NCZ_uploadjson(map,key,jatts))) goto done;
     nullfree(key); key = NULL;
 
@@ -2606,7 +2606,7 @@ downloadzarrobj(NC_FILE_INFO_T* file, struct ZARROBJ* zobj, const char* fullpath
     if((stat = nczm_concat(fullpath,objname,&key))) goto done;
     if((stat=NCZ_downloadjson(map,key,&zobj->obj))) goto done;
     nullfree(key); key = NULL;
-    if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done;
+    if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done;
     if((stat=NCZ_downloadjson(map,key,&zobj->atts))) goto done;
 done:
     nullfree(key);
diff --git a/nczarr_test/ut_map.c b/nczarr_test/ut_map.c
index b769ce643a..72d418a992 100644
--- a/nczarr_test/ut_map.c
+++ b/nczarr_test/ut_map.c
@@ -95,7 +95,7 @@ simplecreate(void)
     if((stat = nczmap_create(impl,url,0,0,NULL,&map)))
 	goto done;
 
-    if((stat=nczm_concat(NULL,ZMETAROOT,&path)))
+    if((stat=nczm_concat(NULL,Z2METAROOT,&path)))
 	goto done;
 
     /* Write empty metadata content */
@@ -135,7 +135,7 @@ writemeta(void)
     if((stat = nczmap_open(impl,url,NC_WRITE,0,NULL,&map)))
 	goto done;
 
-    if((stat=nczm_concat(META1,ZARRAY,&path)))
+    if((stat=nczm_concat(META1,Z2ARRAY,&path)))
 	goto done;
     if((stat = nczmap_write(map, path, strlen(metadata1), metadata1)))
 	goto done;
@@ -158,7 +158,7 @@ writemeta2(void)
     if((stat = nczmap_open(impl,url,NC_WRITE,0,NULL,&map)))
 	goto done;
 
-    if((stat=nczm_concat(META2,ZARRAY,&path)))
+    if((stat=nczm_concat(META2,Z2ARRAY,&path)))
 	goto done;
     if((stat = nczmap_write(map, path, strlen(metadata2), metadata2)))
 	goto done;
@@ -212,7 +212,7 @@ readmeta(void)
     if((stat = nczmap_open(impl,url,0,0,NULL,&map)))
 	goto done;
 
-    if((stat = readkey(map,META1,ZARRAY))) goto done;
+    if((stat = readkey(map,META1,Z2ARRAY))) goto done;
 
 done:
     (void)nczmap_close(map,0);
@@ -228,7 +228,7 @@ readmeta2(void)
     if((stat = nczmap_open(impl,url,0,0,NULL,&map)))
 	goto done;
 
-    if((stat = readkey(map,META2,ZARRAY)))
+    if((stat = readkey(map,META2,Z2ARRAY)))
         goto done;
 
 done:
diff --git a/nczarr_test/ut_mapapi.c b/nczarr_test/ut_mapapi.c
index be0787cc60..0ccc4bf883 100644
--- a/nczarr_test/ut_mapapi.c
+++ b/nczarr_test/ut_mapapi.c
@@ -99,7 +99,7 @@ simplecreate(void)
     
     printf("Pass: create: create: %s\n",url);
 
-    truekey = makekey(ZMETAROOT);
+    truekey = makekey(Z2METAROOT);
     if((stat = nczmap_write(map, truekey, 0, NULL)))
 	goto done;
     printf("Pass: create: defineobj: %s\n",truekey);
@@ -184,13 +184,13 @@ simplemeta(void)
     report(PASS,"open",map);
 	
     /* Make sure .nczarr exists (from simplecreate) */
-    truekey = makekey(ZMETAROOT);
+    truekey = makekey(Z2METAROOT);
     if((stat = nczmap_exists(map,truekey)))
 	goto done;
     report(PASS,".nczarr: exists",map);
     free(truekey); truekey = NULL;
 
-    if((stat=nczm_concat(META1,ZARRAY,&key)))
+    if((stat=nczm_concat(META1,Z2ARRAY,&key)))
 	goto done;
     truekey = makekey(key);
     nullfree(key); key = NULL;
@@ -199,13 +199,13 @@ simplemeta(void)
     report(PASS,".zarray: def",map);
     free(truekey); truekey = NULL;
 
-    truekey = makekey(ZMETAROOT);
+    truekey = makekey(Z2METAROOT);
     if((stat = nczmap_write(map, truekey, strlen(metadata1), metadata1)))
 	goto done;
     report(PASS,".nczarr: writemetadata",map);
     free(truekey); truekey = NULL;
     
-    if((stat=nczm_concat(META1,ZARRAY,&key)))
+    if((stat=nczm_concat(META1,Z2ARRAY,&key)))
 	goto done;
     truekey = makekey(key);
     free(key); key = NULL;    
@@ -225,7 +225,7 @@ simplemeta(void)
     report(PASS,"re-open",map);
 
     /* Read previously written */
-    truekey = makekey(ZMETAROOT);
+    truekey = makekey(Z2METAROOT);
     if((stat = nczmap_exists(map, truekey)))
 	goto done;
     report(PASS,".nczarr: exists",map);
@@ -245,7 +245,7 @@ simplemeta(void)
     else report(PASS,".nczarr: content verify",map);
     nullfree(content); content = NULL;
 
-    if((stat=nczm_concat(META1,ZARRAY,&key)))
+    if((stat=nczm_concat(META1,Z2ARRAY,&key)))
 	goto done;
     truekey = makekey(key);
     nullfree(key); key = NULL;

From e2eef2047e1681ef2a73a3ba33372d2dfc06b29b Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Wed, 18 Dec 2024 11:38:37 +0100
Subject: [PATCH 16/24] Removing dead code

---
 libnczarr/zsync.c | 73 -----------------------------------------------
 1 file changed, 73 deletions(-)

diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 0a33d015fd..4e066c6882 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -32,8 +32,6 @@ static int define_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* diminfo
 static int define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames);
 static int define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname);
 static int define_subgrps(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* subgrpnames);
-static int searchvars(NCZ_FILE_INFO_T*, NC_GRP_INFO_T*, NClist*);
-static int searchsubgrps(NCZ_FILE_INFO_T*, NC_GRP_INFO_T*, NClist*);
 static int locategroup(NC_FILE_INFO_T* file, size_t nsegs, NClist* segments, NC_GRP_INFO_T** grpp);
 static int createdim(NC_FILE_INFO_T* file, const char* name, size64_t dimlen, NC_DIM_INFO_T** dimp);
 static int parsedimrefs(NC_FILE_INFO_T*, NClist* dimnames,  size64_t* shape, NC_DIM_INFO_T** dims, int create);
@@ -1971,77 +1969,6 @@ parse_group_content_pure(NCZ_FILE_INFO_T*  zinfo, NC_GRP_INFO_T* grp, NClist* va
     return ZUNTRACE(THROW(stat));
 }
 
-
-static int
-searchvars(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames)
-{
-    size_t i;
-    int stat = NC_NOERR;
-    char* grpkey = NULL;
-    char* varkey = NULL;
-    char* zarray = NULL;
-    NClist* matches = nclistnew();
-
-    /* Compute the key for the grp */
-    if((stat = NCZ_grpkey(grp,&grpkey))) goto done;
-    /* Get the map and search group */
-    if((stat = nczmap_search(zfile->map,grpkey,matches))) goto done;
-    for(i=0;i<nclistlength(matches);i++) {
-	const char* name = nclistget(matches,i);
-	if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */
-	/* See if name/.zarray exists */
-	if((stat = nczm_concat(grpkey,name,&varkey))) goto done;
-	if((stat = nczm_concat(varkey,Z2ARRAY,&zarray))) goto done;
-	if((stat = nczmap_exists(zfile->map,zarray)) == NC_NOERR)
-	    nclistpush(varnames,strdup(name));
-	stat = NC_NOERR;
-	nullfree(varkey); varkey = NULL;
-	nullfree(zarray); zarray = NULL;
-    }
-
-done:
-    nullfree(grpkey);
-    nullfree(varkey);
-    nullfree(zarray);
-    nclistfreeall(matches);
-    return stat;
-}
-
-static int
-searchsubgrps(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* subgrpnames)
-{
-    size_t i;
-    int stat = NC_NOERR;
-    char* grpkey = NULL;
-    char* subkey = NULL;
-    char* zgroup = NULL;
-    NClist* matches = nclistnew();
-
-    /* Compute the key for the grp */
-    if((stat = NCZ_grpkey(grp,&grpkey))) goto done;
-    /* Get the map and search group */
-    if((stat = nczmap_search(zfile->map,grpkey,matches))) goto done;
-    for(i=0;i<nclistlength(matches);i++) {
-	const char* name = nclistget(matches,i);
-	if(name[0] == NCZM_DOT) continue; /* zarr/nczarr specific */
-	/* See if name/.zgroup exists */
-	if((stat = nczm_concat(grpkey,name,&subkey))) goto done;
-	if((stat = nczm_concat(subkey,Z2GROUP,&zgroup))) goto done;
-	if((stat = nczmap_exists(zfile->map,zgroup)) == NC_NOERR)
-	    nclistpush(subgrpnames,strdup(name));
-	stat = NC_NOERR;
-	nullfree(subkey); subkey = NULL;
-	nullfree(zgroup); zgroup = NULL;
-    }
-
-done:
-    nullfree(grpkey);
-    nullfree(subkey);
-    nullfree(zgroup);
-    nclistfreeall(matches);
-    return stat;
-}
-
 /* Convert a list of integer strings to 64 bit dimension sizes (shapes) */
 static int
 decodeints(const NCjson* jshape, size64_t* shapes)

From 19a23d8b8a385d64e5cab32b4e55fdbd7171cf41 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Fri, 13 Dec 2024 12:11:09 +0100
Subject: [PATCH 17/24] NCjson make uploads take const pointer

---
 libnczarr/zarr.h  | 2 +-
 libnczarr/zsync.c | 6 +++---
 libnczarr/zutil.c | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h
index 714fb3bcea..6704ead9a0 100644
--- a/libnczarr/zarr.h
+++ b/libnczarr/zarr.h
@@ -69,7 +69,7 @@ EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* ty
 EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
 EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
 EXTERNL int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size);
-EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json);
+EXTERNL int NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json);
 EXTERNL int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp);
 EXTERNL int NCZ_isLittleEndian(void);
 EXTERNL int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist);
diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 4e066c6882..5ab384636f 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -188,7 +188,7 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     if((stat = nczm_concat(fullpath,Z2GROUP,&key)))
 	goto done;
     /* Write to map */
-    if((stat=NCZ_uploadjson(map,key,jgroup))) goto done;
+    if((stat=NCZ_uploadjson(map,key,(const NCjson*)jgroup))) goto done;
     nullfree(key); key = NULL;
 
     if(!purezarr) {
@@ -488,7 +488,7 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
 	goto done;
 
     /* Write to map */
-    if((stat=NCZ_uploadjson(map,key,jvar)))
+    if((stat=NCZ_uploadjson(map,key,(const NCjson*)jvar)))
 	goto done;
     nullfree(key); key = NULL;
 
@@ -2433,7 +2433,7 @@ upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts)
 
     /* write .zattrs*/
     if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done;
-    if((stat=NCZ_uploadjson(map,key,jatts))) goto done;
+    if((stat=NCZ_uploadjson(map,key,(const NCjson*)jatts))) goto done;
     nullfree(key); key = NULL;
 
 done:
diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c
index 8ca4602b24..c8d920e961 100644
--- a/libnczarr/zutil.c
+++ b/libnczarr/zutil.c
@@ -272,7 +272,7 @@ NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp)
 @author Dennis Heimbigner
 */
 int
-NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json)
+NCZ_uploadjson(NCZMAP* zmap, const char* key, const NCjson* json)
 {
     int stat = NC_NOERR;
     char* content = NULL;

From 3ffe288778e835583b88029357a0c53ed29bc265 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Fri, 13 Dec 2024 14:24:18 +0100
Subject: [PATCH 18/24] Add  Zarr metadata write layer

---
 libnczarr/zarr.c       |  4 +++
 libnczarr/zmetadata.c  | 61 ++++++++++++++++++++++++++++++++++++++++++
 libnczarr/zmetadata.h  |  5 ++++
 libnczarr/zmetadata2.c | 44 +++++++++++++++++++++++++-----
 libnczarr/zsync.c      | 43 ++++++++---------------------
 5 files changed, 118 insertions(+), 39 deletions(-)

diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c
index 482c167ba3..d8b7ef241e 100644
--- a/libnczarr/zarr.c
+++ b/libnczarr/zarr.c
@@ -79,6 +79,10 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls)
     if((stat = nczmap_create(zinfo->controls.mapimpl,nc->path,nc->mode,zinfo->controls.flags,NULL,&zinfo->map)))
 	goto done;
 
+    /* Initialize metadata handle */
+    assert(zinfo->map != NULL);
+    if((stat = NCZMD_set_metadata_handler(zinfo,&zinfo->metadata_handler))) goto done;
+
 done:
     ncurifree(uri);
     NCJreclaim(json);
diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index 267981f8f4..f7974332c6 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -103,6 +103,67 @@ int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const cha
 	return stat;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+//    				 Update in-memory + storage JSON content 
+////////////////////////////////////////////////////////////////////////////////
+
+int NCZMD_update_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jgroup)
+{
+	int stat = NC_NOERR;
+	char *group= NULL;
+	char *key = NULL;	
+
+	if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) 
+		goto done;
+	if ((stat = nczm_concat(group, name, &key)))
+		goto done;
+
+	stat = zfile->metadata_handler->dispatcher->update_json_content(zfile, NCZMD_GROUP, key, jgroup);
+done:	
+	nullfree(group);
+	nullfree(key);
+	return stat;
+}
+
+int NCZMD_update_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jattrs)
+{
+	int stat = NC_NOERR;
+	char *group= NULL;
+	char *key = NULL;
+
+	if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) 
+		goto done;
+	if ((stat = nczm_concat(group, name, &key)))
+		goto done;
+
+	stat = zfile->metadata_handler->dispatcher->update_json_content(zfile, NCZMD_ATTRS, key , jattrs);
+done:	
+	nullfree(group);
+	nullfree(key);
+	return stat;
+}
+
+int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jarray)
+{
+	int stat = NC_NOERR;
+	char *group= NULL;
+	char *key = NULL;	
+
+	if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) 
+		goto done;
+
+	if ((stat = nczm_concat(group, name, &key)))
+		goto done;
+	
+
+	stat = zfile->metadata_handler->dispatcher->update_json_content(zfile, NCZMD_ARRAY, key, jarray);
+done:
+	nullfree(group);
+	nullfree(key);
+	return stat;
+}
+
+
 ////////////////////////////////////////////////////////////////////////////
 
 int NCZMD_is_metadata_consolidated(NCZ_FILE_INFO_T *zfile)
diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h
index d18b37a410..596d6a2702 100644
--- a/libnczarr/zmetadata.h
+++ b/libnczarr/zmetadata.h
@@ -61,6 +61,7 @@ typedef struct NCZ_Metadata_Dispatcher
 	int (*list_groups)(NCZ_FILE_INFO_T *, NC_GRP_INFO_T *, NClist *subgrpnames);
 	int (*list_variables)(NCZ_FILE_INFO_T *, NC_GRP_INFO_T *, NClist *varnames);
 	int (*fetch_json_content)(NCZ_FILE_INFO_T *, NCZMD_MetadataType, const char *name, NCjson **jobj);
+	int (*update_json_content)(NCZ_FILE_INFO_T *, NCZMD_MetadataType, const char *name, const NCjson *jobj);
 } NCZ_Metadata_Dispatcher;
 
 typedef struct NCZ_Metadata
@@ -86,6 +87,10 @@ extern int NCZMD_fetch_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, co
 extern int NCZMD_fetch_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jattrs);
 extern int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, NCjson **jarrays);
 
+/* Write operations */
+extern int NCZMD_update_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jgroup);
+extern int NCZMD_update_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jattrs);
+extern int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jarrays);
 /**************************************************/
 
 /* Inference for the Metadata handler */
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index e789c5576a..ac5a4c67c0 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -16,8 +16,11 @@ int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist
 int NCZMD_v2_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
 int NCZMD_v2_csl_list_variables(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
 
-int v2_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
-int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
+int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
+int fetch_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj);
+
+int update_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj);
+int update_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj);
 
 /**************************************************/
 
@@ -29,7 +32,8 @@ static const NCZ_Metadata_Dispatcher NCZ_md2_table = {
 	.list_groups = NCZMD_v2_list_groups,
 	.list_variables = NCZMD_v2_list_variables,
 
-	.fetch_json_content = v2_json_content,
+	.fetch_json_content = fetch_json_content_v2,
+	.update_json_content = update_json_content_v2,
 };
 
 static const NCZ_Metadata_Dispatcher NCZ_csl_md2_table = {
@@ -40,7 +44,8 @@ static const NCZ_Metadata_Dispatcher NCZ_csl_md2_table = {
 	.list_groups = NCZMD_v2_csl_list_groups,
 	.list_variables = NCZMD_v2_csl_list_variables,
 
-	.fetch_json_content = v2_csl_json_content,
+	.fetch_json_content = fetch_csl_json_content_v2,
+	.update_json_content = update_csl_json_content_v2,
 };
 
 const NCZ_Metadata_Dispatcher *NCZ_metadata_handler2 = &NCZ_md2_table;
@@ -258,7 +263,7 @@ static int zarr_obj_type2suffix(NCZMD_MetadataType zarr_obj_type, const char **s
 	return NC_NOERR;
 }
 
-int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
+int fetch_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
 {
 	int stat = NC_NOERR;
 	const NCjson *jtmp = NULL;
@@ -284,7 +289,7 @@ int v2_csl_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const
 
 }
 
-int v2_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
+int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, NCjson **jobj)
 {
 	int stat = NC_NOERR;
 	const char *suffix;
@@ -299,4 +304,29 @@ int v2_json_content(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const cha
 	nullfree(key);
 	return stat;
 }
-////////////////////////////////////////////////////////////////////////////
\ No newline at end of file
+
+////////////////////////////////////////////////////////////////////////////
+//		 Write to internal JSON pointer and/or directly to storage
+/////////////////////////////
+int update_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj)
+{
+	int stat = NC_NOERR;
+	return stat;
+
+}
+
+int update_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj)
+{
+	int stat = NC_NOERR;
+	const char *suffix;
+	char * key = NULL;
+	if ((stat = zarr_obj_type2suffix(zobj_t, &suffix))
+		|| (stat = nczm_concat(prefix, suffix, &key))){
+		goto done;
+	}
+
+	stat = NCZ_uploadjson(zfile->map, key, jobj);
+done:
+	nullfree(key);
+	return stat;
+}
\ No newline at end of file
diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 5ab384636f..614bb1719d 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -42,7 +42,6 @@ static int json_convention_read(const NCjson* jdict, NCjson** jtextp);
 static int ncz_validate(NC_FILE_INFO_T* file);
 static int insert_attr(NCjson* jatts, NCjson* jtypes, const char* aname, NCjson* javalue, const char* atype);
 static int insert_nczarr_attr(NCjson* jatts, NCjson* jtypes);
-static int upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts);
 static int getnczarrkey(NC_OBJ* container, const char* name, const NCjson** jncxxxp);
 static int downloadzarrobj(NC_FILE_INFO_T*, struct ZARROBJ* zobj, const char* fullpath, const char* objname);
 static int dictgetalt(const NCjson* jdict, const char* name, const char* alt, const NCjson** jvaluep);
@@ -154,9 +153,6 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     char version[1024];
     int purezarr = 0;
     NCZMAP* map = NULL;
-    char* fullpath = NULL;
-    char* key = NULL;
-    NCjson* json = NULL;
     NCjson* jgroup = NULL;
     NCjson* jdims = NULL;
     NCjson* jvars = NULL;
@@ -167,7 +163,7 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     NCjson* jatts = NULL;
     NCjson* jtypes = NULL;
 
-    LOG((3, "%s: dims: %s", __func__, key));
+    LOG((3, "%s: dims", __func__));
     ZTRACE(3,"file=%s grp=%s isclose=%d",file->controller->path,grp->hdr.name,isclose);
 
     zinfo = file->format_file_info;
@@ -179,17 +175,16 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     if((stat = NCZ_grpkey(grp,&fullpath)))
         goto done;
 
-    /* build group contents */
+    /* build ZGROUP contents */
     NCJnew(NCJ_DICT,&jgroup);
     snprintf(version,sizeof(version),"%d",zinfo->zarr.zarr_version);
     if((stat = NCJaddstring(jgroup,NCJ_STRING,"zarr_format"))<0) {stat = NC_EINVAL; goto done;}
     if((stat = NCJaddstring(jgroup,NCJ_INT,version))<0) {stat = NC_EINVAL; goto done;}
-    /* build group path */
-    if((stat = nczm_concat(fullpath,Z2GROUP,&key)))
+    /* build ZGROUP path */
+    if((stat = nczm_concat(fullpath,ZGROUP,&key)))
 	goto done;
     /* Write to map */
-    if((stat=NCZ_uploadjson(map,key,(const NCjson*)jgroup))) goto done;
-    nullfree(key); key = NULL;
+    if((stat=NCZMD_update_json_group(zinfo,grp,NULL,(const NCjson*)jgroup))) goto done;
 
     if(!purezarr) {
         if(grp->parent == NULL) { /* Root group */
@@ -253,7 +248,7 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     }
 
     /* Write out the .zattrs */
-    if((stat = upload_attrs(file,(NC_OBJ*)grp,jatts))) goto done;
+    if((stat = NCZMD_update_json_attrs(zinfo, grp, NULL, (const NCjson *)jatts))) goto done;
 
     /* Now synchronize all the variables */
     for(i=0; i<ncindexsize(grp->vars); i++) {
@@ -270,7 +265,6 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
 done:
     NCJreclaim(jtmp);
     NCJreclaim(jsuper);
-    NCJreclaim(json);
     NCJreclaim(jgroup);
     NCJreclaim(jdims);
     NCJreclaim(jvars);
@@ -278,8 +272,6 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     NCJreclaim(jnczgrp);
     NCJreclaim(jtypes);
     NCJreclaim(jatts);
-    nullfree(fullpath);
-    nullfree(key);
     return ZUNTRACE(THROW(stat));
 }
 
@@ -301,8 +293,6 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
     NCZ_FILE_INFO_T* zinfo = NULL;
     char number[1024];
     NCZMAP* map = NULL;
-    char* fullpath = NULL;
-    char* key = NULL;
     char* dimpath = NULL;
     NClist* dimrefs = NULL;
     NCjson* jvar = NULL;
@@ -341,10 +331,6 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
     if((stat = NCZ_filter_setup(var))) goto done;
 #endif
 
-    /* Construct var path */
-    if((stat = NCZ_varkey(var,&fullpath)))
-	goto done;
-
     /* Create the zarray json object */
     NCJnew(NCJ_DICT,&jvar);
 
@@ -482,15 +468,10 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
         if((stat = NCJinsert(jvar,"dimension_separator",jtmp))<0) {stat = NC_EINVAL; goto done;}
         jtmp = NULL;
     }
-
-    /* build .zarray path */
-    if((stat = nczm_concat(fullpath,Z2ARRAY,&key)))
-	goto done;
-
+  
     /* Write to map */
-    if((stat=NCZ_uploadjson(map,key,(const NCjson*)jvar)))
+    if((stat=NCZMD_update_json_array(zinfo,var->container,var->hdr.name,(const NCjson*)jvar)))
 	goto done;
-    nullfree(key); key = NULL;
 
     /* Capture dimref names as FQNs */
     if(var->ndims > 0) {
@@ -548,15 +529,13 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose)
 	jtypes = NULL;
     }
 
-    /* Write out the .zattrs */
-    if((stat = upload_attrs(file,(NC_OBJ*)var,jatts))) goto done;
+    /* Write out the <grp>/<var.name>/.zattrs */
+    if((stat = NCZMD_update_json_attrs(zinfo,var->container, var->hdr.name, jatts))) goto done;
 
     var->created = 1;
 
 done:
     nclistfreeall(dimrefs);
-    nullfree(fullpath);
-    nullfree(key);
     nullfree(dtypename);
     nullfree(dimpath);
     NCJreclaim(jvar);
@@ -2432,7 +2411,7 @@ upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts)
     if(stat) goto done;
 
     /* write .zattrs*/
-    if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done;
+    if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done;
     if((stat=NCZ_uploadjson(map,key,(const NCjson*)jatts))) goto done;
     nullfree(key); key = NULL;
 

From 4064baa7769201007cc62a83d41532734e8884f6 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Fri, 13 Dec 2024 19:32:20 +0100
Subject: [PATCH 19/24] Add zarr consolidated write

---
 libnczarr/zmetadata.c  | 13 ++++++++++---
 libnczarr/zmetadata.h  |  1 +
 libnczarr/zmetadata2.c | 44 ++++++++++++++++++++++++++++++++++++++++++
 libnczarr/zsync.c      |  2 ++
 4 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index f7974332c6..e44927f776 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -163,7 +163,15 @@ int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const ch
 	return stat;
 }
 
-
+////////////////////////////////////////////////////////////////////////////
+// Writes .zmetadata file into storage
+int NCZMD_consolidate(NCZ_FILE_INFO_T *zfile) {
+	int stat = NC_ENCZARR;
+	if (zfile->creating == 1 && zfile->metadata_handler != NULL && zfile->metadata_handler->jcsl !=NULL){
+		stat = NCZ_uploadjson(zfile->map, Z2METADATA ,zfile->metadata_handler->jcsl);
+	}
+	return stat;
+}
 ////////////////////////////////////////////////////////////////////////////
 
 int NCZMD_is_metadata_consolidated(NCZ_FILE_INFO_T *zfile)
@@ -220,8 +228,7 @@ int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdha
 		goto done;
 	}
 
-
-	if ((stat = NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl)) == NC_NOERR
+	if (zfile->creating || (stat = NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl)) == NC_NOERR
 		&& jcsl != NULL && NCJsort(jcsl) == NCJ_DICT)
 	{
 		zmd_dispatcher = NCZ_csl_metadata_handler2;
diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h
index 596d6a2702..afb894b209 100644
--- a/libnczarr/zmetadata.h
+++ b/libnczarr/zmetadata.h
@@ -91,6 +91,7 @@ extern int NCZMD_fetch_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, co
 extern int NCZMD_update_json_group(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jgroup);
 extern int NCZMD_update_json_attrs(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jattrs);
 extern int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const char *name, const NCjson *jarrays);
+extern int NCZMD_consolidate(NCZ_FILE_INFO_T *zfile);
 /**************************************************/
 
 /* Inference for the Metadata handler */
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index ac5a4c67c0..60edd68924 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -10,6 +10,8 @@
 extern int NCZF2_initialize(void);
 extern int NCZF2_finalize(void);
 
+#define  MINIMIM_CSL_REP_RAW "{\"metadata\":{},\"zarr_consolidated_format\":1}"
+
 int NCZMD_v2_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
 int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist *subgrpnames);
 
@@ -311,6 +313,48 @@ int fetch_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, con
 int update_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj)
 {
 	int stat = NC_NOERR;
+
+	// uses normal implementation to write all the .z files
+	if ((stat)=update_json_content_v2(zfile,zobj_t,prefix,jobj)){
+	}
+	// Allocating representation if doesn't exist
+	if (zfile->metadata_handler->jcsl == NULL && 
+		(stat = NCJparse(MINIMIM_CSL_REP_RAW,0,&zfile->metadata_handler->jcsl))){
+		goto done;
+	}
+	// Updating the internal JSON representation to be synced later
+	NCjson * jrep = NULL;
+	if (stat = NCJdictget(zfile->metadata_handler->jcsl,"metadata", &jrep) || jrep == NULL) {
+		goto done;
+	}
+	
+	const char *suffix;
+	char * key = NULL;
+	if ((stat = zarr_obj_type2suffix(zobj_t, &suffix))
+		|| (stat = nczm_concat(prefix, suffix, &key))){
+		goto done;
+	}
+	// Concatenate will add separator as prefix if prefix NULL
+	const char * mdkey= key[0] == '/'?key+1:key;
+	
+	NCjson * jval = NULL;
+	// does it exist?
+	if ((stat = NCJdictget(jrep,mdkey, jval))) {
+		goto done; 
+	}
+
+	if (jval != NULL){
+		//  free before updating/overwritting existing value
+		NCJreclaim(jval);
+	}
+
+	if( stat = NCJclone(jobj,&jval)){
+		goto done;
+	}
+	NCJinsert(jrep, mdkey, jval);
+done:
+	// No frees at this point
+	free(key);
 	return stat;
 
 }
diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 614bb1719d..de33c9cefa 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -262,6 +262,8 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
 	if((stat = ncz_sync_grp(file,g,isclose))) goto done;
     }
 
+	// Last step try to write consolidated file
+	stat = NCZMD_consolidate(zinfo);
 done:
     NCJreclaim(jtmp);
     NCJreclaim(jsuper);

From 57bf0b965612d8873aaf34d5ffb6d2cfc842d099 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Tue, 17 Dec 2024 09:37:13 +0100
Subject: [PATCH 20/24] Ensure no repeated keys in JSON dictionaries

---
 libdispatch/ncjson.c | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/libdispatch/ncjson.c b/libdispatch/ncjson.c
index 6a9b46cc6d..14c6ff33cb 100644
--- a/libdispatch/ncjson.c
+++ b/libdispatch/ncjson.c
@@ -898,15 +898,26 @@ NCJaddstring(NCjson* json, int sort, const char* s)
 OPTSTATIC int
 NCJinsert(NCjson* object, const char* key, NCjson* jvalue)
 {
-    int stat = NCJ_OK;
-    NCjson* jkey = NULL;
-    if(object == NULL || object->sort != NCJ_DICT || key == NULL || jvalue == NULL)
-	{stat = NCJTHROW(NCJ_ERR); goto done;}
-    if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done;
-    if((stat = NCJappend(object,jkey))==NCJ_ERR) goto done;
-    if((stat = NCJappend(object,jvalue))==NCJ_ERR) goto done;
+	int stat = NCJ_OK;
+	NCjson *jkey = NULL;
+	if (object == NULL || object->sort != NCJ_DICT || key == NULL || jvalue == NULL) {
+		stat = NCJTHROW(NCJ_ERR);
+		goto done;
+	}
+	for (size_t i = 0; i < NCJlength(object); i += 2) {
+		jkey = NCJith(object, i);
+		if (jkey->string != NULL && strcmp(jkey->string, key) == 0) {
+			// replace existing values for new key
+			NCJreclaim(object->list.contents[i + 1]); // free old value
+			object->list.contents[i + 1] = jvalue;
+			goto done;
+		}
+	}
+	if ((stat = NCJnewstring(NCJ_STRING, key, &jkey)) == NCJ_ERR) goto done;
+	if ((stat = NCJappend(object, jkey)) == NCJ_ERR) goto done;
+	if ((stat = NCJappend(object, jvalue)) == NCJ_ERR) goto done;
 done:
-    return NCJTHROW(stat);
+	return NCJTHROW(stat);
 }
 
 /* Insert key-value pair as strings into a dict object.

From f3d8dfb349778f05b888c20d6846f525f474fde7 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Tue, 17 Dec 2024 09:50:37 +0100
Subject: [PATCH 21/24] Write .zmetadata when syncing file

---
 libnczarr/zmetadata.c  |  4 ++--
 libnczarr/zmetadata2.c | 19 +++----------------
 libnczarr/zsync.c      |  2 +-
 3 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c
index e44927f776..3c0c881c73 100644
--- a/libnczarr/zmetadata.c
+++ b/libnczarr/zmetadata.c
@@ -166,7 +166,7 @@ int NCZMD_update_json_array(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, const ch
 ////////////////////////////////////////////////////////////////////////////
 // Writes .zmetadata file into storage
 int NCZMD_consolidate(NCZ_FILE_INFO_T *zfile) {
-	int stat = NC_ENCZARR;
+	int stat = NC_NOERR;
 	if (zfile->creating == 1 && zfile->metadata_handler != NULL && zfile->metadata_handler->jcsl !=NULL){
 		stat = NCZ_uploadjson(zfile->map, Z2METADATA ,zfile->metadata_handler->jcsl);
 	}
@@ -228,7 +228,7 @@ int NCZMD_set_metadata_handler(NCZ_FILE_INFO_T *zfile, const NCZ_Metadata **mdha
 		goto done;
 	}
 
-	if (zfile->creating || (stat = NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl)) == NC_NOERR
+	if ((zfile->creating || (stat = NCZ_downloadjson(zfile->map, Z2METADATA, &jcsl)) == NC_NOERR)
 		&& jcsl != NULL && NCJsort(jcsl) == NCJ_DICT)
 	{
 		zmd_dispatcher = NCZ_csl_metadata_handler2;
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index 60edd68924..d2b8f77ca5 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -132,7 +132,6 @@ int NCZMD_v2_csl_list_groups(NCZ_FILE_INFO_T *zfile, NC_GRP_INFO_T *grp, NClist
 		NCjson *jname = NCJith(jmetadata, i);
 		const char *fullname = NCJstring(jname);
 		size_t lfullname = strlen(fullname);
-
 		if (lfullname < lgroup ||
 			strncmp(fullname, group, lgroup) ||
 			(lgroup > 0 && fullname[lgroup] != NCZM_SEP[0]))
@@ -323,7 +322,7 @@ int update_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t
 		goto done;
 	}
 	// Updating the internal JSON representation to be synced later
-	NCjson * jrep = NULL;
+	const NCjson * jrep = NULL;
 	if (stat = NCJdictget(zfile->metadata_handler->jcsl,"metadata", &jrep) || jrep == NULL) {
 		goto done;
 	}
@@ -336,21 +335,9 @@ int update_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t
 	}
 	// Concatenate will add separator as prefix if prefix NULL
 	const char * mdkey= key[0] == '/'?key+1:key;
-	
 	NCjson * jval = NULL;
-	// does it exist?
-	if ((stat = NCJdictget(jrep,mdkey, jval))) {
-		goto done; 
-	}
-
-	if (jval != NULL){
-		//  free before updating/overwritting existing value
-		NCJreclaim(jval);
-	}
-
-	if( stat = NCJclone(jobj,&jval)){
-		goto done;
-	}
+	NCJclone(jobj,&jval);
+	// We overwrite existing values if key is the same
 	NCJinsert(jrep, mdkey, jval);
 done:
 	// No frees at this point
diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index de33c9cefa..f39a492f2c 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -79,6 +79,7 @@ ncz_sync_file(NC_FILE_INFO_T* file, int isclose)
     if((stat = ncz_sync_grp(file, file->root_grp, isclose)))
         goto done;
 
+	stat = NCZMD_consolidate((NCZ_FILE_INFO_T*)file->format_file_info);
 done:
     NCJreclaim(json);
     return ZUNTRACE(stat);
@@ -263,7 +264,6 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
     }
 
 	// Last step try to write consolidated file
-	stat = NCZMD_consolidate(zinfo);
 done:
     NCJreclaim(jtmp);
     NCJreclaim(jsuper);

From 207a8a65300d73fa4e309d18df5b394773bd6ae2 Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Tue, 17 Dec 2024 10:23:48 +0100
Subject: [PATCH 22/24] Fix warnings

---
 libnczarr/zarr.c       | 2 +-
 libnczarr/zmetadata2.c | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c
index d8b7ef241e..1a1467d1b4 100644
--- a/libnczarr/zarr.c
+++ b/libnczarr/zarr.c
@@ -81,7 +81,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls)
 
     /* Initialize metadata handle */
     assert(zinfo->map != NULL);
-    if((stat = NCZMD_set_metadata_handler(zinfo,&zinfo->metadata_handler))) goto done;
+    if((stat = NCZMD_set_metadata_handler(zinfo,(const NCZ_Metadata**)&zinfo->metadata_handler))) goto done;
 
 done:
     ncurifree(uri);
diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c
index d2b8f77ca5..f8eec609e8 100644
--- a/libnczarr/zmetadata2.c
+++ b/libnczarr/zmetadata2.c
@@ -314,7 +314,8 @@ int update_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t
 	int stat = NC_NOERR;
 
 	// uses normal implementation to write all the .z files
-	if ((stat)=update_json_content_v2(zfile,zobj_t,prefix,jobj)){
+	if ((stat=update_json_content_v2(zfile,zobj_t,prefix,jobj))){
+		goto done;
 	}
 	// Allocating representation if doesn't exist
 	if (zfile->metadata_handler->jcsl == NULL && 
@@ -322,8 +323,8 @@ int update_csl_json_content_v2(NCZ_FILE_INFO_T *zfile, NCZMD_MetadataType zobj_t
 		goto done;
 	}
 	// Updating the internal JSON representation to be synced later
-	const NCjson * jrep = NULL;
-	if (stat = NCJdictget(zfile->metadata_handler->jcsl,"metadata", &jrep) || jrep == NULL) {
+	NCjson * jrep = NULL;
+	if ((stat = NCJdictget(zfile->metadata_handler->jcsl,"metadata", (const NCjson**)&jrep)) || jrep == NULL) {
 		goto done;
 	}
 	

From 40e2e85065a3ae788d35cdf5978bc9d7b12ddaef Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Wed, 18 Dec 2024 12:02:57 +0100
Subject: [PATCH 23/24] Rebase issues

---
 libnczarr/zsync.c | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index f39a492f2c..702dbe3408 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -172,18 +172,10 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose)
 
     purezarr = (zinfo->controls.flags & FLAG_PUREZARR)?1:0;
 
-    /* Construct grp key */
-    if((stat = NCZ_grpkey(grp,&fullpath)))
-        goto done;
-
-    /* build ZGROUP contents */
     NCJnew(NCJ_DICT,&jgroup);
     snprintf(version,sizeof(version),"%d",zinfo->zarr.zarr_version);
     if((stat = NCJaddstring(jgroup,NCJ_STRING,"zarr_format"))<0) {stat = NC_EINVAL; goto done;}
     if((stat = NCJaddstring(jgroup,NCJ_INT,version))<0) {stat = NC_EINVAL; goto done;}
-    /* build ZGROUP path */
-    if((stat = nczm_concat(fullpath,ZGROUP,&key)))
-	goto done;
     /* Write to map */
     if((stat=NCZMD_update_json_group(zinfo,grp,NULL,(const NCjson*)jgroup))) goto done;
 

From eaa936b1aa25e1b18122a8ce992cd68e951105ba Mon Sep 17 00:00:00 2001
From: Manuel Reis <reis@dkrz.de>
Date: Wed, 18 Dec 2024 12:03:11 +0100
Subject: [PATCH 24/24] Dead code

---
 libnczarr/zsync.c | 49 -----------------------------------------------
 1 file changed, 49 deletions(-)

diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c
index 702dbe3408..46072cfae6 100644
--- a/libnczarr/zsync.c
+++ b/libnczarr/zsync.c
@@ -2365,55 +2365,6 @@ insert_nczarr_attr(NCjson* jatts, NCjson* jtypes)
     return NC_NOERR;
 }
 
-/**
-Upload a .zattrs object
-Optionally take control of jatts and jtypes
-@param file
-@param container
-@param jattsp
-@param jtypesp
-*/
-static int
-upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts)
-{
-    int stat = NC_NOERR;
-    NCZ_FILE_INFO_T* zinfo = NULL;
-    NC_VAR_INFO_T* var = NULL;
-    NC_GRP_INFO_T* grp = NULL;
-    NCZMAP* map = NULL;
-    char* fullpath = NULL;
-    char* key = NULL;
-
-    ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name);
-
-    if(jatts == NULL) goto done;    
-
-    zinfo = file->format_file_info;
-    map = zinfo->map;
-
-    if(container->sort == NCVAR) {
-        var = (NC_VAR_INFO_T*)container;
-    } else if(container->sort == NCGRP) {
-        grp = (NC_GRP_INFO_T*)container;
-    }
-
-    /* Construct container path */
-    if(container->sort == NCGRP)
-	stat = NCZ_grpkey(grp,&fullpath);
-    else
-	stat = NCZ_varkey(var,&fullpath);
-    if(stat) goto done;
-
-    /* write .zattrs*/
-    if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done;
-    if((stat=NCZ_uploadjson(map,key,(const NCjson*)jatts))) goto done;
-    nullfree(key); key = NULL;
-
-done:
-    nullfree(fullpath);
-    return ZUNTRACE(THROW(stat));
-}
-
 #if 0
 /**
 @internal Get contents of a meta object; fail it it does not exist