From 44b090e48b1beb11677a215f8f54c08e83250f36 Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Wed, 7 Feb 2024 15:43:52 +0100
Subject: [PATCH] fdb5 interface: update to release 5.11.29
---
src/cdi_fdb.c | 103 +++++++++++++++++++++++++++----------------
src/cdi_fdb.h | 20 ++++++---
src/cdi_int.h | 11 ++++-
src/grb_read.c | 7 +--
src/grb_write.c | 8 ++--
src/stream.c | 17 +++++--
src/stream_gribapi.c | 44 +++++++++---------
src/stream_record.c | 4 +-
8 files changed, 132 insertions(+), 82 deletions(-)
diff --git a/src/cdi_fdb.c b/src/cdi_fdb.c
index 924c80199..d28a4e8be 100644
--- a/src/cdi_fdb.c
+++ b/src/cdi_fdb.c
@@ -13,7 +13,21 @@ int cdi_fdb_dummy;
void ensureBufferSize(size_t requiredSize, size_t *curSize, void **buffer);
void
-decode_fdbitem(const char *fdbItem, KeyValueEntry *keyValue)
+fdb_delete_kvlist(int numItems, fdbKeyValueEntry *keyValueList)
+{
+ for (int i = 0; i < numItems; ++i)
+ {
+ for (int k = 0; k < keyValueList[i].numKeys; ++k)
+ {
+ if (keyValueList[i].keys[k]) free(keyValueList[i].keys[k]);
+ if (keyValueList[i].values[k]) free(keyValueList[i].values[k]);
+ }
+ }
+ free(keyValueList);
+}
+
+void
+decode_fdbitem(const char *fdbItem, KeyValueItem *keyValue)
{
keyValue->item = strdup(fdbItem);
char *pItem = keyValue->item;
@@ -77,78 +91,90 @@ fdb_request_add1(fdb_request_t *req, const char *param, const char *value)
}
static void
-fdbitem_to_request(const char *fdbItem, fdb_request_t *request)
+fdbitem_to_request(const fdbKeyValueEntry *keyValue, fdb_request_t *request)
{
- KeyValueEntry keyValue;
- keyValue.item = NULL;
- decode_fdbitem(fdbItem, &keyValue);
-
- for (int i = 0; i < keyValue.numKeys; i++)
+ // printf("numKeys: %d\n", keyValue->numKeys);
+ for (int i = 0; i < keyValue->numKeys; i++)
{
- // printf("key <%s> value <%s>\n", keyValue.keys[i], keyValue.values[i]);
- fdb_request_add1(request, keyValue.keys[i], keyValue.values[i]);
+ // printf("%d: key <%s> value <%s>\n", i, keyValue->keys[i], keyValue->values[i]);
+ fdb_request_add1(request, keyValue->keys[i], keyValue->values[i]);
}
-
- if (keyValue.item) free(keyValue.item);
}
int
-fdb_fill_itemlist(fdb_handle_t *fdb, fdb_request_t *request, char ***itemList)
+fdb_fill_kvlist(fdb_handle_t *fdb, fdb_request_t *request, fdbKeyValueEntry **pkeyValueList)
{
- const char **item = (const char **) malloc(sizeof(const char *));
-
fdb_listiterator_t *it;
- // fdb_new_listiterator(&it);
- // fdb_list(fdb, request, it);
fdb_list(fdb, request, &it, true);
int numItems = 0;
while (true)
{
- // bool exist;
- // fdb_listiterator_next(it, &exist, item);
int err = fdb_listiterator_next(it);
- // if (!exist) break;
if (err != FDB_SUCCESS) break;
numItems++;
}
if (CDI_Debug) Message("numItems = %d", numItems);
+ // Message("numItems = %d", numItems);
- if (*itemList == NULL) *itemList = (char **) malloc(numItems * sizeof(char *));
+ if (*pkeyValueList == NULL) *pkeyValueList = (fdbKeyValueEntry *) malloc(numItems * sizeof(fdbKeyValueEntry));
fdb_list(fdb, request, &it, true);
int itemNum = 0;
while (true)
{
- // bool exist;
- // fdb_listiterator_next(it, &exist, item);
int err = fdb_listiterator_next(it);
- // if (!exist) break;
if (err != FDB_SUCCESS) break;
- (*itemList)[itemNum++] = strdup(*item);
+ // const char *uri;
+ // size_t off, attr_len;
+ // fdb_listiterator_attrs(it, &uri, &off, &attr_len);
+ // printf("uri=%s, off=%zu, attr_len=%zu\n", uri, off, attr_len);
+
+ fdb_split_key_t *sk = NULL;
+ fdb_new_splitkey(&sk);
+ err = fdb_listiterator_splitkey(it, sk);
+ assert(err == FDB_SUCCESS);
+
+ int keyNum = 0;
+ while (true)
+ {
+ const char *k;
+ const char *v;
+ size_t l;
+ bool checkLevel = true;
+ err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : NULL);
+ if (err != FDB_SUCCESS) break;
+ // printf("k, v, l %s %s %zu\n", k, v, l);
+ if (keyNum < 32)
+ {
+ (*pkeyValueList)[itemNum].keys[keyNum] = strdup(k);
+ (*pkeyValueList)[itemNum].values[keyNum] = strdup(v);
+ keyNum++;
+ }
+ }
+
+ (*pkeyValueList)[itemNum++].numKeys = keyNum;
+
+ err = fdb_delete_splitkey(sk);
}
fdb_delete_listiterator(it);
- free(item);
-
return numItems;
}
long
-fdb_read_record(fdb_handle_t *fdb, char *item, size_t *buffersize, void **gribbuffer)
+fdb_read_record(fdb_handle_t *fdb, const fdbKeyValueEntry *keyValue, size_t *buffersize, void **gribbuffer)
{
- // Message("%s", item);
-
fdb_datareader_t *dataReader = NULL;
fdb_new_datareader(&dataReader);
fdb_request_t *singleRequest = NULL;
fdb_new_request(&singleRequest);
- fdbitem_to_request(item, singleRequest);
+ fdbitem_to_request(keyValue, singleRequest);
int status = fdb_retrieve(fdb, singleRequest, dataReader);
fdb_delete_request(singleRequest);
if (status != FDB_SUCCESS) Error("fdb_retrieve failed!");
@@ -188,10 +214,10 @@ check_numKey(const char *key, int numKeys, int numItems)
}
int
-check_keyvalueList(int numItems, KeyValueEntry *keyValueList)
+check_keyvalueList(int numItems, fdbKeyValueEntry *keyValueList)
{
const char *searchKeys[] = { "date", "time", "param", "levtype" };
- const int numSearchKeys = sizeof(searchKeys) / sizeof(searchKeys[0]);
+ int numSearchKeys = sizeof(searchKeys) / sizeof(searchKeys[0]);
int searchKeysCount[numSearchKeys];
for (int k = 0; k < numSearchKeys; k++) searchKeysCount[k] = 0;
@@ -201,7 +227,6 @@ check_keyvalueList(int numItems, KeyValueEntry *keyValueList)
char **itemKeys = keyValueList[i].keys;
for (int k = 0; k < numSearchKeys; k++)
{
-
for (int j = 0; j < numKeys; j++)
{
if (str_is_equal(itemKeys[j], searchKeys[k]))
@@ -266,8 +291,8 @@ compare_cmpkeys(const CmpKeys *cmpKeys1, const CmpKeys *cmpKeys2)
int
get_num_records(int numItems, RecordInfoEntry *recordInfoList)
{
- const int date = recordInfoList[0].date;
- const int time = recordInfoList[0].time;
+ int date = recordInfoList[0].date;
+ int time = recordInfoList[0].time;
int numRecords = 0;
for (int i = 0; i < numItems; i++)
@@ -308,14 +333,14 @@ get_ilevtype(const char *levtype)
// clang-format off
if (str_is_equal(levtype, "sfc")) ilevtype = levTypeSFC;
else if (str_is_equal(levtype, "ml")) ilevtype = levTypeML;
- else if (str_is_equal(levtype, "pl")) ilevtype = levTypeML;
+ else if (str_is_equal(levtype, "pl")) ilevtype = levTypePL;
// clang-format on
return ilevtype;
}
void
-decode_keyvalue(KeyValueEntry *keyValue, RecordInfoEntry *recordInfo)
+decode_keyvalue(fdbKeyValueEntry *keyValue, RecordInfoEntry *recordInfo)
{
char **itemKeys = keyValue->keys;
char **itemValues = keyValue->values;
@@ -346,7 +371,7 @@ remove_duplicate_timesteps(RecordInfoEntry *recordInfoList, int numRecords, int
int k = 0;
for (k = 0; k < numTimesteps; k++)
{
- const int index = (i + k) * numRecords;
+ int index = (i + k) * numRecords;
if (date != recordInfoList[index].date || time != recordInfoList[index].time) break;
}
@@ -378,7 +403,7 @@ create_fdb_request(const char *filename)
size_t len = strlen(filename);
if (len == 6) Error("Empty FDB request!");
- KeyValueEntry keyValue;
+ KeyValueItem keyValue;
keyValue.item = NULL;
decode_fdbitem(filename + 6, &keyValue);
diff --git a/src/cdi_fdb.h b/src/cdi_fdb.h
index a1a2d2210..faf5b10e6 100644
--- a/src/cdi_fdb.h
+++ b/src/cdi_fdb.h
@@ -17,7 +17,14 @@ typedef struct
char *keys[32];
char *values[32];
int numKeys;
-} KeyValueEntry;
+} KeyValueItem;
+
+typedef struct
+{
+ char *keys[32];
+ char *values[32];
+ int numKeys;
+} fdbKeyValueEntry;
typedef struct
{
@@ -28,13 +35,14 @@ typedef struct
int ilevel;
} RecordInfoEntry;
-void decode_fdbitem(const char *fdbItem, KeyValueEntry *keyValue);
-int fdb_fill_itemlist(fdb_handle_t *fdb, fdb_request_t *request, char ***itemList);
-long fdb_read_record(fdb_handle_t *fdb, char *item, size_t *buffersize, void **gribbuffer);
-int check_keyvalueList(int numItems, KeyValueEntry *keyValueList);
+void fdb_delete_kvlist(int numItems, fdbKeyValueEntry *keyValueList);
+void decode_fdbitem(const char *fdbItem, KeyValueItem *keyValue);
+int fdb_fill_kvlist(fdb_handle_t *fdb, fdb_request_t *request, fdbKeyValueEntry **keyValueList);
+long fdb_read_record(fdb_handle_t *fdb, const fdbKeyValueEntry *keyValue, size_t *buffersize, void **gribbuffer);
+int check_keyvalueList(int numItems, fdbKeyValueEntry *keyValueList);
void record_info_entry_init(RecordInfoEntry *recordInfo);
int get_num_records(int numItems, RecordInfoEntry *recordInfoList);
-void decode_keyvalue(KeyValueEntry *keyValue, RecordInfoEntry *recordInfo);
+void decode_keyvalue(fdbKeyValueEntry *keyValue, RecordInfoEntry *recordInfo);
int remove_duplicate_timesteps(RecordInfoEntry *recordInfoList, int numRecords, int numTimesteps, int *timestepRecordOffset);
fdb_request_t *create_fdb_request(const char *filename);
diff --git a/src/cdi_int.h b/src/cdi_int.h
index c7449eb7a..c44667b79 100644
--- a/src/cdi_int.h
+++ b/src/cdi_int.h
@@ -13,6 +13,10 @@
#include "config.h"
#endif
+#ifdef HAVE_LIBFDB5
+#include "cdi_fdb.h"
+#endif
+
#include <assert.h>
#include <stdio.h>
#include <stdbool.h>
@@ -185,7 +189,7 @@ typedef struct
VarScanKeys scanKeys;
var_tile_t tiles; // tile-related meta-data, currently for GRIB-API only.
#ifdef HAVE_LIBFDB5
- void *fdbItem;
+ int fdbItemIndex;
#endif
} record_t;
@@ -325,6 +329,11 @@ typedef struct
int protocol;
void *protocolData;
+
+#ifdef HAVE_LIBFDB5
+ int fdbNumItems;
+ fdbKeyValueEntry *fdbKeyValueList;
+#endif
} stream_t;
// Length of optional keyword/value pair list
diff --git a/src/grb_read.c b/src/grb_read.c
index 0b3e609ef..323570e08 100644
--- a/src/grb_read.c
+++ b/src/grb_read.c
@@ -144,10 +144,11 @@ grb_read_raw_data(stream_t *streamptr, int tsID, int recID, int memType, void *g
if (streamptr->protocol == CDI_PROTOCOL_FDB)
{
#ifdef HAVE_LIBFDB5
- void *fdbItem = streamptr->tsteps[tsID].records[recID].fdbItem;
- if (!fdbItem) Error("fdbItem not available!");
+ int fdbItemIndex = streamptr->tsteps[tsID].records[recID].fdbItemIndex;
+ if (fdbItemIndex == -1) Error("fdbItem not available!");
- recsize = fdb_read_record(streamptr->protocolData, fdbItem, &(streamptr->record->buffersize), &gribbuffer);
+ recsize = fdb_read_record(streamptr->protocolData, &(streamptr->fdbKeyValueList[fdbItemIndex]),
+ &(streamptr->record->buffersize), &gribbuffer);
#endif
}
else
diff --git a/src/grb_write.c b/src/grb_write.c
index 4158f0a9c..ceddd4b13 100644
--- a/src/grb_write.c
+++ b/src/grb_write.c
@@ -124,7 +124,7 @@ cdi_fdb_store(void *fdbHandle, const char *filename, void *gribbuffer, size_t nb
size_t len = strlen(filename);
if (len == 6) Error("FDB keys missing!");
- KeyValueEntry keyValue;
+ KeyValueItem keyValue;
keyValue.item = NULL;
decode_fdbitem(filename + 6, &keyValue);
@@ -233,11 +233,11 @@ grbCopyRecord(stream_t *streamptr2, stream_t *streamptr1)
if (streamptr1->protocol == CDI_PROTOCOL_FDB)
{
#ifdef HAVE_LIBFDB5
- void *fdbItem = streamptr1->tsteps[tsID].records[recID].fdbItem;
- if (!fdbItem) Error("fdbItem not available!");
+ int fdbItemIndex = streamptr1->tsteps[tsID].records[recID].fdbItemIndex;
+ if (fdbItemIndex == -1) Error("fdbItem not available!");
size_t buffersize = 0;
- recsize = fdb_read_record(streamptr1->protocolData, fdbItem, &buffersize, &gribbuffer);
+ recsize = fdb_read_record(streamptr1->protocolData, &(streamptr1->fdbKeyValueList[fdbItemIndex]), &buffersize, &gribbuffer);
// round up recsize to next multiple of 8
size_t gribbuffersize = ((recsize + 7U) & ~7U);
diff --git a/src/stream.c b/src/stream.c
index 77f273284..6a869bffe 100644
--- a/src/stream.c
+++ b/src/stream.c
@@ -1067,6 +1067,11 @@ streamDefaultValue(stream_t *streamptr)
streamptr->jobManager = NULL;
streamptr->protocolData = NULL;
+
+#ifdef HAVE_LIBFDB5
+ streamptr->fdbNumItems = 0;
+ streamptr->fdbKeyValueList = NULL;
+#endif
}
static stream_t *
@@ -1251,9 +1256,6 @@ streamDestroy(stream_t *streamptr)
for (int index = 0; index < maxSteps; ++index)
{
tsteps_t *tstep = &(streamptr->tsteps[index]);
-#ifdef HAVE_LIBFDB5
- if (tstep->records && tstep->records->fdbItem) free(tstep->records->fdbItem);
-#endif
if (tstep->records) Free(tstep->records);
if (tstep->recIDs) Free(tstep->recIDs);
taxisDestroyKernel(&(tstep->taxis));
@@ -1263,6 +1265,15 @@ streamDestroy(stream_t *streamptr)
streamptr->tsteps = NULL;
}
+#ifdef HAVE_LIBFDB5
+ if (streamptr->fdbKeyValueList)
+ {
+ fdb_delete_kvlist(streamptr->fdbNumItems, streamptr->fdbKeyValueList);
+ streamptr->fdbNumItems = 0;
+ streamptr->fdbKeyValueList = NULL;
+ }
+#endif
+
if (vlistID != -1)
{
int taxisID = (streamptr->filemode != 'w') ? vlistInqTaxis(vlistID) : -1;
diff --git a/src/stream_gribapi.c b/src/stream_gribapi.c
index 16a7c74f8..468f65f3a 100644
--- a/src/stream_gribapi.c
+++ b/src/stream_gribapi.c
@@ -699,7 +699,7 @@ gribapiDefProjHEALPIX(grib_handle *gh, int gridID)
static void
gribapiAddRecord(stream_t *streamptr, int param, grib_handle *gh, size_t recsize, off_t position, int datatype, int comptype,
const char *varname, int leveltype1, int leveltype2, int lbounds, int level1, int level2, int level_sf,
- int level_unit, VarScanKeys *scanKeys, const var_tile_t *tiles, bool lread_additional_keys, void *fdbItem)
+ int level_unit, VarScanKeys *scanKeys, const var_tile_t *tiles, bool lread_additional_keys, int fdbItemIndex)
{
int vlistID = streamptr->vlistID;
int tsID = streamptr->curTsID;
@@ -721,9 +721,9 @@ gribapiAddRecord(stream_t *streamptr, int param, grib_handle *gh, size_t recsize
record->scanKeys = *scanKeys;
record->tiles = tiles ? *tiles : dummy_tiles;
#ifdef HAVE_LIBFDB5
- record->fdbItem = fdbItem;
+ record->fdbItemIndex = fdbItemIndex;
#else
- (void) fdbItem;
+ (void) fdbItemIndex;
#endif
strncpy(record->varname, varname, sizeof(record->varname) - 1);
@@ -1032,38 +1032,37 @@ fdbScanTimesteps(stream_t *streamptr)
fdb_handle_t *fdbHandle = streamptr->protocolData;
- char **fdbItemList = NULL;
fdb_request_t *request = create_fdb_request(streamptr->filename);
- int numItems = fdb_fill_itemlist(fdbHandle, request, &fdbItemList);
+
+ fdbKeyValueEntry *keyValueList = NULL;
+ int numItems = fdb_fill_kvlist(fdbHandle, request, &keyValueList);
fdb_delete_request(request);
- if (numItems == 0) Error("FDB request does not find any database entries!");
+ if (numItems == 0) Error("FDB request doesn't find any database entries!");
if (CDI_Debug)
- for (int i = 0; i < numItems; ++i) printf("item[%d] = %s\n", i + 1, fdbItemList[i]);
-
- KeyValueEntry *keyValueList = (KeyValueEntry *) malloc(numItems * sizeof(KeyValueEntry));
- for (int i = 0; i < numItems; ++i) keyValueList[i].item = NULL;
+ for (int i = 0; i < numItems; ++i)
+ {
+ for (int k = 0; k < keyValueList[i].numKeys; ++k)
+ printf("item=%d %s=%s\n", i + 1, keyValueList[i].keys[k], keyValueList[i].values[k]);
+ }
RecordInfoEntry *recordInfoList = (RecordInfoEntry *) malloc(numItems * sizeof(RecordInfoEntry));
for (int i = 0; i < numItems; ++i) record_info_entry_init(&recordInfoList[i]);
- for (int i = 0; i < numItems; ++i) decode_fdbitem(fdbItemList[i], &keyValueList[i]);
if (check_keyvalueList(numItems, keyValueList) != 0) return 1;
+
for (int i = 0; i < numItems; ++i) decode_keyvalue(&keyValueList[i], &recordInfoList[i]);
int numRecords = get_num_records(numItems, recordInfoList);
int numTimesteps = numItems / numRecords;
if (CDI_Debug) Message("numRecords=%d numTimesteps=%d", numRecords, numTimesteps);
- for (int i = 0; i < numItems; ++i)
- if (keyValueList[i].item) free(keyValueList[i].item);
- free(keyValueList);
-
if (numRecords == 0) return CDI_EUFSTRUCT;
int *timestepRecordOffset = (int *) malloc(numTimesteps * sizeof(int));
for (int i = 0; i < numTimesteps; i++) timestepRecordOffset[i] = i * numRecords;
numTimesteps = remove_duplicate_timesteps(recordInfoList, numRecords, numTimesteps, timestepRecordOffset);
if (CDI_Debug) Message("numRecords=%d numTimesteps=%d", numRecords, numTimesteps);
+ // Message("numRecords=%d numTimesteps=%d", numRecords, numTimesteps);
CdiDateTime vDateTime0;
cdiDateTime_init(&vDateTime0);
@@ -1078,7 +1077,7 @@ fdbScanTimesteps(stream_t *streamptr)
for (int recID = 0; recID < numRecords; recID++)
{
- long recsize = fdb_read_record(fdbHandle, fdbItemList[recID], &buffersize, &gribbuffer);
+ long recsize = fdb_read_record(fdbHandle, &keyValueList[recID], &buffersize, &gribbuffer);
int datatype, comptype = 0;
gh = gribapiGetDiskRepresentation(recsize, &buffersize, &gribbuffer, &datatype, &comptype);
@@ -1122,8 +1121,7 @@ fdbScanTimesteps(stream_t *streamptr)
var_tile_t *ptiles = memcmp(&tiles, &dummy_tiles, sizeof(var_tile_t)) ? &tiles : NULL;
int recpos = 0;
gribapiAddRecord(streamptr, param, gh, recsize, recpos, datatype, comptype, varname, leveltype1, leveltype2, lbounds, level1,
- level2, level_sf, level_unit, &scanKeys, ptiles, true, fdbItemList[recID]);
- fdbItemList[recID] = NULL;
+ level2, level_sf, level_unit, &scanKeys, ptiles, true, recID);
grib_handle_delete(gh);
gh = NULL;
@@ -1178,8 +1176,7 @@ fdbScanTimesteps(stream_t *streamptr)
records[recID].position = 0;
records[recID].size = 0;
- records[recID].fdbItem = fdbItemList[recordOffset + recID];
- fdbItemList[recordOffset + recID] = NULL;
+ records[recID].fdbItemIndex = recordOffset + recID;
}
if (tsID == 1) streamptr->tsteps[1].nrecs = numRecords;
@@ -1188,10 +1185,9 @@ fdbScanTimesteps(stream_t *streamptr)
streamptr->rtsteps = numTimesteps;
streamptr->ntsteps = numTimesteps;
- for (int i = 0; i < numItems; ++i)
- if (fdbItemList[i]) free(fdbItemList[i]);
+ streamptr->fdbNumItems = numItems;
+ streamptr->fdbKeyValueList = keyValueList;
- if (fdbItemList) free(fdbItemList);
if (recordInfoList) free(recordInfoList);
if (timestepRecordOffset) free(timestepRecordOffset);
#endif
@@ -1311,7 +1307,7 @@ gribapiScanTimestep1(stream_t *streamptr)
var_tile_t *ptiles = memcmp(&tiles, &dummy_tiles, sizeof(var_tile_t)) ? &tiles : NULL;
gribapiAddRecord(streamptr, param, gh, recsize, recpos, datatype, comptype, varname, leveltype1, leveltype2, lbounds, level1,
- level2, level_sf, level_unit, &scanKeys, ptiles, true, NULL);
+ level2, level_sf, level_unit, &scanKeys, ptiles, true, -1);
grib_handle_delete(gh);
gh = NULL;
diff --git a/src/stream_record.c b/src/stream_record.c
index 59f10059e..807f9eecd 100644
--- a/src/stream_record.c
+++ b/src/stream_record.c
@@ -32,7 +32,7 @@ recordInitEntry(record_t *record)
varScanKeysInit(&record->scanKeys);
memset(&record->tiles, 0, sizeof(record->tiles));
#ifdef HAVE_LIBFDB5
- record->fdbItem = NULL;
+ record->fdbItemIndex = -1;
#endif
}
@@ -291,7 +291,7 @@ cdi_create_records(stream_t *streamptr, int tsID)
destTstep->recordSize = (int) maxrecords;
destTstep->nallrecs = (int) nrecords;
#ifdef HAVE_LIBFDB5
- destTstep->records->fdbItem = NULL;
+ destTstep->records->fdbItemIndex = -1;
#endif
if (tsID == 0)
--
GitLab