Skip to content
Snippets Groups Projects
Commit cf893bc7 authored by Nils-Arne Dreier's avatar Nils-Arne Dreier
Browse files

fix: make buffering optional per variable

parent 7d8a8f61
No related branches found
No related tags found
1 merge request!21improve loco configuration
Pipeline #91464 passed
......@@ -20,7 +20,9 @@ class DataHandler:
dim: chunk_size for dim, chunk_size in zip(var.attrs["_ARRAY_DIMENSIONS"], var.chunks)
}
self.time_chunk_size = chunks["time"]
self.buf = np.empty([self.time_chunk_size, *var.shape[1:-1], ma - mi], dtype=var.dtype)
self.use_buffer = var.attrs.get("hiopy::use_buffer", self.time_chunk_size > 1)
if self.use_buffer:
self.buf = np.empty([self.time_chunk_size, *var.shape[1:-1], ma - mi], dtype=var.dtype)
def flush(self):
if self.prune_offset is not None:
......@@ -38,12 +40,15 @@ class DataHandler:
self.loco_server.set_timestep(self.var.name, self.t - 1)
def __call__(self, event):
self.buf[self.t % self.time_chunk_size, ...] = np.transpose(event.data)
logging.info(
f"received {self.var.name} at {self.t % self.time_chunk_size+1}/{self.time_chunk_size}"
)
if self.use_buffer:
self.buf[self.t % self.time_chunk_size, ...] = np.transpose(event.data)
else:
self.buf = np.transpose(event.data)
self.t += 1
if self.t % self.time_chunk_size == 0: # chunk complete
if self.t % self.time_chunk_size == 0 or not self.use_buffer: # chunk complete
self.flush()
def set_loco_server(self, loco_server):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment