Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
mpim-sw
libcdi
Commits
8da9e1bb
Commit
8da9e1bb
authored
Dec 07, 2015
by
Thomas Jahns
🤸
Browse files
Let netCDF reads derive from base grid class to insert data lazily later.
parent
f014032d
Changes
3
Hide whitespace changes
Inline
Side-by-side
src/cdi.inc
View file @
8da9e1bb
...
...
@@ -4,7 +4,7 @@
!
!
Author
:
!
-------
!
Uwe
Schulzweida
,
MPI
-
MET
,
Hamburg
,
Nov
ember
2015
!
Uwe
Schulzweida
,
MPI
-
MET
,
Hamburg
,
Dec
ember
2015
!
INTEGER
CDI_MAX_NAME
...
...
src/cdipio.inc
View file @
8da9e1bb
...
...
@@ -4,7 +4,7 @@
!
!
Author
:
!
-------
!
Uwe
Schulzweida
,
MPI
-
MET
,
Hamburg
,
Nov
ember
2015
!
Uwe
Schulzweida
,
MPI
-
MET
,
Hamburg
,
Dec
ember
2015
!
!
...
...
src/stream_cdf.c
View file @
8da9e1bb
...
...
@@ -12,6 +12,17 @@
#include
<ctype.h>
#include
<math.h>
#include
<float.h>
#include
<stdbool.h>
#ifdef HAVE_MMAP
#include
<unistd.h>
#include
<sys/mman.h>
#include
<sys/types.h>
#include
<sys/stat.h>
#include
<fcntl.h>
#endif
#ifdef HAVE_LIBPTHREAD
#include
<pthread.h>
#endif
#include
<netcdf.h>
...
...
@@ -4442,6 +4453,99 @@ void grid_set_chunktype(grid_t *grid, ncvar_t *ncvar)
}
}
static
struct
gridVirtTable
cdfLazyGridVtable
;
static
double
*
cdfPendingLoad
;
#ifdef HAVE_LIBPTHREAD
static
pthread_once_t
cdfLazyInitialized
=
PTHREAD_ONCE_INIT
;
#else
static
bool
cdfLazyInitialized
;
#endif
struct
cdfLazyGrid
{
grid_t
base
;
const
struct
gridVirtTable
*
baseVtable
;
};
static
void
cdfLazyGridDestroy
(
struct
cdfLazyGrid
*
lazyGrid
)
{
lazyGrid
->
base
.
extraData
=
NULL
;
}
static
void
cdfLazyGridDelete
(
grid_t
*
grid
)
{
struct
cdfLazyGrid
*
cdfGrid
=
(
struct
cdfLazyGrid
*
)
grid
;
void
(
*
baseDestroy
)(
grid_t
*
grid
)
=
cdfGrid
->
baseVtable
->
destroy
;
cdfLazyGridDestroy
(
cdfGrid
);
baseDestroy
(
grid
);
}
static
void
cdfLazyGridDestroyOnce
(
void
)
{
#ifdef HAVE_MMAP
size_t
pgSize
=
cdiGetPageSize
(
false
);
munmap
(
cdfPendingLoad
,
pgSize
);
#endif
}
static
void
cdfLazyGridInitOnce
(
void
)
{
cdfLazyGridVtable
=
cdiGridVtable
;
cdfLazyGridVtable
.
destroy
=
cdfLazyGridDelete
;
/* create inaccessible memory area, if possible, this serves as
* dummy value for pointers to data not yet loaded */
#ifdef HAVE_MMAP
{
size_t
pgSize
=
cdiGetPageSize
(
false
);
static
const
char
devZero
[]
=
"/dev/zero"
;
int
fd
=
open
(
devZero
,
O_RDWR
);
if
(
fd
==
-
1
)
SysError
(
"Could not open %s to map anonymous memory"
,
devZero
);
void
*
cdfInvalid
=
mmap
(
NULL
,
pgSize
,
PROT_NONE
,
MAP_PRIVATE
,
fd
,
0
);
if
(
cdfInvalid
==
MAP_FAILED
)
SysError
(
"Could not mmap anonymous memory"
);
cdfPendingLoad
=
cdfInvalid
;
int
rc
=
close
(
fd
);
if
(
rc
==
-
1
)
SysError
(
"Could not close %s file handle %d after mapping anonymous"
" memory"
,
devZero
,
fd
);
}
#else
cdfPendingLoad
=
(
double
*
)
&
cdfPendingLoad
;
#endif
atexit
(
cdfLazyGridDestroyOnce
);
#ifndef HAVE_LIBPTHREAD
cdfLazyInitialized
=
true
;
#endif
}
static
void
cdfLazyGridInit
(
struct
cdfLazyGrid
*
grid
,
int
gridtype
)
{
#ifdef HAVE_LIBPTHREAD
pthread_once
(
&
cdfLazyInitialized
,
cdfLazyGridInitOnce
);
#else
if
(
cdfLazyInitialized
)
;
else
cdfLazyGridInitOnce
();
#endif
grid_init
(
&
grid
->
base
);
cdiGridTypeInit
(
&
grid
->
base
,
gridtype
,
0
);
grid
->
baseVtable
=
grid
->
base
.
vtable
;
grid
->
base
.
vtable
=
&
cdfLazyGridVtable
;
}
static
void
cdfLazyGridRenew
(
struct
cdfLazyGrid
*
restrict
*
restrict
gridpptr
,
int
gridtype
)
{
struct
cdfLazyGrid
*
restrict
grid
=
*
gridpptr
;
if
(
!
grid
)
*
gridpptr
=
grid
=
(
struct
cdfLazyGrid
*
)
Malloc
(
sizeof
(
*
grid
));
cdfLazyGridInit
(
grid
,
gridtype
);
}
/* define all input grids */
static
void
define_all_grids
(
stream_t
*
streamptr
,
int
vlistID
,
ncdim_t
*
ncdims
,
int
nvars
,
ncvar_t
*
ncvars
,
int
timedimid
,
unsigned
char
*
uuidOfHGrid
,
char
*
gridfile
,
int
number_of_grid_used
)
...
...
@@ -4456,7 +4560,9 @@ void define_all_grids(stream_t *streamptr, int vlistID, ncdim_t *ncdims, int nva
size_t
attlen
;
char
attname
[
CDI_MAX_NAME
];
double
datt
;
grid_t
*
restrict
grid
=
NULL
,
*
restrict
proj
=
NULL
;
struct
cdfLazyGrid
*
restrict
lazyGrid
=
NULL
,
*
restrict
lazyProj
=
NULL
;
#define grid (&lazyGrid->base)
#define proj (&lazyProj->base)
for
(
ncvarid
=
0
;
ncvarid
<
nvars
;
++
ncvarid
)
{
...
...
@@ -4541,12 +4647,8 @@ void define_all_grids(stream_t *streamptr, int vlistID, ncdim_t *ncdims, int nva
if
(
ncvars
[
ncvarid
].
gridtype
==
UNDEFID
||
ncvars
[
ncvarid
].
gridtype
==
GRID_GENERIC
)
if
(
xdimid
!=
UNDEFID
&&
xdimid
==
ydimid
&&
nydims
==
0
)
ncvars
[
ncvarid
].
gridtype
=
GRID_UNSTRUCTURED
;
grid
=
(
grid_t
*
)
Realloc
(
grid
,
sizeof
(
*
grid
));
grid_init
(
grid
);
cdiGridTypeInit
(
grid
,
ncvars
[
ncvarid
].
gridtype
,
0
);
proj
=
(
grid_t
*
)
Realloc
(
proj
,
sizeof
(
*
proj
));
grid_init
(
proj
);
cdiGridTypeInit
(
grid
,
GRID_PROJECTION
,
0
);
cdfLazyGridRenew
(
&
lazyGrid
,
ncvars
[
ncvarid
].
gridtype
);
cdfLazyGridRenew
(
&
lazyProj
,
GRID_PROJECTION
);
grid
->
prec
=
DATATYPE_FLT64
;
grid
->
trunc
=
ncvars
[
ncvarid
].
truncation
;
...
...
@@ -5144,13 +5246,25 @@ void define_all_grids(stream_t *streamptr, int vlistID, ncdim_t *ncdims, int nva
}
if
(
gridAdded
.
isNew
)
g
rid
=
NULL
;
lazyG
rid
=
NULL
;
if
(
projAdded
.
isNew
)
p
roj
=
NULL
;
lazyP
roj
=
NULL
;
}
}
Free
(
grid
);
Free
(
proj
);
if
(
lazyGrid
)
{
cdfLazyGridDestroy
(
lazyGrid
);
grid_free
(
grid
);
Free
(
grid
);
}
if
(
lazyProj
)
{
cdfLazyGridDestroy
(
lazyProj
);
grid_free
(
proj
);
Free
(
proj
);
}
#undef proj
#undef grid
}
/* define all input zaxes */
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment