Commit 124fab99 authored by Thomas Jahns's avatar Thomas Jahns 🤸
Browse files

Align block buffer to allow for direct I/O.

* It is unclear if MPI I/O can make use of this, but it cannot hurt either.
parent e72a4b5b
......@@ -29440,6 +29440,98 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: MPI launch command unavailable" >&5
$as_echo "$as_me: WARNING: MPI launch command unavailable" >&2;}
fi
ac_fn_c_check_decl "$LINENO" "_SC_LARGE_PAGESIZE" "ac_cv_have_decl__SC_LARGE_PAGESIZE" "$ac_includes_default
#include <limits.h>
#include <unistd.h>
"
if test "x$ac_cv_have_decl__SC_LARGE_PAGESIZE" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL__SC_LARGE_PAGESIZE $ac_have_decl
_ACEOF
ac_fn_c_check_decl "$LINENO" "PAGESIZE" "ac_cv_have_decl_PAGESIZE" "$ac_includes_default
#include <limits.h>
#include <unistd.h>
"
if test "x$ac_cv_have_decl_PAGESIZE" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL_PAGESIZE $ac_have_decl
_ACEOF
ac_fn_c_check_decl "$LINENO" "PAGE_SIZE" "ac_cv_have_decl_PAGE_SIZE" "$ac_includes_default
#include <limits.h>
#include <unistd.h>
"
if test "x$ac_cv_have_decl_PAGE_SIZE" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL_PAGE_SIZE $ac_have_decl
_ACEOF
ac_fn_c_check_decl "$LINENO" "_SC_PAGE_SIZE" "ac_cv_have_decl__SC_PAGE_SIZE" "$ac_includes_default
#include <limits.h>
#include <unistd.h>
"
if test "x$ac_cv_have_decl__SC_PAGE_SIZE" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL__SC_PAGE_SIZE $ac_have_decl
_ACEOF
ac_fn_c_check_decl "$LINENO" "_SC_PAGESIZE" "ac_cv_have_decl__SC_PAGESIZE" "$ac_includes_default
#include <limits.h>
#include <unistd.h>
"
if test "x$ac_cv_have_decl__SC_PAGESIZE" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL__SC_PAGESIZE $ac_have_decl
_ACEOF
ac_fn_c_check_decl "$LINENO" "_PC_REC_XFER_ALIGN" "ac_cv_have_decl__PC_REC_XFER_ALIGN" "$ac_includes_default
#include <limits.h>
#include <unistd.h>
"
if test "x$ac_cv_have_decl__PC_REC_XFER_ALIGN" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL__PC_REC_XFER_ALIGN $ac_have_decl
_ACEOF
ac_fn_c_check_decl "$LINENO" "POSIX_REC_XFER_ALIGN" "ac_cv_have_decl_POSIX_REC_XFER_ALIGN" "$ac_includes_default
#include <limits.h>
#include <unistd.h>
"
if test "x$ac_cv_have_decl_POSIX_REC_XFER_ALIGN" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL_POSIX_REC_XFER_ALIGN $ac_have_decl
_ACEOF
 
 
pkg_failed=no
......@@ -195,6 +195,10 @@ main(int argc, char **argv)
])
AS_IF([test "x$MPI_LAUNCH" = xtrue],
[AC_MSG_WARN([MPI launch command unavailable])])
AC_CHECK_DECLS([_SC_LARGE_PAGESIZE, PAGESIZE, PAGE_SIZE, _SC_PAGE_SIZE, dnl
_SC_PAGESIZE, _PC_REC_XFER_ALIGN, POSIX_REC_XFER_ALIGN],,,[AC_INCLUDES_DEFAULT
@%:@include <limits.h>
@%:@include <unistd.h>])
PKG_CHECK_MODULES([YAXT],[yaxt],
[AC_DEFINE([HAVE_YAXT],,[yaxt library is available])],
......
......@@ -22,6 +22,18 @@
*/
#undef HAVE_DECL_ISNAN
/* Define to 1 if you have the declaration of `PAGESIZE', and to 0 if you
don't. */
#undef HAVE_DECL_PAGESIZE
/* Define to 1 if you have the declaration of `PAGE_SIZE', and to 0 if you
don't. */
#undef HAVE_DECL_PAGE_SIZE
/* Define to 1 if you have the declaration of `POSIX_REC_XFER_ALIGN', and to 0
if you don't. */
#undef HAVE_DECL_POSIX_REC_XFER_ALIGN
/* Define to 1 if you have the declaration of `uuid_create', and to 0 if you
don't. */
#undef HAVE_DECL_UUID_CREATE
......@@ -30,6 +42,22 @@
don't. */
#undef HAVE_DECL_UUID_GENERATE
/* Define to 1 if you have the declaration of `_PC_REC_XFER_ALIGN', and to 0
if you don't. */
#undef HAVE_DECL__PC_REC_XFER_ALIGN
/* Define to 1 if you have the declaration of `_SC_LARGE_PAGESIZE', and to 0
if you don't. */
#undef HAVE_DECL__SC_LARGE_PAGESIZE
/* Define to 1 if you have the declaration of `_SC_PAGESIZE', and to 0 if you
don't. */
#undef HAVE_DECL__SC_PAGESIZE
/* Define to 1 if you have the declaration of `_SC_PAGE_SIZE', and to 0 if you
don't. */
#undef HAVE_DECL__SC_PAGE_SIZE
/* Define to 1 if you have the declaration of `__builtin_ctz', and to 0 if you
don't. */
#undef HAVE_DECL___BUILTIN_CTZ
......
......@@ -93,6 +93,75 @@ initReblockPendingMsg(aFiledataM *of, size_t i)
of->reqs[i] = MPI_REQUEST_NULL;
}
static inline size_t
gcd(size_t a, size_t b)
{
while (b != 0)
{
size_t t = a;
a = b;
b = t % b;
}
return a;
}
static inline size_t
lcm(size_t a, size_t b)
{
return a / gcd(a, b) * b;
}
enum {
/* 8192 is known to work on most systems (4096 isn't on Alpha) */
commonPageSize = 8192,
};
static size_t
getPageSize(void)
{
long pagesize = -1L;
#if HAVE_DECL__SC_LARGE_PAGESIZE || HAVE_DECL__SC_PAGE_SIZE || HAVE_DECL__SC_PAGESIZE
int name =
#if HAVE_DECL__SC_LARGE_PAGESIZE
_SC_LARGE_PAGESIZE
#elif HAVE_DECL__SC_PAGESIZE
_SC_PAGESIZE
#elif HAVE_DECL__SC_PAGE_SIZE
_SC_PAGE_SIZE
#endif
;
pagesize = sysconf(name);
#endif
if (pagesize == -1L)
pagesize =
#if HAVE_DECL_PAGESIZE
PAGESIZE
#elif HAVE_DECL_PAGE_SIZE
PAGE_SIZE
#else
commonPageSize
#endif
;
return (size_t)pagesize;
}
static size_t
getXferBufAlign(const char *path)
{
long align = -1L;
#if HAVE_DECL__PC_REC_XFER_ALIGN
align = pathconf(path, _PC_REC_XFER_ALIGN);
#endif
if (align == -1L)
align =
#if HAVE_DECL_POSIX_REC_XFER_ALIGN
POSIX_REC_XFER_ALIGN
#else
commonPageSize
#endif
;
return (size_t)align;
}
static aFiledataM *
initAFiledataFileWriteAtReblock(const char *filename, size_t bufSize)
......@@ -114,7 +183,7 @@ initAFiledataFileWriteAtReblock(const char *filename, size_t bufSize)
open_info, &fh));
xmpi(MPI_Info_free(&open_info));
/* find block size of underlying file system */
size_t blockSize;
size_t blockSize, bufBlockAlign;
{
struct stat posixstat;
int rc = stat(filename, &posixstat);
......@@ -127,11 +196,28 @@ initAFiledataFileWriteAtReblock(const char *filename, size_t bufSize)
blockSize = (size_t)posixstat.st_blksize < SIZE_MAX
? (size_t)posixstat.st_blksize
: SIZE_MAX/2;
/* prevent inefficiently small writes on granular file systems */
#define MINBLOCKSIZE ((size_t)1 << 19)
if ((blockSize < MINBLOCKSIZE) & !(MINBLOCKSIZE % blockSize))
blockSize = MINBLOCKSIZE;
#undef MINBLOCKSIZE
/* ensure block size also meets page and direct I/O buffer
* alignment requirements */
size_t pageSize = getPageSize(),
IOAlign = getXferBufAlign(filename);
bufBlockAlign = lcm(pageSize, IOAlign);
blockSize = lcm(blockSize, bufBlockAlign);
}
/* round bufSize to next multiple of block size */
/* round bufSize to next multiple of block size and I/O alignment */
bufSize = roundUpToMultiple(bufSize, blockSize);
size_t numBlockBuf = bufSize / blockSize;
assert(blockSize <= INT_MAX && numBlockBuf <= INT_MAX);
/* never go less than double-buffering */
if (numBlockBuf < 2)
{
numBlockBuf = 2;
bufSize = blockSize * 2;
}
aFiledataM *of
= Malloc(sizeof (*of) + sizeof (of->collWriteSize[0]) * (size_t)sizePio
+ nameSize);
......@@ -139,7 +225,15 @@ initAFiledataFileWriteAtReblock(const char *filename, size_t bufSize)
of->name = (char *)((unsigned char *)of + sizeof (*of)
+ sizeof (of->collWriteSize[0]) * (size_t)sizePio);
memcpy(of->name, filename, nameSize);
of->blockBuf = Malloc(bufSize);
{
void *ptr;
int err = posix_memalign(&ptr, bufBlockAlign, bufSize);
if (!err)
of->blockBuf = ptr;
else
cdiAbort(__FILE__, __func__, __LINE__, "posix_memalign failed: %s",
strerror(errno));
}
of->pending = (struct pendingBufWrite *)
Malloc(numBlockBuf * sizeof (struct pendingBufWrite));
for (size_t i = 0; i < numBlockBuf; ++i)
......@@ -198,7 +292,7 @@ destroyAFiledataFileWriteAtReblock(void *v)
int iret = MPI_File_close(&of->fh);
for (size_t i = 0; i < (size_t)of->msgSize; ++i)
Free(of->msgs[i].sendBuf);
Free(of->blockBuf);
free(of->blockBuf);
Free(of->msgs);
Free(of->reqs);
Free(of->pending);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment