file/data: review write timing

f825382f · Karl-Hermann Wieners · 73df62e7 · f825382f
Commit f825382f authored 10 months ago by Karl-Hermann Wieners
--- a/lectures/file-and-data-systems/timer.ipynb
+++ b/lectures/file-and-data-systems/timer.ipynb
 {
 "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "14cac4fd",
+   "metadata": {},
+   "source": [
+    "# Measure write behavior"
+   ]
+  },
  {
   "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
   "id": "bc0cdf17-1d21-4ade-a090-45101abf8ab5",
   "metadata": {},
   "outputs": [],
   "source": [
-    "import timeit"
+    "import getpass\n",
+    "import os\n",
+    "import timeit\n",
+    "\n",
+    "user = getpass.getuser()\n",
+    "\n",
+    "destination = f\"/scratch/{user[0]}/{user}\""
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
   "id": "bcf39570-a5bc-4af4-9354-06a801222826",
   "metadata": {},
   "outputs": [],
   "source": [
-    "def write (path, length, blocksize=1024**2):\n",
+    "def run_write (path, length, blocksize=1024**2):\n",
-    "    with open(path, \"wb\") as of:\n",
+    "\n",
-    "        data = bytearray(blocksize)\n",
+    "    (times, remainder) = divmod(length, blocksize)\n",
-    "        times = length // (blocksize)\n",
+    "    data = bytearray(blocksize)\n",
-    "        remainder = length % blocksize\n",
+    "\n",
-    "        for i in range (times):\n",
+    "    of = open(path, \"wb\")\n",
-    "            of.write(data)\n",
+    "    for i in range(times):\n",
-    "        if remainder:\n",
+    "        of.write(data)\n",
-    "            of.write(bytearray(remainder))\n",
+    "    if remainder:\n",
-    "        of.close()"
+    "        of.write(bytearray(remainder))\n",
+    "    of.close()"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
   "id": "34a59ea9-52c9-4c3f-8664-a5a97882e5e0",
   "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'1GB': 0.5966405299986945}\n"
-     ]
-    }
-   ],
   "source": [
    "duration = {}\n",
-    "duration['1GB'] = timeit.timeit(lambda : write(\"/scratch/k/k202134/test.1GB\", (1024**3)), number =1)\n",
+    "duration['1GB'] = timeit.timeit(lambda : run_write(f\"{destination}/test.1GB\", (1024**3)), number = 1)\n",
-    "print (duration)"
+    "print(duration)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6b6b7c16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.remove(f\"{destination}/test.1GB\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b1615912",
+   "metadata": {},
+   "source": [
+    "### Other ideas"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "658a2847",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "destination = f\"/dev/shm/{user}\"\n",
+    "# destination = f\"/tmp/{user}\"\n",
+    "os.mkdir(destination)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
-   "display_name": "0 Python 3 (based on the module python3/unstable",
+   "display_name": "Python 3 (ipykernel)",
   "language": "python",
-   "name": "python3_unstable"
+   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
@@ -66,7 +103,7 @@
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
-   "version": "3.10.10"
+   "version": "3.10.12"
  }
 },
 "nbformat": 4,

+%% Cell type:markdown id:14cac4fd tags:
+# Measure write behavior
 %% Cell type:code id:bc0cdf17-1d21-4ade-a090-45101abf8ab5 tags:
 ``` python
+import getpass
+import os
 import timeit
+user = getpass.getuser()
+destination = f"/scratch/{user[0]}/{user}"
 ```
 %% Cell type:code id:bcf39570-a5bc-4af4-9354-06a801222826 tags:
 ``` python
-def write (path, length, blocksize=1024**2):
+def run_write (path, length, blocksize=1024**2):
-    with open(path, "wb") as of:
-        data = bytearray(blocksize)
+    (times, remainder) = divmod(length, blocksize)
-        times = length // (blocksize)
+    data = bytearray(blocksize)
-        remainder = length % blocksize
-        for i in range (times):
+    of = open(path, "wb")
-            of.write(data)
+    for i in range(times):
-        if remainder:
+        of.write(data)
-            of.write(bytearray(remainder))
+    if remainder:
-        of.close()
+        of.write(bytearray(remainder))
+    of.close()
 ```
 %% Cell type:code id:34a59ea9-52c9-4c3f-8664-a5a97882e5e0 tags:
 ``` python
 duration = {}
-duration['1GB'] = timeit.timeit(lambda : write("/scratch/k/k202134/test.1GB", (1024**3)), number =1)
+duration['1GB'] = timeit.timeit(lambda : run_write(f"{destination}/test.1GB", (1024**3)), number = 1)
-print (duration)
+print(duration)
+```
+%% Cell type:code id:6b6b7c16 tags:
+``` python
+os.remove(f"{destination}/test.1GB")
 ```
-%% Output
+%% Cell type:markdown id:b1615912 tags:
+### Other ideas
-    {'1GB': 0.5966405299986945}
+%% Cell type:code id:658a2847 tags:
+``` python
+destination = f"/dev/shm/{user}"
+# destination = f"/tmp/{user}"
+os.mkdir(destination)
+```