feat(file-io): add fileChunks()

thi-ng · Jan 19, 2023 · bcff691 · bcff691
1 parent 6b185f4
commit bcff691
Show file tree

Hide file tree

Showing 4 changed files with 83 additions and 0 deletions.
diff --git a/packages/file-io/package.json b/packages/file-io/package.json
@@ -36,6 +36,7 @@
 	"dependencies": {
 		"@thi.ng/api": "^8.6.3",
 		"@thi.ng/checks": "^3.3.7",
+		"@thi.ng/hex": "^2.3.4",
 		"@thi.ng/logger": "^1.4.7",
 		"@thi.ng/random": "^3.3.21"
 	},
@@ -83,6 +84,9 @@
 		"./ext": {
 			"default": "./ext.js"
 		},
+		"./file-chunks": {
+			"default": "./file-chunks.js"
+		},
 		"./files": {
 			"default": "./files.js"
 		},

diff --git a/packages/file-io/src/file-chunks.ts b/packages/file-io/src/file-chunks.ts
@@ -0,0 +1,77 @@
+import type { Nullable } from "@thi.ng/api";
+import { U32 } from "@thi.ng/hex";
+import type { ILogger } from "@thi.ng/logger";
+import { FileHandle, open } from "fs/promises";
+
+export interface FileChunkOpts {
+	/**
+	 * Optional logger instance
+	 */
+	logger: ILogger;
+	/**
+	 * Chunk size (in bytes).
+	 *
+	 * @defaultValue 1024
+	 */
+	size: number;
+	/**
+	 * Read start position (in bytes)
+	 *
+	 * @defaultValue 0
+	 */
+	start: number;
+	/**
+	 * Read end position (in bytes)
+	 *
+	 * @defaultValue Infinity
+	 */
+	end: number;
+}
+
+/**
+ * Async iterator. Yields chunks of byte buffers from given file. User
+ * configurable size and byte ranges.
+ *
+ * @example
+ * ```ts
+ * for await(let buf of fileChunks("file.bin", { start: 16*1024*1024 })) {
+ *   // ...
+ * }
+ * ```
+ *
+ * @param path
+ * @param opts
+ */
+export async function* fileChunks(path: string, opts?: Partial<FileChunkOpts>) {
+	let { logger, size, start, end } = {
+		size: 1024,
+		start: 0,
+		end: Infinity,
+		...opts,
+	};
+	logger &&
+		logger.debug(`start reading file chunks (size: 0x${size}): ${path}`);
+	let fd: Nullable<FileHandle> = undefined;
+	try {
+		fd = await open(path, "r");
+		while (start < end) {
+			logger &&
+				logger.debug(
+					`reading chunk: 0x${U32(start)} - 0x${U32(
+						start + size - 1
+					)} (${path})`
+				);
+			const { buffer, bytesRead } = await fd.read({
+				buffer: Buffer.alloc(size),
+				length: size,
+				position: start,
+			});
+			if (bytesRead === 0) break;
+			yield buffer;
+			if (bytesRead < size) break;
+			start += bytesRead;
+		}
+	} finally {
+		await fd?.close();
+	}
+}
diff --git a/packages/file-io/src/index.ts b/packages/file-io/src/index.ts
@@ -1,6 +1,7 @@
 export * from "./delete.js";
 export * from "./dir.js";
 export * from "./ext.js";
+export * from "./file-chunks.js";
 export * from "./files.js";
 export * from "./hash.js";
 export * from "./json.js";

diff --git a/yarn.lock b/yarn.lock
@@ -3067,6 +3067,7 @@ __metadata:
     "@microsoft/api-extractor": ^7.33.7
     "@thi.ng/api": ^8.6.3
     "@thi.ng/checks": ^3.3.7
+    "@thi.ng/hex": ^2.3.4
     "@thi.ng/logger": ^1.4.7
     "@thi.ng/random": ^3.3.21
     "@thi.ng/testament": ^0.3.9