Read Large Files in Node.js with Streams (5GB in 64KB RAM)
Process GB-sized files in Node.js with createReadStream, readline, Transform pipelines, and backpressure — constant memory, working code.
Overview
Reading a large file with fs.readFile() loads it entirely into memory — a 5GB file needs 5GB of heap and crashes. Streams process data in ~64KB chunks instead, so a 5GB file costs about 64KB of RAM. This recipe covers createReadStream, readline, pipeline, and Transform streams in Node.js, with backpressure done correctly.
Processing the same file in Python, Java, or Bash? See the multi-language version: Read Large Files.
When to Use
- Files bigger than RAM: logs, CSV exports, JSONL datasets
- You transform data while reading (filter, map, convert formats)
- Memory must stay flat regardless of input size
- You parse line-by-line or in fixed-size chunks
When NOT to Use
- Small files (<100MB):
fs.readFileis simpler and fast enough - You need random access to the middle of a file: streams are sequential
- The whole result must exist in memory anyway (a JSON object): use
fs.readFile+JSON.parse
Solution
Read file line by line with readline
const fs = require("fs");
const readline = require("readline");
async function processLineByLine(filePath) {
const fileStream = fs.createReadStream(filePath);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
let lineCount = 0;
for await (const line of rl) {
lineCount++;
if (lineCount % 100_000 === 0) {
console.log(`Processed ${lineCount} lines`);
}
}
console.log(`Total lines: ${lineCount}`);
}
processLineByLine("large_file.log");
Process CSV with transform streams
The parser keeps a leftover buffer: a line split across two chunks stays whole, and flush() emits the final partial line:
const fs = require("fs");
const { Transform } = require("stream");
function createCsvParser() {
let leftover = "";
return new Transform({
readableObjectMode: true, // reads bytes, pushes row arrays
transform(chunk, encoding, callback) {
const lines = (leftover + chunk.toString()).split("\n");
leftover = lines.pop(); // incomplete last line waits for next chunk
for (const line of lines) {
if (line.trim()) {
this.push(line.split(","));
}
}
callback();
},
flush(callback) {
if (leftover.trim()) {
this.push(leftover.split(","));
}
callback();
},
});
}
function createFilter(filterFn) {
return new Transform({
objectMode: true,
transform(row, encoding, callback) {
if (filterFn(row)) {
this.push(row);
}
callback();
},
});
}
function createJsonStringifier() {
return new Transform({
objectMode: true,
transform(row, encoding, callback) {
this.push(JSON.stringify(row) + "\n");
callback();
},
});
}
// Pipeline: read CSV -> parse -> filter -> convert to JSON -> write
fs.createReadStream("data.csv")
.pipe(createCsvParser())
.pipe(createFilter(row => row[2] === "active"))
.pipe(createJsonStringifier())
.pipe(fs.createWriteStream("active_users.jsonl"));
Use pipeline for error handling
pipeline() destroys every stream in the chain when one fails — .pipe() leaks them:
const fs = require("fs");
const { pipeline } = require("stream/promises");
async function processFile(inputPath, outputPath) {
try {
await pipeline(
fs.createReadStream(inputPath),
createCsvParser(),
createFilter(row => row[2] === "active"),
createJsonStringifier(),
fs.createWriteStream(outputPath)
);
console.log("Pipeline succeeded");
} catch (err) {
console.error("Pipeline failed:", err);
}
}
processFile("data.csv", "output.jsonl");
Process in batches
const fs = require("fs");
const readline = require("readline");
async function processInBatches(filePath, batchSize = 1000) {
const rl = readline.createInterface({
input: fs.createReadStream(filePath),
crlfDelay: Infinity,
});
let batch = [];
for await (const line of rl) {
batch.push(line);
if (batch.length >= batchSize) {
await processBatch(batch); // pauses reading until done
batch = [];
}
}
if (batch.length > 0) {
await processBatch(batch);
}
}
async function processBatch(lines) {
// e.g., bulk insert, API call
console.log(`Processing batch of ${lines.length} lines`);
}
processInBatches("large_file.log", 500);
Async work per line with a concurrency limit
Do concurrency outside the stream — for await on the output gives natural backpressure, no callback juggling:
const fs = require("fs");
const readline = require("readline");
async function enrichFile(inputPath, concurrency = 5) {
const rl = readline.createInterface({
input: fs.createReadStream(inputPath),
crlfDelay: Infinity,
});
const pending = new Set();
for await (const line of rl) {
const row = line.split(",");
const task = enrichRow(row);
pending.add(task);
task.finally(() => pending.delete(task));
if (pending.size >= concurrency) {
await Promise.race(pending); // backpressure: wait for one slot
}
}
await Promise.all(pending);
}
async function enrichRow(row) {
const response = await fetch(`https://api.example.com/user/${row[0]}`);
const data = await response.json();
return [...row, data.name];
}
enrichFile("users.csv");
Read a gzipped file
const fs = require("fs");
const zlib = require("zlib");
const readline = require("readline");
async function processGzippedFile(filePath) {
// fileStream MUST be piped into gunzip — readline reads from gunzip's output
const gunzipped = fs.createReadStream(filePath).pipe(zlib.createGunzip());
const rl = readline.createInterface({ input: gunzipped });
let count = 0;
for await (const line of rl) {
count++;
}
console.log(`Total lines: ${count}`);
}
processGzippedFile("large_file.log.gz");
Memory usage: readFile vs stream
// BAD — loads the entire file into memory
// 5GB file = 5GB heap, likely crash
fs.readFile("huge.log", (err, data) => {
const lines = data.toString().split("\n");
console.log(lines.length);
});
// GOOD — constant memory, ~64KB at any time
const rl = readline.createInterface({
input: fs.createReadStream("huge.log", { highWaterMark: 64 * 1024 }),
});
let count = 0;
rl.on("line", () => count++);
rl.on("close", () => console.log(count));
How Streams Work
Node.js has four stream types: Readable produces data (file reads, HTTP responses), Writable consumes it (file writes), Transform does both while modifying the data, and Duplex is a bidirectional channel. A read-modify-write pipeline is Readable → Transform(s) → Writable.
Two settings matter:
highWaterMark— internal buffer size. It’s 64KB for byte streams and 16 objects when you useobjectMode. A bigger buffer cuts down on syscalls but costs you more RAM.objectMode— pushes objects instead of bytes; required for parsed records (CSV rows).
Backpressure is automatic through .pipe() and pipeline(): when the destination is busy, upstream pauses. If you write to a stream manually, respect write() returning false and wait for the drain event.
Variants
| Approach | Memory | Speed | Use When |
|---|---|---|---|
fs.readFile | O(file size) | Fast | Files < 100MB |
readline + createReadStream | O(buffer) | Medium | Line-based files |
| Transform pipeline | O(buffer) | Medium | Transform while reading |
Readable.from + async generator | O(buffer) | Flexible | Custom data sources |
| csv-parse / stream-json | O(buffer) | Medium | Real CSV/JSON with quoting, nesting |
For production CSV/JSON parsing, prefer battle-tested streaming parsers (csv-parse handles quoted fields; stream-json handles nested JSON) over hand-rolled splits.
What Works
- Reach for streams once a file crosses ~100MB.
fs.readFilecrashes on multi-GB files. pipeline()(orstream/promisesversion) over.pipe()chains — errors destroy every stream in the chain.readlinefor line formats (logs, CSV, JSONL): it handles\nand\r\ncorrectly.objectModetransforms for parsed records.- Batch external calls (inserts, API calls) instead of one per line.
for await...ofon a stream for natural backpressure — the loop simply waits.
Common Mistakes
fs.readFile()on large files — OOM crash waiting to happen..pipe()without error handling — a failed stream leaves the others leaking file descriptors.- Splitting chunks on
\nwithout a leftover buffer — lines get cut at chunk boundaries. - Calling
callback()twice inside a Transform —ERR_MULTIPLE_CALLBACK. Store the deferred callback if you defer it. readlineon binary files — it’s a text tool. Use raw chunks for binary.- Forgetting
objectModewhen pushing objects — they get stringified. - Forgetting the
callback()entirely — the stream hangs mid-pipeline.
See Also
- Node.js stream API: pipeline, Transform, backpressure.
- Node.js readline: line-by-line interface details.
- csv-parse: streaming CSV parser for real-world quoting.
- Stream processing guide: the long-form version.
Companion code: a runnable CSV→JSONL pipeline lives in the companion repo.
Frequently Asked Questions
What's the difference between readline and createReadStream?
createReadStream emits raw byte chunks (~64KB each) with no concept of lines. readline wraps a stream and emits complete lines, handling \n and \r\n. Use createReadStream directly for binary data or fixed-size chunks; use readline for text formats.
How do I read a file without loading it into memory?
fs.createReadStream(path) plus readline or a Transform pipeline — the file is processed in ~64KB chunks and only the buffer lives in RAM.
What is the maximum file size I can process with streams?
No practical limit. Memory stays flat because data flows through in chunks — you can process files larger than available RAM.
Should I use .pipe() or pipeline()?
pipeline(). It destroys every stream in the chain when one fails; .pipe() forwards no errors and leaves streams open.
How do I handle backpressure in a custom Transform?
this.push() returning false means the destination is full — stop pushing until the next transform() call. Let the stream framework pace callback(); if you defer it, store the callback and call it exactly once.
Can I process binary files with readline?
No — readline splits on line endings and mangles binary data. Use raw createReadStream chunks and Buffer operations.
Related Resources
Read Large Files
How to read large files efficiently without running out of memory.
RecipeProcess Large Files with Streams
How to read, transform, and write large files efficiently using streams without loading entire files into memory in Python, Node.js, and Java.
RecipeWrite Large Files
How to write large files efficiently using buffered and streaming output.
RecipeCompress and Decompress Files with ZIP, GZIP, and TAR
Create and extract ZIP, GZIP, and TAR archives in Python, Node.js, Java, and Bash, with streaming reads and safe-extraction patterns.
RecipeWatch File Changes
How to monitor file system changes in real time. Covers kernel-level watchers in Python, JavaScript, Java, and Bash, with debouncing, OVERFLOW handling, and polling fallbacks.
GuideStream Processing: Event-Driven Data Pipelines with
A practical guide to stream processing: choosing between Kafka Streams, Flink, and Spark Streaming, designing event schemas, handling stateful operations, and building exactly-once processing pipelines for real-time data.