StackPractices
intermediate By Mathias Paulenko

Read Large Files in Node.js with Streams (5GB in 64KB RAM)

Process GB-sized files in Node.js with createReadStream, readline, Transform pipelines, and backpressure — constant memory, working code.

Overview

Reading a large file with fs.readFile() loads it entirely into memory — a 5GB file needs 5GB of heap and crashes. Streams process data in ~64KB chunks instead, so a 5GB file costs about 64KB of RAM. This recipe covers createReadStream, readline, pipeline, and Transform streams in Node.js, with backpressure done correctly.

Processing the same file in Python, Java, or Bash? See the multi-language version: Read Large Files.

When to Use

  • Files bigger than RAM: logs, CSV exports, JSONL datasets
  • You transform data while reading (filter, map, convert formats)
  • Memory must stay flat regardless of input size
  • You parse line-by-line or in fixed-size chunks

When NOT to Use

  • Small files (<100MB): fs.readFile is simpler and fast enough
  • You need random access to the middle of a file: streams are sequential
  • The whole result must exist in memory anyway (a JSON object): use fs.readFile + JSON.parse

Solution

Read file line by line with readline

const fs = require("fs");
const readline = require("readline");

async function processLineByLine(filePath) {
    const fileStream = fs.createReadStream(filePath);
    const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity,
    });

    let lineCount = 0;
    for await (const line of rl) {
        lineCount++;
        if (lineCount % 100_000 === 0) {
            console.log(`Processed ${lineCount} lines`);
        }
    }

    console.log(`Total lines: ${lineCount}`);
}

processLineByLine("large_file.log");

Process CSV with transform streams

The parser keeps a leftover buffer: a line split across two chunks stays whole, and flush() emits the final partial line:

const fs = require("fs");
const { Transform } = require("stream");

function createCsvParser() {
    let leftover = "";
    return new Transform({
        readableObjectMode: true, // reads bytes, pushes row arrays
        transform(chunk, encoding, callback) {
            const lines = (leftover + chunk.toString()).split("\n");
            leftover = lines.pop(); // incomplete last line waits for next chunk
            for (const line of lines) {
                if (line.trim()) {
                    this.push(line.split(","));
                }
            }
            callback();
        },
        flush(callback) {
            if (leftover.trim()) {
                this.push(leftover.split(","));
            }
            callback();
        },
    });
}

function createFilter(filterFn) {
    return new Transform({
        objectMode: true,
        transform(row, encoding, callback) {
            if (filterFn(row)) {
                this.push(row);
            }
            callback();
        },
    });
}

function createJsonStringifier() {
    return new Transform({
        objectMode: true,
        transform(row, encoding, callback) {
            this.push(JSON.stringify(row) + "\n");
            callback();
        },
    });
}

// Pipeline: read CSV -> parse -> filter -> convert to JSON -> write
fs.createReadStream("data.csv")
    .pipe(createCsvParser())
    .pipe(createFilter(row => row[2] === "active"))
    .pipe(createJsonStringifier())
    .pipe(fs.createWriteStream("active_users.jsonl"));

Use pipeline for error handling

pipeline() destroys every stream in the chain when one fails — .pipe() leaks them:

const fs = require("fs");
const { pipeline } = require("stream/promises");

async function processFile(inputPath, outputPath) {
    try {
        await pipeline(
            fs.createReadStream(inputPath),
            createCsvParser(),
            createFilter(row => row[2] === "active"),
            createJsonStringifier(),
            fs.createWriteStream(outputPath)
        );
        console.log("Pipeline succeeded");
    } catch (err) {
        console.error("Pipeline failed:", err);
    }
}

processFile("data.csv", "output.jsonl");

Process in batches

const fs = require("fs");
const readline = require("readline");

async function processInBatches(filePath, batchSize = 1000) {
    const rl = readline.createInterface({
        input: fs.createReadStream(filePath),
        crlfDelay: Infinity,
    });

    let batch = [];
    for await (const line of rl) {
        batch.push(line);
        if (batch.length >= batchSize) {
            await processBatch(batch); // pauses reading until done
            batch = [];
        }
    }
    if (batch.length > 0) {
        await processBatch(batch);
    }
}

async function processBatch(lines) {
    // e.g., bulk insert, API call
    console.log(`Processing batch of ${lines.length} lines`);
}

processInBatches("large_file.log", 500);

Async work per line with a concurrency limit

Do concurrency outside the stream — for await on the output gives natural backpressure, no callback juggling:

const fs = require("fs");
const readline = require("readline");

async function enrichFile(inputPath, concurrency = 5) {
    const rl = readline.createInterface({
        input: fs.createReadStream(inputPath),
        crlfDelay: Infinity,
    });

    const pending = new Set();
    for await (const line of rl) {
        const row = line.split(",");
        const task = enrichRow(row);
        pending.add(task);
        task.finally(() => pending.delete(task));

        if (pending.size >= concurrency) {
            await Promise.race(pending); // backpressure: wait for one slot
        }
    }
    await Promise.all(pending);
}

async function enrichRow(row) {
    const response = await fetch(`https://api.example.com/user/${row[0]}`);
    const data = await response.json();
    return [...row, data.name];
}

enrichFile("users.csv");

Read a gzipped file

const fs = require("fs");
const zlib = require("zlib");
const readline = require("readline");

async function processGzippedFile(filePath) {
    // fileStream MUST be piped into gunzip — readline reads from gunzip's output
    const gunzipped = fs.createReadStream(filePath).pipe(zlib.createGunzip());
    const rl = readline.createInterface({ input: gunzipped });

    let count = 0;
    for await (const line of rl) {
        count++;
    }
    console.log(`Total lines: ${count}`);
}

processGzippedFile("large_file.log.gz");

Memory usage: readFile vs stream

// BAD — loads the entire file into memory
// 5GB file = 5GB heap, likely crash
fs.readFile("huge.log", (err, data) => {
    const lines = data.toString().split("\n");
    console.log(lines.length);
});

// GOOD — constant memory, ~64KB at any time
const rl = readline.createInterface({
    input: fs.createReadStream("huge.log", { highWaterMark: 64 * 1024 }),
});
let count = 0;
rl.on("line", () => count++);
rl.on("close", () => console.log(count));

How Streams Work

Node.js has four stream types: Readable produces data (file reads, HTTP responses), Writable consumes it (file writes), Transform does both while modifying the data, and Duplex is a bidirectional channel. A read-modify-write pipeline is Readable → Transform(s) → Writable.

flowchart diagram: createReadStream<br>64KB chunks

Two settings matter:

  • highWaterMark — internal buffer size. It’s 64KB for byte streams and 16 objects when you use objectMode. A bigger buffer cuts down on syscalls but costs you more RAM.
  • objectMode — pushes objects instead of bytes; required for parsed records (CSV rows).

Backpressure is automatic through .pipe() and pipeline(): when the destination is busy, upstream pauses. If you write to a stream manually, respect write() returning false and wait for the drain event.

Variants

ApproachMemorySpeedUse When
fs.readFileO(file size)FastFiles < 100MB
readline + createReadStreamO(buffer)MediumLine-based files
Transform pipelineO(buffer)MediumTransform while reading
Readable.from + async generatorO(buffer)FlexibleCustom data sources
csv-parse / stream-jsonO(buffer)MediumReal CSV/JSON with quoting, nesting

For production CSV/JSON parsing, prefer battle-tested streaming parsers (csv-parse handles quoted fields; stream-json handles nested JSON) over hand-rolled splits.

What Works

  • Reach for streams once a file crosses ~100MB. fs.readFile crashes on multi-GB files.
  • pipeline() (or stream/promises version) over .pipe() chains — errors destroy every stream in the chain.
  • readline for line formats (logs, CSV, JSONL): it handles \n and \r\n correctly.
  • objectMode transforms for parsed records.
  • Batch external calls (inserts, API calls) instead of one per line.
  • for await...of on a stream for natural backpressure — the loop simply waits.

Common Mistakes

  • fs.readFile() on large files — OOM crash waiting to happen.
  • .pipe() without error handling — a failed stream leaves the others leaking file descriptors.
  • Splitting chunks on \n without a leftover buffer — lines get cut at chunk boundaries.
  • Calling callback() twice inside a Transform — ERR_MULTIPLE_CALLBACK. Store the deferred callback if you defer it.
  • readline on binary files — it’s a text tool. Use raw chunks for binary.
  • Forgetting objectMode when pushing objects — they get stringified.
  • Forgetting the callback() entirely — the stream hangs mid-pipeline.

See Also

Companion code: a runnable CSV→JSONL pipeline lives in the companion repo.

Frequently Asked Questions

What's the difference between readline and createReadStream?

createReadStream emits raw byte chunks (~64KB each) with no concept of lines. readline wraps a stream and emits complete lines, handling \n and \r\n. Use createReadStream directly for binary data or fixed-size chunks; use readline for text formats.

How do I read a file without loading it into memory?

fs.createReadStream(path) plus readline or a Transform pipeline — the file is processed in ~64KB chunks and only the buffer lives in RAM.

What is the maximum file size I can process with streams?

No practical limit. Memory stays flat because data flows through in chunks — you can process files larger than available RAM.

Should I use .pipe() or pipeline()?

pipeline(). It destroys every stream in the chain when one fails; .pipe() forwards no errors and leaves streams open.

How do I handle backpressure in a custom Transform?

this.push() returning false means the destination is full — stop pushing until the next transform() call. Let the stream framework pace callback(); if you defer it, store the callback and call it exactly once.

Can I process binary files with readline?

No — readline splits on line endings and mangles binary data. Use raw createReadStream chunks and Buffer operations.