From d5bd43cd27754c4bd92fb91712ec64c2da7c7144 Mon Sep 17 00:00:00 2001 From: Tobin Date: Wed, 4 Sep 2019 15:00:47 +0100 Subject: [PATCH] Example code for a simple node stream + buffer There isn't anything too obvious about writing to streams in the readme so thought it'd be useful for others :) --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 82a47a8c..db32e779 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,30 @@ avoid leaking file descriptors. await reader.close(); ``` +### Using Streams + +Instead of using a file, a writeable stream can consume the parquet output. The stream needs a `close` method defined which is called when the parquetWriter is closed. To write to a `Buffer`, add it as part of the stream. + +```js +let stream = require('stream'); + +let writeableStream = new stream.Writable(); +let buffer = Buffer.from(''); + +writeableStream._write = (chunk, encoding, done) => { + buffer = Buffer.concat([buffer, chunk]); + done(); + }; +writeableStream.close = e => e(); // pass in an error if there was one + +let parquetWriter = await parquet.ParquetWriter.openStream(schema, writeableStream); + +parquetWriter.appendRow() +parquetWriter.close() + +buffer // contains all appended rows in parquet format +``` + Encodings ---------