From dbc6ce28791930c85a0b724677543158df02aa34 Mon Sep 17 00:00:00 2001 From: contra Date: Sat, 15 Aug 2020 14:15:54 -0700 Subject: [PATCH] chore: code cleanup, new docs, API cleanup --- .gitignore | 3 ++- cli.js | 31 +++++++++++++++------------ collaborators.md | 7 ------- index.js | 25 +++++++++------------- package.json | 18 +++++++++------- readme.md | 45 ++++++++++++++++++++-------------------- test/data.txt | 2 ++ test.js => test/index.js | 28 ++++++++++++------------- 8 files changed, 79 insertions(+), 80 deletions(-) delete mode 100644 collaborators.md create mode 100644 test/data.txt rename test.js => test/index.js (75%) diff --git a/.gitignore b/.gitignore index 25c8fdb..960be9a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules -package-lock.json \ No newline at end of file +package-lock.json +yarn.lock \ No newline at end of file diff --git a/cli.js b/cli.js index 7cc9b42..7bd4431 100755 --- a/cli.js +++ b/cli.js @@ -1,23 +1,28 @@ #!/usr/bin/env node -var fs = require('fs') -var ndjson = require('./index.js') -var minimist = require('minimist') +const fs = require('fs') +const { pipeline } = require('readable-stream') +const minimist = require('minimist') +const ndjson = require('./index.js') -var args = minimist(process.argv.slice(2)) +const args = minimist(process.argv.slice(2)) +const first = args._[0] -var inputStream - -var first = args._[0] if (!first) { console.error('Usage: ndjson [input] ') process.exit(1) } -if (first === '-') inputStream = process.stdin -else inputStream = fs.createReadStream(first) +const inputStream = first === '-' + ? process.stdin + : fs.createReadStream(first) -var parse = ndjson.parse(args) -var serializer = ndjson.serialize(args) - -inputStream.pipe(parse).pipe(serializer).pipe(process.stdout) +pipeline( + inputStream, + ndjson.parse(args), + ndjson.stringify(args), + process.stdout, + (err) => { + err ? process.exit(1) : process.exit(0) + } +) \ No newline at end of file diff --git a/collaborators.md b/collaborators.md deleted file mode 100644 index 21497a2..0000000 --- a/collaborators.md +++ /dev/null @@ -1,7 +0,0 @@ -## Collaborators - -ldjson-stream is only possible due to the excellent work of the following collaborators: - - - -
maxogdenGitHub/maxogden
finnpGitHub/finnp
diff --git a/index.js b/index.js index 15edcc7..c4c480c 100644 --- a/index.js +++ b/index.js @@ -1,13 +1,14 @@ -var through = require('through2') -var split = require('split2') -var EOL = require('os').EOL -var stringify = require('json-stringify-safe') +const through = require('through2') +const split = require('split2') +const { EOL } = require('os') +const stringify = require('json-stringify-safe') -module.exports = parse -module.exports.serialize = module.exports.stringify = serialize -module.exports.parse = parse +module.exports.stringify = (opts) => + through.obj(opts, (obj, _, cb) => { + cb(null, stringify(obj) + EOL) + }) -function parse (opts) { +module.exports.parse = (opts) => { opts = opts || {} opts.strict = opts.strict !== false @@ -22,10 +23,4 @@ function parse (opts) { } return split(parseRow, opts) -} - -function serialize (opts) { - return through.obj(opts, function(obj, enc, cb) { - cb(null, stringify(obj) + EOL) - }) -} +} \ No newline at end of file diff --git a/package.json b/package.json index a735c65..c6c343c 100644 --- a/package.json +++ b/package.json @@ -1,19 +1,23 @@ { - "name": "ndjson-next", - "version": "1.5.1", - "description": "streaming newline delimited json parser + serializer", + "name": "ndjson", + "version": "2.0.0", + "description": "Streaming newline delimited json parser + serializer", "main": "index.js", "scripts": { - "test": "tape test.js" + "test": "tape test/index.js" }, "bin": { "ndjson": "cli.js" }, "author": "max ogden", "license": "BSD-3-Clause", + "engines": { + "node": ">=10" + }, "dependencies": { "json-stringify-safe": "^5.0.1", "minimist": "^1.2.5", + "readable-stream": "^3.6.0", "split2": "^3.0.0", "through2": "^4.0.0" }, @@ -23,12 +27,12 @@ }, "repository": { "type": "git", - "url": "git://github.com/contra/ndjson.git" + "url": "git://github.com/ndjson/ndjson.js.git" }, "bugs": { - "url": "https://github.com/contra/ndjson/issues" + "url": "https://github.com/ndjson/ndjson.js/issues" }, - "homepage": "https://github.com/contra/ndjson", + "homepage": "https://github.com/ndjson/ndjson.js", "keywords": [ "ndjson", "ldjson" diff --git a/readme.md b/readme.md index 10e7486..38c8697 100644 --- a/readme.md +++ b/readme.md @@ -1,38 +1,27 @@ -# Fork Notes +# ndjson -- Updates all dependencies -- Promises to stay maintained and responsive to PRs +Streaming [newline delimited json](https://en.wikipedia.org/wiki/Line_Delimited_JSON) parser + serializer. Available as a JS API and a CLI. -All thanks to the original maintainer Max Ogden. +[![NPM](https://nodei.co/npm/ndjson.png)](https://nodei.co/npm/ndjson/) -# ndjson-next - -streaming [newline delimited json](https://en.wikipedia.org/wiki/Line_Delimited_JSON) parser + serializer. Available as a JS API or a command line tool - -[![NPM](https://nodei.co/npm/ndjson-next.png)](https://nodei.co/npm/ndjson-next/) - -## usage +## Usage ``` -var ndjson = require('ndjson-next') +const ndjson = require('ndjson') ``` -#### ndjson.parse(opts) +#### ndjson.parse([opts]) -returns a transform stream that accepts newline delimited json and emits objects +Returns a transform stream that accepts newline delimited json buffers and emits objects of parsed data. -example newline delimited json: - -`data.txt`: +Example file: ``` {"foo": "bar"} {"hello": "world"} ``` -If you want to discard non-valid JSON messages, you can call `ndjson.parse({strict: false})` - -usage: +Parsing it: ```js fs.createReadStream('data.txt') @@ -42,9 +31,15 @@ fs.createReadStream('data.txt') }) ``` -#### ndjson.serialize() / ndjson.stringify() -returns a transform stream that accepts json objects and emits newline delimited json +##### Options + +- `strict` can be set to false to discard non-valid JSON messages +- All other options are passed through to the stream class. + +#### ndjson.stringify([opts]) + +Returns a transform stream that accepts JSON objects and emits newline delimited json buffers. example usage: @@ -57,6 +52,10 @@ serialize.write({"foo": "bar"}) serialize.end() ``` -### license +##### Options + +Options are passed through to the stream class. + +### LICENSE BSD-3-Clause diff --git a/test/data.txt b/test/data.txt new file mode 100644 index 0000000..aa33ed5 --- /dev/null +++ b/test/data.txt @@ -0,0 +1,2 @@ +{"foo": "bar"} +{"hello": "world"} \ No newline at end of file diff --git a/test.js b/test/index.js similarity index 75% rename from test.js rename to test/index.js index 46b4b75..49d1ffe 100644 --- a/test.js +++ b/test/index.js @@ -1,10 +1,10 @@ -var test = require('tape') -var ndj = require('./') -var os = require('os') -var concat = require('concat-stream') +const test = require('tape') +const os = require('os') +const concat = require('concat-stream') +const ndj = require('../') test('.parse', function(t) { - var parser = ndj.parse() + const parser = ndj.parse() parser.on('data', function(obj) { t.equal(obj.hello, 'world') t.end() @@ -14,7 +14,7 @@ test('.parse', function(t) { }) test('.parse twice', function(t) { - var parser = ndj.parse() + const parser = ndj.parse() parser.once('data', function(obj) { t.equal(obj.hello, 'world') parser.once('data', function(obj) { @@ -27,7 +27,7 @@ test('.parse twice', function(t) { }) test('.parse - strict:true error', function (t) { - var parser = ndj.parse({strict: true}) + const parser = ndj.parse({strict: true}) try { parser.write('{"no":"json"\n') } catch(e) { @@ -37,7 +37,7 @@ test('.parse - strict:true error', function (t) { }) test('.parse - strict:true error event', function (t) { - var parser = ndj.parse({strict: true}) + const parser = ndj.parse({strict: true}) parser.on('error', function (err) { t.pass('error event called') t.end() @@ -50,7 +50,7 @@ test('.parse - strict:true error event', function (t) { }) test('.parse - strict:false error', function (t) { - var parser = ndj.parse({strict: false}) + const parser = ndj.parse({strict: false}) parser.once('data', function (data) { t.ok(data.json, 'parse second one') t.end() @@ -62,8 +62,8 @@ test('.parse - strict:false error', function (t) { } }) -test('.serialize', function(t) { - var serializer = ndj.serialize() +test('.stringify', function(t) { + const serializer = ndj.stringify() serializer.pipe(concat(function(data) { t.equal(data, '{"hello":"world"}' + os.EOL) t.end() @@ -72,13 +72,13 @@ test('.serialize', function(t) { serializer.end() }) -test('.serialize circular', function(t) { - var serializer = ndj.serialize() +test('.stringify circular', function(t) { + const serializer = ndj.stringify() serializer.pipe(concat(function(data) { t.equal(data, '{"obj":"[Circular ~]"}' + os.EOL) t.end() })) - var obj = {} + const obj = {} obj.obj = obj serializer.write(obj) serializer.end()