diff --git a/.jshintrc b/.jshintrc index be2e55d..7131937 100644 --- a/.jshintrc +++ b/.jshintrc @@ -1,6 +1,6 @@ { "asi": true, - "esnext": true, + "esversion": 6, "eqeqeq": true, "camelcase": true, "funcscope": true, diff --git a/config/packer.js b/config/packer.js new file mode 100644 index 0000000..2c0f0c1 --- /dev/null +++ b/config/packer.js @@ -0,0 +1,154 @@ +/** + * Packer options + * + * converters: + * extensions: file extensions which packer should find + * + * pack: list of output formats + * format: output format + * title: title for console output + * force: force reencoding if source and output formats are same + * + * rules: + * encoders: + * dictionary with encode rules + * key: output format + * value: encoder rule + * + * encoder: + * which encoder to use. + * Available encoders: ffmpeg, sox, qac, afconvert + * + * options: + * options to pass to the encoder. + * + * inputFormats: + * supported input formats. If presented, any unsupported format should be decoded to wav. + * + * decoders: + * dictionary with decode rules + * key: input format + * value: decoder rule + * + * decoder: + * which decoder to use + * Available decoders: ffmpeg, sox, xa + * + * force: + * always decode specified format + */ + +module.exports = { + audio: { + extensions: ['mp3', 'ogg', 'wav', 'xa'], + pack: [ + { + format: 'ogg', + title: 'better audio performance', + force: true, + }, + { + format: 'm4a', + title: 'for iOS and Safari', + }, + ], + rules: { + encoders: { + // encode ogg with sox + + ogg: { + encoder: 'sox', + options: [ + '-t', 'ogg', + '-C', '3', + ], + }, + + // encode ogg with ffmpeg + + // ogg: { + // encoder: 'ffmpeg', + // options: [ + // '-q:a', '6', + // '-c:a', 'libvorbis', + // '-f', 'ogg', + // ], + // }, + + // encode m4a with afconvert + + // m4a: { + // inputFormats: ['wav'], + // encoder: 'afconvert', + // options: [ + // '-f', 'm4af', + // '-b', '128000', + // '-q', '127', + // '-s', '2', + // ], + // }, + + // encode m4a with qaac + + m4a: { + inputFormats: ['wav'], + encoder: 'qaac', + options: ['-c', '128'], + }, + + // encode m4a with ffmpeg + + // m4a: { + // encoder: 'ffmpeg', + // options: [ + // '-b:a', '192k', + // '-c:a', 'aac', + // '-movflags', 'frag_keyframe+empty_moov', + // '-f', 'ipod', + // '-vn', + // ], + // }, + }, + decoders: { + // decode ogg with sox + + ogg: { + decoder: 'sox', + options: [ + '-t', 'wav', + ], + }, + + // decode ogg with ffmpeg + + // ogg: { + // decoder: 'ffmpeg', + // options: ['-f', 'wav'], + // }, + + // decode mp3 with sox + + mp3: { + decoder: 'sox', + options: [ + '-t', 'wav', + ], + }, + + // decode mp3 with ffmpeg + + // mp3: { + // decoder: 'ffmpeg', + // options: ['-f', 'wav'], + // }, + + // decode xa with xa + + xa: { + decoder: 'xa', + force: true, + }, + }, + }, + } +} diff --git a/package.json b/package.json index ed35af8..f74673d 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "homepage": "https://github.com/bemusic/bemuse-tools", "dependencies": { "babel": "^5.4.3", - "bemuse-indexer": "^3.1.0", + "bemuse-indexer": "drummaniac/bemuse-indexer#dtx-support", "bluebird": "^2.9.9", "bytes": "^1.0.0", "chalk": "^1.0.0", @@ -30,6 +30,7 @@ "cors": "^2.7.1", "endpoint": "^0.4.2", "express": "^4.12.4", + "fluent-ffmpeg": "^2.1.2", "format-json": "^1.0.3", "glob": "^4.3.5", "gulp-util": "^3.0.1", @@ -40,7 +41,8 @@ "mkdirp": "^0.5.0", "rx": "^2.5.3", "temp": "^0.8.1", - "throat": "^1.0.0" + "throat": "^1.0.0", + "xa-dtx": "^0.0.4" }, "devDependencies": { "chai": "^1.10.0", diff --git a/src/audio.js b/src/audio.js index 377d1ba..627a3ec 100644 --- a/src/audio.js +++ b/src/audio.js @@ -1,103 +1,94 @@ - -import Promise from 'bluebird' -import co from 'co' -import fs from 'fs' -import Throat from 'throat' -import { cpus } from 'os' -import endpoint from 'endpoint' -import { spawn, execFile as _execFile } from 'child_process' +import co from 'co' import { extname, basename } from 'path' - -import tmp from './temporary' - -let readFile = Promise.promisify(fs.readFile, fs) -let writeFile = Promise.promisify(fs.writeFile, fs) -let execFile = Promise.promisify(_execFile) +import { cpus } from 'os' +import createConverter from './converters' +import Throat from 'throat' let throat = new Throat(cpus().length || 1) export class AudioConvertor { - constructor(type, ...extra) { - this._target = type - this._extra = extra + constructor(options, rules) { + this._target = options.format + this._force = options.force + this._rules = rules } + convert(file) { - let ext = extname(file.name).toLowerCase() - if (ext === '.' + this._target && !this.force) { - return Promise.resolve(file) - } else { + return co(function*() { + let ext = extname(file.name) let name = basename(file.name, ext) + '.' + this._target - return this._doConvert(file.path, this._target) - .then(buffer => file.derive(name, buffer)) - } - } - _doConvert(path, type) { - if (type === 'm4a') { - return co(function*() { - let wav = yield this._SoX(path, 'wav') - let prefix = tmp() - let wavPath = prefix + '.wav' - let m4aPath = prefix + '.m4a' - yield writeFile(wavPath, wav) - if (process.platform.match(/^win/)) { - yield execFile('qaac', ['-o', m4aPath, '-c', '128', wavPath]) - } else { - yield execFile('afconvert', [wavPath, m4aPath, '-f', 'm4af', - '-b', '128000', '-q', '127', '-s', '2']) - } - return yield readFile(m4aPath) - }.bind(this)) - } else { - return this._SoX(path, type) - } + let inputFormat = yield this.guessFormat(file) + + // return original file if format did not changed and encoding not forced + if (inputFormat === this._target && !this._force) { + return yield file.derive(name) + } + + // get encode rule for target format + let encodeRule = this._rules.encoders[this._target] + if (!encodeRule) { + return Promise.reject(new Error('Encode rule not found')) + } + + // get decodeRule for source format + let decodeRule = this._rules.decoders[inputFormat] || + this._rules.decoders.default + + let buffer + // decode if encoder does not support source format or source should be force decoded + if (decodeRule.force || + encodeRule.inputFormats && + encodeRule.inputFormats.indexOf(inputFormat) === -1) { + buffer = yield this.decode(file.buffer, inputFormat, decodeRule) + inputFormat = 'wav' + } else { + buffer = file.buffer + } + + // encode buffer to target format + buffer = yield this.encode(buffer, inputFormat, encodeRule) + + return yield Promise.resolve(file.derive(name, buffer)) + }.bind(this)) } - _SoX(path, type) { - return co(function*() { - let typeArgs = [ ] - try { - let fd = yield Promise.promisify(fs.open, fs)(path, 'r') - let buffer = new Buffer(4) - let read = yield Promise.promisify(fs.read, fs)(fd, buffer, 0, 4, null) - yield Promise.promisify(fs.close, fs)(fd) - if (read === 0) { - console.error('[WARN] Empty keysound file.') - } else if (buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) { - typeArgs = [ '-t', 'mp3' ] - } else if (buffer[0] === 0xFF && buffer[1] === 0xFB) { - typeArgs = [ '-t', 'mp3' ] - } else if (buffer[0] === 0x4F && buffer[1] === 0x67 && buffer[2] === 0x67 && buffer[3] === 0x53) { - typeArgs = [ '-t', 'ogg' ] - } - } catch (e) { - console.error('[WARN] Unable to detect file type!') + + guessFormat(file) { + return new Promise((resolve, reject) => { + let buffer = file.buffer + + if (buffer.length < 4) { + return reject(new Error('Empty keysound file')) + } + + if (buffer[0] === 0x49 && buffer[1] === 0x44 && buffer[2] === 0x33) { + resolve('mp3') + } else if (buffer[0] === 0xFF && buffer[1] === 0xFB) { + resolve('mp3') + } else if (buffer[0] === 0x4F && buffer[1] === 0x67 && buffer[2] === 0x67 && buffer[3] === 0x53) { + resolve('ogg') + } else { + let ext = extname(file.name).substr(1).toLowerCase() + resolve(ext) } - return yield this._doSoX(path, type, typeArgs) - }.bind(this)); + }) } - _doSoX(path, type, inputTypeArgs) { - return throat(() => new Promise((resolve, reject) => { - let sox = spawn('sox', [...inputTypeArgs, path, '-t', type, ...this._extra, '-']) - sox.stdin.end() - sox.stderr.on('data', x => process.stderr.write(x)) - let data = new Promise((resolve, reject) => { - sox.stdout.pipe(endpoint((err, buffer) => { - if (err) { - console.error('Error reading audio!') - reject(err) - } else { - resolve(buffer) - } - })) - }) - sox.on('close', (code) => { - if (code === 0) { - resolve(data) - } else { - console.error('Unable to convert audio file -- SoX exited ' + code) - reject(new Error('SoX process exited: ' + code)) - } - }) - })) + + decode(buffer, inputFormat, rule) { + let decoder = createConverter(rule.decoder, rule.options) + if (!decoder) { + return Promise.reject(new Error(`Decoder ${rule.decoder} not found`)) + } + + return throat(() => decoder.convert(buffer, inputFormat, rule.options)) + } + + encode(buffer, inputFormat, rule) { + let encoder = createConverter(rule.encoder, rule.options) + if (!encoder) { + return Promise.reject(new Error(`Encoder ${rule.decoder} not found`)) + } + + return throat(() => encoder.convert(buffer, inputFormat, rule.options)) } } diff --git a/src/bufferstream.js b/src/bufferstream.js new file mode 100644 index 0000000..452ea72 --- /dev/null +++ b/src/bufferstream.js @@ -0,0 +1,42 @@ +import stream from 'stream' + +export class ReadableBufferStream extends stream.PassThrough { + constructor(buffer) { + super() + + this.pause() + this.end(buffer) + } +} + +export class WritableBufferStream extends stream.Writable { + constructor() { + super() + + this._buffer = Buffer.alloc(1024 * 64) + this._bufferPos = 0 + } + + _write(chunk, encoding, callback) { + let size = chunk.length + + let newBufferSize = this._buffer.length + while (size + this._bufferPos > newBufferSize) { + newBufferSize *= 2 + } + if (newBufferSize > this._buffer.length) { + let newBuffer = Buffer.alloc(newBufferSize) + this._buffer.copy(newBuffer, 0, 0, this._buffer.length) + this._buffer = newBuffer + } + + chunk.copy(this._buffer, this._bufferPos, 0) + this._bufferPos += size + + callback() + } + + get buffer() { + return this._buffer.slice(0, this._bufferPos) + } +} diff --git a/src/converters/afconvert.js b/src/converters/afconvert.js new file mode 100644 index 0000000..6b3c6f1 --- /dev/null +++ b/src/converters/afconvert.js @@ -0,0 +1,30 @@ +import fs from 'fs' +import Promise from 'bluebird' +import { execFile } from 'child_process' +import co from 'co' +import BaseConverter from './base' +import tmp from '../temporary' + +let readFile = Promise.promisify(fs.readFile, fs) +let writeFile = Promise.promisify(fs.readFile, fs) + +export class AfconvertConverter extends BaseConverter { + convert(input, inputFormat, options) { + if (inputFormat !== 'wav') { + return Promise.reject( + new Error('Trying to convert non-wav format with qaac converter') + ) + } + + return co(function*() { + let wavPath = tmp() + let m4aPath = tmp() + + yield writeFile(wavPath, input) + yield execFile('afconvert', [wavPath, m4aPath, ...options]) + return yield readFile(m4aPath) + }.bind(this)) + } +} + +export default AfconvertConverter diff --git a/src/converters/base.js b/src/converters/base.js new file mode 100644 index 0000000..27579c7 --- /dev/null +++ b/src/converters/base.js @@ -0,0 +1,11 @@ +export class BaseConverter { + constructor(options) { + this._options = options + } + + convert(input, inputFormat, options) { + return new Promise.reject(new Error('Not implemented')) + } +} + +export default BaseConverter diff --git a/src/converters/ffmpeg.js b/src/converters/ffmpeg.js new file mode 100644 index 0000000..67bdf90 --- /dev/null +++ b/src/converters/ffmpeg.js @@ -0,0 +1,34 @@ +import ffmpeg from 'fluent-ffmpeg' +import BaseConverter from './base' +import {ReadableBufferStream, WritableBufferStream} from '../bufferstream' + +export class FFMpegConverter extends BaseConverter { + convert(input) { + return new Promise((resolve, reject) => { + let readStream + + // init input stream + if (typeof input === 'string') { + readStream = fs.createReadStream(input) + } else { + readStream = new ReadableBufferStream(input) + } + + let writeStream = new WritableBufferStream() + + // do ffmpeg + ffmpeg(readStream) + .output(writeStream) + .outputOptions(this._options) + .on('end', () => { + resolve(writeStream.buffer) + }) + .on('error', (err) => { + reject(new Error('ffmpeg process exited: ' + err)) + }) + .run() + }) + } +} + +export default FFMpegConverter diff --git a/src/converters/index.js b/src/converters/index.js new file mode 100644 index 0000000..7e97dd1 --- /dev/null +++ b/src/converters/index.js @@ -0,0 +1,19 @@ +import FFMpegConverter from './ffmpeg' +import XAConverter from './xa' +import SoXConverter from './sox' +import QaacConverter from './qaac' +import AfconvertConverter from './afconvert' + +export function createConverter(format, options) { + return new Converters[format](options) +} + +let Converters = { + ffmpeg: FFMpegConverter, + xa: XAConverter, + sox: SoXConverter, + qaac: QaacConverter, + afconvert: AfconvertConverter, +} + +export default createConverter diff --git a/src/converters/qaac.js b/src/converters/qaac.js new file mode 100644 index 0000000..9e53885 --- /dev/null +++ b/src/converters/qaac.js @@ -0,0 +1,38 @@ +import fs from 'fs' +import Promise from "bluebird" +import { spawn } from 'child_process' +import BaseConverter from './base' +import tmp from '../temporary' + +let readFile = Promise.promisify(fs.readFile, fs) + +export class QaacConverter extends BaseConverter { + convert(input, inputFormat, options) { + if (inputFormat !== 'wav') { + return Promise.reject( + new Error('Trying to convert non-wav format with qaac converter') + ) + } + + return new Promise((resolve, reject) => { + let tmpFile = tmp() + + let qaac = spawn('vendor/bin/qaac', ['-o', tmpFile, ...options, '-']) + qaac.stdin.write(input) + qaac.stdin.end() + qaac.on('close', (code) => { + if (code === 0) { + readFile(tmpFile).then(buffer => { + resolve(buffer) + fs.writeFileSync('tmp.m4a', buffer) + }) + } else { + reject(new Error('SoX process exited: ' + code)) + } + }) + }) + + } +} + +export default QaacConverter diff --git a/src/converters/sox.js b/src/converters/sox.js new file mode 100644 index 0000000..34f08df --- /dev/null +++ b/src/converters/sox.js @@ -0,0 +1,32 @@ +import { spawn } from 'child_process' +import BaseConverter from './base' +import endpoint from 'endpoint' + +export class SoXConverter extends BaseConverter { + convert(input, inputFormat, options) { + return new Promise((resolve, reject) => { + let sox = spawn('vendor/bin/sox', ['-t', inputFormat, '-', ...options, '-']) + sox.stdin.write(input) + sox.stdin.end() + sox.stderr.on('data', x => process.stderr.write(x)) + let data = new Promise((resolve, reject) => { + sox.stdout.pipe(endpoint((err, buffer) => { + if (err) { + reject(new Error('Error reading audio!')) + } else { + resolve(buffer) + } + })) + }) + sox.on('close', (code) => { + if (code === 0) { + resolve(data) + } else { + reject(new Error('SoX process exited: ' + code)) + } + }) + }) + } +} + +export default SoXConverter diff --git a/src/converters/xa.js b/src/converters/xa.js new file mode 100644 index 0000000..5e2a405 --- /dev/null +++ b/src/converters/xa.js @@ -0,0 +1,15 @@ +import BaseConverter from './base' +import convert from 'xa-dtx' + +export class XAConverter extends BaseConverter { + convert(input, inputFormat) { + if (inputFormat !== 'xa') { + return Promise.reject( + new Error('Trying to convert non-xa format with XA converter') + ) + } + return convert(input).then(wav => Promise.resolve(wav.buffer)) + } +} + +export default XAConverter diff --git a/src/directory.js b/src/directory.js index f43d41c..b40cc11 100644 --- a/src/directory.js +++ b/src/directory.js @@ -11,7 +11,7 @@ export class Directory { this._path = path } files(pattern) { - return glob(pattern, { cwd: this._path }) + return glob(pattern, { cwd: this._path, nocase: true }) .map(name => readFile(path.join(this._path, name)).then(buffer => new FileEntry(this, name, buffer))) } diff --git a/src/indexer.js b/src/indexer.js index 18d7e34..7db79a4 100644 --- a/src/indexer.js +++ b/src/indexer.js @@ -61,7 +61,7 @@ export function index(path, { recursive }) { console.log('-> Scanning files...') let dirs = new Map() - let pattern = (recursive ? '**/' : '') + '*/*.{bms,bme,bml,bmson}' + let pattern = (recursive ? '**/' : '') + '*/*.{bms,bme,bml,bmson,dtx}' for (var name of yield glob(pattern, { cwd: path })) { let bmsPath = join(path, name) put(dirs, dirname(bmsPath), () => []).push(basename(bmsPath)) diff --git a/src/packer.js b/src/packer.js index 7d51ca5..5914c12 100644 --- a/src/packer.js +++ b/src/packer.js @@ -10,6 +10,8 @@ import AudioConvertor from './audio' import Directory from './directory' import BemusePacker from './bemuse-packer' +import config from '../config/packer' + let mkdirp = Promise.promisify(require('mkdirp')) let fileStat = Promise.promisify(fs.stat, fs) @@ -23,25 +25,21 @@ export function packIntoBemuse(path) { let packer = new BemusePacker(directory) console.log('-> Loading audios') - let audio = yield directory.files('*.{mp3,wav,ogg}') - - console.log('-> Converting audio to ogg [better audio performance]') - let oggc = new AudioConvertor('ogg', '-C', '3') - oggc.force = true - let oggs = yield dotMap(audio, file => oggc.convert(file)) + let extensions = config.audio.extensions.join(',') + let audio = yield directory.files('**/*.{' + extensions + '}') - console.log('-> Converting audio to m4a [for iOS and Safari]') - let m4ac = new AudioConvertor('m4a') - let m4as = yield dotMap(audio, file => m4ac.convert(file)) - - packer.pack('m4a', m4as) - packer.pack('ogg', oggs) + for (let i in config.audio.pack) { + let options = config.audio.pack[i] + console.log(`-> Converting audio to ${options.format} [${options.title}]`) + let converter = new AudioConvertor(options, config.audio.rules) + let audios = yield dotMap(audio, file => converter.convert(file)) + packer.pack(options.format, audios) + } console.log('-> Writing...') let out = join(path, 'assets') yield mkdirp(out) yield packer.write(out) - }) }