index.js

'use strict';

const Events = require('events');
const assert = require('assert');
const State = require('./lib/state');
const Token = require('./lib/token');
const Location = require('./lib/location');
const { Position } = Location;

/**
 * Create a new `Lexer` with the given `options`.
 *
 * ```js
 * const Lexer = require('snapdragon-lexer');
 * const lexer = new Lexer('foo/bar');
 * ```
 * @name Lexer
 * @param {string|Object} `input` (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
 * @param {object} `options`
 * @api public
 */

class Lexer extends Events {
  constructor(input, options = {}) {
    super();

    if (typeof input !== 'string') {
      options = input || {};
      input = '';
    }

    this.options = { ...options };
    this.handlers = new Map();
    this.types = new Set();
    this.state = new State(input);

    if (options.handlers) {
      for (const [type, handler] of options.handlers) {
        this.handler(type, handler);
      }
    }
  }

  /**
   * Returns true if we are still at the beginning-of-string, and
   * no part of the string has been consumed.
   *
   * @name .bos
   * @return {boolean}
   * @api public
   */

  bos() {
    return !this.state.consumed;
  }

  /**
   * Returns true if `lexer.string` and `lexer.queue` are empty.
   *
   * @name .eos
   * @return {boolean}
   * @api public
   */

  eos() {
    return this.state.string === '' && this.state.queue.length === 0;
  }

  /**
   * Register a handler function.
   *
   * ```js
   * lexer.set('star', function() {
   *   // do parser, lexer, or compiler stuff
   * });
   * ```
   * @name .set
   * @param {string} `type`
   * @param {function} `fn` The handler function to register.
   * @api public
   */

  set(type, handler = tok => tok) {
    this.types.add(type);
    const lexer = this;
    // can't do fat arrow here, we need to ensure that the handler
    // context is always correct whether handlers are called directly
    // or re-registered on a new instance, etc.
    this.handlers.set(type, function(...args) {
      let ctx = this || lexer;
      let loc = ctx.location();
      let tok = handler.call(ctx, ...args);
      if (tok && isObject(tok) && !Token.isToken(tok)) {
        tok = ctx.token(tok);
      }
      if (Token.isToken(tok) && !tok.type) {
        tok.type = type;
      }
      return Token.isToken(tok) ? loc(tok) : tok;
    });
    return this;
  }

  /**
   * Get a registered handler function.
   *
   * ```js
   * lexer.set('star', function() {
   *   // do lexer stuff
   * });
   * const star = lexer.get('star');
   * ```
   * @name .get
   * @param {string} `type`
   * @param {function} `fn` The handler function to register.
   * @api public
   */

  get(type) {
    let handler = this.handlers.get(type) || this.handlers.get('unknown');
    assert(handler, `expected handler "${type}" to be a function`);
    return handler;
  }

  /**
   * Returns true if the lexer has a registered handler of the given `type`.
   *
   * ```js
   * lexer.set('star', function() {});
   * console.log(lexer.has('star')); // true
   * ```
   * @name .has
   * @param {string} type
   * @return {boolean}
   * @api public
   */

  has(type) {
    return this.handlers.has(type);
  }

  /**
   * Create a new [Token][snapdragon-token] with the given `type` and `value`.
   *
   * ```js
   * console.log(lexer.token({type: 'star', value: '*'}));
   * console.log(lexer.token('star', '*'));
   * console.log(lexer.token('star'));
   * ```
   * @name .token
   * @emits token
   * @param {string|Object} `type` (required) The type of token to create
   * @param {string} `value` (optional) The captured string
   * @param {array} `match` (optional) Match results from `String.match()` or `RegExp.exec()`
   * @return {Object} Returns an instance of [snapdragon-token][]
   * @api public
   */

  token(type, value, match) {
    let token = new Token(type, value, match);
    this.emit('token', token);
    return token;
  }

  /**
   * Returns true if the given value is a [snapdragon-token][] instance.
   *
   * ```js
   * const Token = require('snapdragon-token');
   * lexer.isToken({}); // false
   * lexer.isToken(new Token({type: 'star', value: '*'})); // true
   * ```
   * @name .isToken
   * @param {object} `token`
   * @return {boolean}
   * @api public
   */

  isToken(token) {
    return Token.isToken(token);
  }

  /**
   * Consume the given length from `lexer.string`. The consumed value is used
   * to update `lexer.state.consumed`, as well as the current position.
   *
   * ```js
   * lexer.consume(1);
   * lexer.consume(1, '*');
   * ```
   * @name .consume
   * @param {number} `len`
   * @param {string} `value` Optionally pass the value being consumed.
   * @return {String} Returns the consumed value
   * @api public
   */

  consume(len, value = this.state.string.slice(0, len)) {
    this.state.consumed += value;
    this.state.string = this.state.string.slice(len);
    this.updateLocation(value, len);
    return value;
  }

  /**
   * Update column and line number based on `value`.
   *
   * @param {string} `value`
   * @return {undefined}
   */

  updateLocation(value, len = value.length) {
    let i = value.lastIndexOf('\n');
    this.state.loc.column = ~i ? len - i : this.state.loc.column + len;
    this.state.loc.line += Math.max(0, value.split('\n').length - 1);
    this.state.loc.index += len;
  }

  /**
   * Returns a function for updating a token with lexer
   * location information.
   *
   * @return {function}
   * @api public
   */

  location() {
    let start = new Position(this);

    return token => {
      let end = new Position(this);
      define(token, 'loc', new Location(start, end, this));
      return token;
    };
  }

  /**
   * Use the given `regex` to match a substring from `lexer.string`. Also validates
   * the regex to ensure that it starts with `^` since matching should always be
   * against the beginning of the string, and throws if the regex matches an empty
   * string, which can cause catastrophic backtracking.
   *
   * ```js
   * const lexer = new Lexer('foo/bar');
   * const match = lexer.match(/^\w+/);
   * console.log(match);
   * //=> [ 'foo', index: 0, input: 'foo/bar' ]
   * ```
   * @name .match
   * @param {regExp} `regex` (required)
   * @return {Array|null} Returns the match array from `RegExp.exec` or null.
   * @api public
   */

  match(regex) {
    assert(regex instanceof RegExp, 'expected a regular expression');

    if (regex.validated !== true) {
      assert(regex.source[0] === '^', 'expected regex to start with "^"');
      regex.validated = true;
    }

    let consumed = this.state.consumed;
    let match = regex.exec(this.state.string);
    if (!match) return null;

    if (match[0] === '') {
      throw new SyntaxError('regex should not match an empty string');
    }

    this.emit('match', match);
    define(match, 'consumed', consumed);
    this.consume(match[0].length, match[0]);
    return match;
  }

  /**
   * Scan for a matching substring by calling [.match()](#match)
   * with the given `regex`. If a match is found, 1) a token of the
   * specified `type` is created, 2) `match[0]` is used as `token.value`,
   * and 3) the length of `match[0]` is sliced from `lexer.string`
   * (by calling [.consume()](#consume)).
   *
   * ```js
   * lexer.string = '/foo/';
   * console.log(lexer.scan(/^\//, 'slash'));
   * //=> Token { type: 'slash', value: '/' }
   * console.log(lexer.scan(/^\w+/, 'text'));
   * //=> Token { type: 'text', value: 'foo' }
   * console.log(lexer.scan(/^\//, 'slash'));
   * //=> Token { type: 'slash', value: '/' }
   * ```
   * @name .scan
   * @emits scan
   * @param {string} `type`
   * @param {regExp} `regex`
   * @return {Object} Returns a token if a match is found, otherwise undefined.
   * @api public
   */

  scan(regex, type) {
    try {
      let match = this.match(regex);
      if (match) {
        let tok = this.token(type, match[0], match);
        this.emit('scan', tok);
        return tok;
      }
    } catch (err) {
      err.regex = regex;
      err.type = type;
      throw err;
    }
  }

  /**
   * Capture a token of the specified `type` using the provide `regex`
   * for scanning and matching substrings. Automatically registers a handler
   * when a function is passed as the last argument.
   *
   * ```js
   * lexer.capture('text', /^\w+/);
   * lexer.capture('text', /^\w+/, token => {
   *   if (token.value === 'foo') {
   *     // do stuff
   *   }
   *   return token;
   * });
   * ```
   * @name .capture
   * @param {string} `type` (required) The type of token being captured.
   * @param {regExp} `regex` (required) The regex for matching substrings.
   * @param {function} `fn` (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
   * @return {Object}
   * @api public
   */

  capture(type, regex, fn) {
    let handler = function() {
      let token = this.scan(regex, type);
      if (token) {
        return fn ? fn.call(this, token) : token;
      }
    };
    this.set(type, handler);
    return this;
  }

  /**
   * Calls handler `type` on `lexer.string`.
   *
   * ```js
   * const lexer = new Lexer('/a/b');
   * lexer.capture('slash', /^\//);
   * lexer.capture('text', /^\w+/);
   * console.log(lexer.handle('text'));
   * //=> undefined
   * console.log(lexer.handle('slash'));
   * //=> { type: 'slash', value: '/' }
   * console.log(lexer.handle('text'));
   * //=> { type: 'text', value: 'a' }
   * ```
   * @name .handle
   * @emits handle
   * @param {string} `type` The handler type to call on `lexer.string`
   * @return {Object} Returns a token of the given `type` or undefined.
   * @api public
   */

  handle(type) {
    let token = this.get(type).call(this);
    if (token) {
      this.current = token;
      this.emit('handle', token);
      return token;
    }
  }

  /**
   * Get the next token by iterating over `lexer.handlers` and
   * calling each handler on `lexer.string` until a handler returns
   * a token. If no handlers return a token, an error is thrown
   * with the substring that couldn't be lexed.
   *
   * ```js
   * const token = lexer.advance();
   * ```
   * @name .advance
   * @return {Object} Returns the first token returned by a handler, or the first character in the remaining string if `options.mode` is set to `character`.
   * @api public
   */

  advance() {
    if (this.eos()) return;
    if (this.options.mode === 'character') {
      return (this.current = this.consume(1));
    }
    for (let type of this.types) {
      let token = this.handle(type);
      if (token) {
        return token;
      }
    }
    this.fail();
  }

  /**
   * Tokenizes a string and returns an array of tokens.
   *
   * ```js
   * let lexer = new Lexer({ handlers: otherLexer.handlers })
   * lexer.capture('slash', /^\//);
   * lexer.capture('text', /^\w+/);
   * const tokens = lexer.lex('a/b/c');
   * console.log(tokens);
   * // Results in:
   * // [ Token { type: 'text', value: 'a' },
   * //   Token { type: 'slash', value: '/' },
   * //   Token { type: 'text', value: 'b' },
   * //   Token { type: 'slash', value: '/' },
   * //   Token { type: 'text', value: 'c' } ]
   * ```
   * @name .lex
   * @param {string} `input` The string to lex.
   * @return {Array} Returns an array of tokens.
   * @api public
   */

  lex(input, options) {
    if (options) this.options = { ...options };
    if (input) this.state = new State(input);
    while (this.push(this.next()));
    return this.state.tokens;
  }

  tokenize(...args) {
    return this.lex(...args);
  }

  /**
   * Push a token onto the `lexer.queue` array.
   *
   * ```js
   * console.log(lexer.queue.length); // 0
   * lexer.enqueue(new Token('star', '*'));
   * console.log(lexer.queue.length); // 1
   * ```
   * @name .enqueue
   * @param {object} `token`
   * @return {Object} Returns the given token with updated `token.index`.
   * @api public
   */

  enqueue(token) {
    token && this.state.queue.push(token);
    return token;
  }

  /**
   * Shift a token from `lexer.queue`.
   *
   * ```js
   * console.log(lexer.queue.length); // 1
   * lexer.dequeue();
   * console.log(lexer.queue.length); // 0
   * ```
   * @name .dequeue
   * @return {Object} Returns the given token with updated `token.index`.
   * @api public
   */

  dequeue() {
    return this.state.queue.length && this.state.queue.shift();
  }

  /**
   * Lookbehind `n` tokens.
   *
   * ```js
   * const token = lexer.lookbehind(2);
   * ```
   * @name .lookbehind
   * @param {number} `n`
   * @return {Object}
   * @api public
   */

  lookbehind(n) {
    assert(Number.isInteger(n), 'expected a positive integer');
    return this.state.tokens[this.state.tokens.length - n];
  }

  /**
   * Get the previously lexed token.
   *
   * ```js
   * const token = lexer.prev();
   * ```
   * @name .prev
   * @returns {Object|undefined} Returns a token or undefined.
   * @api public
   */

  prev() {
    return this.lookbehind(1);
  }

  /**
   * Lookahead `n` tokens and return the last token. Pushes any
   * intermediate tokens onto `lexer.tokens.` To lookahead a single
   * token, use [.peek()](#peek).
   *
   * ```js
   * const token = lexer.lookahead(2);
   * ```
   * @name .lookahead
   * @param {number} `n`
   * @return {Object}
   * @api public
   */

  lookahead(n) {
    assert(Number.isInteger(n), 'expected a positive integer');
    let fetch = n - this.state.queue.length;
    while (fetch-- > 0 && this.enqueue(this.advance()));
    return this.state.queue[--n];
  }

  /**
   * Lookahead a single token.
   *
   * ```js
   * const token = lexer.peek();
   * ```
   * @name .peek
   * @return {Object} Returns a token.
   * @api public
   */

  peek() {
    return this.lookahead(1);
  }

  /**
   * Get the next token, either from the `queue` or by [advancing](#advance).
   *
   * ```js
   * const token = lexer.next();
   * ```
   * @name .next
   * @returns {Object|String} Returns a token, or (when `options.mode` is set to `character`) either gets the next character from `lexer.queue`, or consumes the next charcter in the string.
   * @api public
   */

  next() {
    return this.dequeue() || this.advance();
  }

  /**
   * Skip `n` tokens or characters in the string. Skipped values are not enqueued.
   *
   * ```js
   * const token = lexer.skip(1);
   * ```
   * @name .skip
   * @param {number} `n`
   * @returns {Object} returns an array of skipped tokens.
   * @api public
   */

  skip(n) {
    assert.equal(typeof n, 'number', 'expected a number');
    return this.skipWhile(() => n-- > 0);
  }

  /**
   * Skip tokens while the given `fn` returns true.
   *
   * ```js
   * lexer.skipWhile(tok => tok.type !== 'space');
   * ```
   * @name .skipWhile
   * @param {function} `fn` Return true if a token should be skipped.
   * @returns {Array} Returns an array if skipped tokens.
   * @api public
   */

  skipWhile(fn = !this.eos()) {
    const skipped = [];
    while (fn.call(this, this.peek())) skipped.push(this.next());
    return skipped;
  }

  /**
   * Skip the given token `types`.
   *
   * ```js
   * lexer.skipWhile(tok => tok.type !== 'space');
   * ```
   * @name .skipType
   * @param {string|Array} `types` One or more token types to skip.
   * @returns {Array} Returns an array if skipped tokens.
   * @api public
   */

  skipTo(type) {
    return this.skipWhile(tok => tok && tok.type !== type).concat(this.next());
  }

  /**
   * Skip the given token `types`.
   *
   * ```js
   * lexer.skipType('space');
   * lexer.skipType(['newline', 'space']);
   * ```
   * @name .skipType
   * @param {string|Array} `types` One or more token types to skip.
   * @returns {Array} Returns an array if skipped tokens
   * @api public
   */

  skipType(types) {
    return this.skipWhile(tok => [].concat(types).includes(tok.type));
  }

  /**
   * Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push
   * `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting
   * `lexer.options.append` or `token.append` to `false`.
   *
   * ```js
   * console.log(lexer.tokens.length); // 0
   * lexer.push(new Token('star', '*'));
   * console.log(lexer.tokens.length); // 1
   * console.log(lexer.stash) // ['*']
   * ```
   * @name .push
   * @emits push
   * @param {object|String} `token`
   * @return {Object} Returns the given `token`.
   * @api public
   */

  push(token) {
    if (!token && token !== '') return;
    if (this.options.mode !== 'character') {
      assert(this.isToken(token), 'expected token to be an instance of Token');
    }

    this.emit('push', token);
    this.state.tokens.push(token);

    if (this.options.stash === false || token.stash === false) {
      return token;
    }

    if (this.options.mode === 'character') {
      this.append(token);
    } else {
      this.append(token.value);
    }
    return token;
  }

  /**
   * Append a string to the last element on `lexer.stash`, or push the
   * string onto the stash if no elements exist.
   *
   * ```js
   * const stack = new Stack();
   * stack.push('a');
   * stack.push('b');
   * stack.push('c');
   * stack.append('_foo');
   * stack.append('_bar');
   * console.log(stack);
   * //=> Stack ['a', 'b', 'c_foo_bar']
   * ```
   * @name .append
   * @param {String} `value`
   * @return {String} Returns the last value in the array.
   * @api public
   */

  append(value) {
    if (typeof value !== 'string') return;
    let n = this.state.stash.length - 1;
    if (this.state.stash[n] === '') {
      this.state.stash[n] += value;
    } else {
      this.state.stash.push(value);
    }
    this.emit('append', value);
    return this;
  }

  /**
   * Returns true if a token with the given `type` is on the stack.
   *
   * ```js
   * if (lexer.isInside('bracket') || lexer.isInside('brace')) {
   *   // do stuff
   * }
   * ```
   * @name .isInside
   * @param {string} `type` The type to check for.
   * @return {boolean}
   * @api public
   */

  isInside(type) {
    return this.state.stack.some(tok => tok.type === type);
  }

  /**
   * Throw a formatted error message with details including the cursor position.
   *
   * ```js
   * lexer.set('foo', function(tok) {
   *   if (tok.value !== 'foo') {
   *     throw this.state.error('expected token.value to be "foo"', tok);
   *   }
   * });
   * ```
   * @name .error
   * @param {string} `msg` Message to use in the Error.
   * @param {object} `node`
   * @return {undefined}
   * @api public
   */

  error(err) {
    if (typeof err === 'string') err = new Error(err);
    if (this.listenerCount('error') > 0) {
      this.emit('error', err);
    } else {
      throw err;
    }
  }

  /**
   * Throw an error if `lexer.stack` is not empty, or when the remaining string
   * cannot be lexed by the currently registered handlers.
   * @api private
   */

  fail() {
    let token = this.state.stack.pop();
    if (token) {
      const match = token && token.match;
      const value = match ? match[0] : token[this.options.value || 'value'];
      throw new Error(`unclosed: "${value}"`);
    }
    if (this.state.string) {
      throw new Error(`unmatched input: "${this.state.string.slice(0, 10)}"`);
    }
  }

  /**
   * Call a plugin function on the lexer instance.
   *
   * ```js
   * lexer.use(function(lexer) {
   *   // do stuff to lexer
   * });
   * ```
   * @name .use
   * @param {function} `fn`
   * @return {object} Returns the lexer instance.
   * @api public
   */

  use(fn) {
    fn.call(this, this);
    return this;
  }

  /**
   * Static method that returns true if the given value is an
   * instance of `snapdragon-lexer`.
   *
   * ```js
   * const Lexer = require('snapdragon-lexer');
   * const lexer = new Lexer();
   * console.log(Lexer.isLexer(lexer)); //=> true
   * console.log(Lexer.isLexer({})); //=> false
   * ```
   * @name Lexer#isLexer
   * @param {object} `lexer`
   * @returns {Boolean}
   * @api public
   * @static
   */

  static isLexer(lexer) {
    return lexer instanceof Lexer;
  }

  /**
   * Static method that returns true if the given value is an
   * instance of `snapdragon-token`. This is a proxy to `Token#isToken`.
   *
   * ```js
   * const Token = require('snapdragon-token');
   * const Lexer = require('snapdragon-lexer');
   * console.log(Lexer.isToken(new Token({type: 'foo'}))); //=> true
   * console.log(Lexer.isToken({})); //=> false
   * ```
   * @name Lexer#isToken
   * @param {object} `lexer`
   * @returns {Boolean}
   * @api public
   * @static
   */

  static isToken(token) {
    return token instanceof Token;
  }

  /**
   * The State class, exposed as a static property.
   * @name Lexer#State
   * @api public
   * @static
   */

  static get State() {
    return State;
  }

  /**
   * The Token class, exposed as a static property.
   * @name Lexer#Token
   * @api public
   * @static
   */

  static get Token() {
    return Token;
  }
}

/**
 * Returns true if value is an object
 */

function isObject(val) {
  return val && typeof val === 'object' && !Array.isArray(val);
}

/**
 * Define a non-enumerable property on `obj`
 */

function define(obj, key, value) {
  Reflect.defineProperty(obj, key, { value });
}

/**
 * Expose `Lexer`
 * @type {Class}
 */

module.exports = Lexer;