lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/coffee/lexer.js in gollum-3.1.2 vs lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/coffee/lexer.js in gollum-3.1.3

- old
+ new

@@ -1,7 +1,7 @@ /** - * Copyright (c) 2009-2012 Jeremy Ashkenas + * Copyright (c) 2009-2013 Jeremy Ashkenas * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, @@ -19,46 +19,47 @@ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. + * */ define(function(require, exports, module) { -// Generated by CoffeeScript 1.3.3 +// Generated by CoffeeScript 1.6.3 - var BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, INVERSES, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, STRICT_PROSCRIBED, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, key, last, starts, _ref, _ref1, + var BOM, BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, INVERSES, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, STRICT_PROSCRIBED, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, invertLiterate, key, last, locationDataToString, repeat, starts, throwSyntaxError, _ref, _ref1, __indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; }; _ref = require('./rewriter'), Rewriter = _ref.Rewriter, INVERSES = _ref.INVERSES; - _ref1 = require('./helpers'), count = _ref1.count, starts = _ref1.starts, compact = _ref1.compact, last = _ref1.last; + _ref1 = require('./helpers'), count = _ref1.count, starts = _ref1.starts, compact = _ref1.compact, last = _ref1.last, repeat = _ref1.repeat, invertLiterate = _ref1.invertLiterate, locationDataToString = _ref1.locationDataToString, throwSyntaxError = _ref1.throwSyntaxError; exports.Lexer = Lexer = (function() { - function Lexer() {} Lexer.prototype.tokenize = function(code, opts) { - var i, tag; + var consumed, i, tag, _ref2; if (opts == null) { opts = {}; } - if (WHITESPACE.test(code)) { - code = "\n" + code; - } - code = code.replace(/\r/g, '').replace(TRAILING_SPACES, ''); - this.code = code; - this.line = opts.line || 0; + this.literate = opts.literate; this.indent = 0; + this.baseIndent = 0; this.indebt = 0; this.outdebt = 0; this.indents = []; this.ends = []; this.tokens = []; + this.chunkLine = opts.line || 0; + this.chunkColumn = opts.column || 0; + code = this.clean(code); i = 0; while (this.chunk = code.slice(i)) { - i += this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken(); + consumed = this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken(); + _ref2 = this.getLineAndColumnFromChunk(consumed), this.chunkLine = _ref2[0], this.chunkColumn = _ref2[1]; + i += consumed; } this.closeIndentation(); if (tag = this.ends.pop()) { this.error("missing " + tag); } @@ -66,21 +67,38 @@ return this.tokens; } return (new Rewriter).rewrite(this.tokens); }; + Lexer.prototype.clean = function(code) { + if (code.charCodeAt(0) === BOM) { + code = code.slice(1); + } + code = code.replace(/\r/g, '').replace(TRAILING_SPACES, ''); + if (WHITESPACE.test(code)) { + code = "\n" + code; + this.chunkLine--; + } + if (this.literate) { + code = invertLiterate(code); + } + return code; + }; + Lexer.prototype.identifierToken = function() { - var colon, forcedIdentifier, id, input, match, prev, tag, _ref2, _ref3; + var colon, colonOffset, forcedIdentifier, id, idLength, input, match, poppedToken, prev, tag, tagToken, _ref2, _ref3, _ref4; if (!(match = IDENTIFIER.exec(this.chunk))) { return 0; } input = match[0], id = match[1], colon = match[2]; + idLength = id.length; + poppedToken = void 0; if (id === 'own' && this.tag() === 'FOR') { this.token('OWN', id); return id.length; } - forcedIdentifier = colon || (prev = last(this.tokens)) && (((_ref2 = prev[0]) === '.' || _ref2 === '?.' || _ref2 === '::') || !prev.spaced && prev[0] === '@'); + forcedIdentifier = colon || (prev = last(this.tokens)) && (((_ref2 = prev[0]) === '.' || _ref2 === '?.' || _ref2 === '::' || _ref2 === '?::') || !prev.spaced && prev[0] === '@'); tag = 'IDENTIFIER'; if (!forcedIdentifier && (__indexOf.call(JS_KEYWORDS, id) >= 0 || __indexOf.call(COFFEE_KEYWORDS, id) >= 0)) { tag = id.toUpperCase(); if (tag === 'WHEN' && (_ref3 = this.tag(), __indexOf.call(LINE_BREAK, _ref3) >= 0)) { tag = 'LEADING_WHEN'; @@ -95,11 +113,11 @@ tag = 'FOR' + tag; this.seenFor = false; } else { tag = 'RELATION'; if (this.value() === '!') { - this.tokens.pop(); + poppedToken = this.tokens.pop(); id = '!' + id; } } } } @@ -135,13 +153,17 @@ default: return tag; } })(); } - this.token(tag, id); + tagToken = this.token(tag, id, 0, idLength); + if (poppedToken) { + _ref4 = [poppedToken[2].first_line, poppedToken[2].first_column], tagToken[2].first_line = _ref4[0], tagToken[2].first_column = _ref4[1]; + } if (colon) { - this.token(':', ':'); + colonOffset = input.lastIndexOf(':'); + this.token(':', ':', colonOffset, colon.length); } return input.length; }; Lexer.prototype.numberToken = function() { @@ -159,45 +181,43 @@ } else if (/^0\d+/.test(number)) { this.error("octal literal '" + number + "' must be prefixed with '0o'"); } lexedLength = number.length; if (octalLiteral = /^0o([0-7]+)/.exec(number)) { - number = '0x' + (parseInt(octalLiteral[1], 8)).toString(16); + number = '0x' + parseInt(octalLiteral[1], 8).toString(16); } if (binaryLiteral = /^0b([01]+)/.exec(number)) { - number = '0x' + (parseInt(binaryLiteral[1], 2)).toString(16); + number = '0x' + parseInt(binaryLiteral[1], 2).toString(16); } - this.token('NUMBER', number); + this.token('NUMBER', number, 0, lexedLength); return lexedLength; }; Lexer.prototype.stringToken = function() { - var match, octalEsc, string; - switch (this.chunk.charAt(0)) { + var octalEsc, quote, string, trimmed; + switch (quote = this.chunk.charAt(0)) { case "'": - if (!(match = SIMPLESTR.exec(this.chunk))) { - return 0; - } - this.token('STRING', (string = match[0]).replace(MULTILINER, '\\\n')); + string = SIMPLESTR.exec(this.chunk)[0]; break; case '"': - if (!(string = this.balancedString(this.chunk, '"'))) { - return 0; - } - if (0 < string.indexOf('#{', 1)) { - this.interpolateString(string.slice(1, -1)); - } else { - this.token('STRING', this.escapeLines(string)); - } - break; - default: - return 0; + string = this.balancedString(this.chunk, '"'); } + if (!string) { + return 0; + } + trimmed = this.removeNewlines(string.slice(1, -1)); + if (quote === '"' && 0 < string.indexOf('#{', 1)) { + this.interpolateString(trimmed, { + strOffset: 1, + lexedLength: string.length + }); + } else { + this.token('STRING', quote + this.escapeLines(trimmed) + quote, 0, string.length); + } if (octalEsc = /^(?:\\.|[^\\])*\\(?:0[0-7]|[1-7])/.test(string)) { this.error("octal escape sequences " + string + " are not allowed"); } - this.line += count(string, '\n'); return string.length; }; Lexer.prototype.heredocToken = function() { var doc, heredoc, match, quote; @@ -210,16 +230,17 @@ quote: quote, indent: null }); if (quote === '"' && 0 <= doc.indexOf('#{')) { this.interpolateString(doc, { - heredoc: true + heredoc: true, + strOffset: 3, + lexedLength: heredoc.length }); } else { - this.token('STRING', this.makeString(doc, quote, true)); + this.token('STRING', this.makeString(doc, quote, true), 0, heredoc.length); } - this.line += count(heredoc, '\n'); return heredoc.length; }; Lexer.prototype.commentToken = function() { var comment, here, match; @@ -228,35 +249,32 @@ } comment = match[0], here = match[1]; if (here) { this.token('HERECOMMENT', this.sanitizeHeredoc(here, { herecomment: true, - indent: Array(this.indent + 1).join(' ') - })); + indent: repeat(' ', this.indent) + }), 0, comment.length); } - this.line += count(comment, '\n'); return comment.length; }; Lexer.prototype.jsToken = function() { var match, script; if (!(this.chunk.charAt(0) === '`' && (match = JSTOKEN.exec(this.chunk)))) { return 0; } - this.token('JS', (script = match[0]).slice(1, -1)); - this.line += count(script, '\n'); + this.token('JS', (script = match[0]).slice(1, -1), 0, script.length); return script.length; }; Lexer.prototype.regexToken = function() { var flags, length, match, prev, regex, _ref2, _ref3; if (this.chunk.charAt(0) !== '/') { return 0; } if (match = HEREGEX.exec(this.chunk)) { length = this.heregexToken(match); - this.line += count(match[0], '\n'); return length; } prev = last(this.tokens); if (prev && (_ref2 = prev[0], __indexOf.call((prev.spaced ? NOT_REGEX : NOT_SPACED_REGEX), _ref2) >= 0)) { return 0; @@ -269,94 +287,110 @@ this.error('regular expressions cannot begin with `*`'); } if (regex === '//') { regex = '/(?:)/'; } - this.token('REGEX', "" + regex + flags); + this.token('REGEX', "" + regex + flags, 0, match.length); return match.length; }; Lexer.prototype.heregexToken = function(match) { - var body, flags, heregex, re, tag, tokens, value, _i, _len, _ref2, _ref3, _ref4, _ref5; + var body, flags, flagsOffset, heregex, plusToken, prev, re, tag, token, tokens, value, _i, _len, _ref2, _ref3, _ref4; heregex = match[0], body = match[1], flags = match[2]; if (0 > body.indexOf('#{')) { - re = body.replace(HEREGEX_OMIT, '').replace(/\//g, '\\/'); + re = this.escapeLines(body.replace(HEREGEX_OMIT, '$1$2').replace(/\//g, '\\/'), true); if (re.match(/^\*/)) { this.error('regular expressions cannot begin with `*`'); } - this.token('REGEX', "/" + (re || '(?:)') + "/" + flags); + this.token('REGEX', "/" + (re || '(?:)') + "/" + flags, 0, heregex.length); return heregex.length; } - this.token('IDENTIFIER', 'RegExp'); - this.tokens.push(['CALL_START', '(']); + this.token('IDENTIFIER', 'RegExp', 0, 0); + this.token('CALL_START', '(', 0, 0); tokens = []; _ref2 = this.interpolateString(body, { regex: true }); for (_i = 0, _len = _ref2.length; _i < _len; _i++) { - _ref3 = _ref2[_i], tag = _ref3[0], value = _ref3[1]; + token = _ref2[_i]; + tag = token[0], value = token[1]; if (tag === 'TOKENS') { tokens.push.apply(tokens, value); - } else { - if (!(value = value.replace(HEREGEX_OMIT, ''))) { + } else if (tag === 'NEOSTRING') { + if (!(value = value.replace(HEREGEX_OMIT, '$1$2'))) { continue; } value = value.replace(/\\/g, '\\\\'); - tokens.push(['STRING', this.makeString(value, '"', true)]); + token[0] = 'STRING'; + token[1] = this.makeString(value, '"', true); + tokens.push(token); + } else { + this.error("Unexpected " + tag); } - tokens.push(['+', '+']); + prev = last(this.tokens); + plusToken = ['+', '+']; + plusToken[2] = prev[2]; + tokens.push(plusToken); } tokens.pop(); - if (((_ref4 = tokens[0]) != null ? _ref4[0] : void 0) !== 'STRING') { - this.tokens.push(['STRING', '""'], ['+', '+']); + if (((_ref3 = tokens[0]) != null ? _ref3[0] : void 0) !== 'STRING') { + this.token('STRING', '""', 0, 0); + this.token('+', '+', 0, 0); } - (_ref5 = this.tokens).push.apply(_ref5, tokens); + (_ref4 = this.tokens).push.apply(_ref4, tokens); if (flags) { - this.tokens.push([',', ','], ['STRING', '"' + flags + '"']); + flagsOffset = heregex.lastIndexOf(flags); + this.token(',', ',', flagsOffset, 0); + this.token('STRING', '"' + flags + '"', flagsOffset, flags.length); } - this.token(')', ')'); + this.token(')', ')', heregex.length - 1, 0); return heregex.length; }; Lexer.prototype.lineToken = function() { var diff, indent, match, noNewlines, size; if (!(match = MULTI_DENT.exec(this.chunk))) { return 0; } indent = match[0]; - this.line += count(indent, '\n'); this.seenFor = false; size = indent.length - 1 - indent.lastIndexOf('\n'); noNewlines = this.unfinished(); if (size - this.indebt === this.indent) { if (noNewlines) { this.suppressNewlines(); } else { - this.newlineToken(); + this.newlineToken(0); } return indent.length; } if (size > this.indent) { if (noNewlines) { this.indebt = size - this.indent; this.suppressNewlines(); return indent.length; } + if (!this.tokens.length) { + this.baseIndent = this.indent = size; + return indent.length; + } diff = size - this.indent + this.outdebt; - this.token('INDENT', diff); + this.token('INDENT', diff, indent.length - size, size); this.indents.push(diff); this.ends.push('OUTDENT'); this.outdebt = this.indebt = 0; + } else if (size < this.baseIndent) { + this.error('missing indentation', indent.length); } else { this.indebt = 0; - this.outdentToken(this.indent - size, noNewlines); + this.outdentToken(this.indent - size, noNewlines, indent.length); } this.indent = size; return indent.length; }; - Lexer.prototype.outdentToken = function(moveOut, noNewlines) { + Lexer.prototype.outdentToken = function(moveOut, noNewlines, outdentLength) { var dent, len; while (moveOut > 0) { len = this.indents.length - 1; if (this.indents[len] === void 0) { moveOut = 0; @@ -365,25 +399,25 @@ this.outdebt = 0; } else if (this.indents[len] < this.outdebt) { this.outdebt -= this.indents[len]; moveOut -= this.indents[len]; } else { - dent = this.indents.pop() - this.outdebt; + dent = this.indents.pop() + this.outdebt; moveOut -= dent; this.outdebt = 0; this.pair('OUTDENT'); - this.token('OUTDENT', dent); + this.token('OUTDENT', dent, 0, outdentLength); } } if (dent) { this.outdebt -= moveOut; } while (this.value() === ';') { this.tokens.pop(); } if (!(this.tag() === 'TERMINATOR' || noNewlines)) { - this.token('TERMINATOR', '\n'); + this.token('TERMINATOR', '\n', outdentLength, 0); } return this; }; Lexer.prototype.whitespaceToken = function() { @@ -400,16 +434,16 @@ } else { return 0; } }; - Lexer.prototype.newlineToken = function() { + Lexer.prototype.newlineToken = function(offset) { while (this.value() === ';') { this.tokens.pop(); } if (this.tag() !== 'TERMINATOR') { - this.token('TERMINATOR', '\n'); + this.token('TERMINATOR', '\n', offset, 0); } return this; }; Lexer.prototype.suppressNewlines = function() { @@ -490,11 +524,11 @@ indent = options.indent, herecomment = options.herecomment; if (herecomment) { if (HEREDOC_ILLEGAL.test(doc)) { this.error("block comment cannot contain \"*/\", starting"); } - if (doc.indexOf('\n') <= 0) { + if (doc.indexOf('\n') < 0) { return doc; } } else { while (match = HEREDOC_INDENT.exec(doc)) { attempt = match[1]; @@ -579,15 +613,18 @@ } return this.error("missing " + (stack.pop()) + ", starting"); }; Lexer.prototype.interpolateString = function(str, options) { - var expr, heredoc, i, inner, interpolated, len, letter, nested, pi, regex, tag, tokens, value, _i, _len, _ref2, _ref3, _ref4; + var column, expr, heredoc, i, inner, interpolated, len, letter, lexedLength, line, locationToken, nested, offsetInChunk, pi, plusToken, popped, regex, rparen, strOffset, tag, token, tokens, value, _i, _len, _ref2, _ref3, _ref4; if (options == null) { options = {}; } - heredoc = options.heredoc, regex = options.regex; + heredoc = options.heredoc, regex = options.regex, offsetInChunk = options.offsetInChunk, strOffset = options.strOffset, lexedLength = options.lexedLength; + offsetInChunk = offsetInChunk || 0; + strOffset = strOffset || 0; + lexedLength = lexedLength || str.length; tokens = []; pi = 0; i = -1; while (letter = str.charAt(i += 1)) { if (letter === '\\') { @@ -596,61 +633,79 @@ } if (!(letter === '#' && str.charAt(i + 1) === '{' && (expr = this.balancedString(str.slice(i + 1), '}')))) { continue; } if (pi < i) { - tokens.push(['NEOSTRING', str.slice(pi, i)]); + tokens.push(this.makeToken('NEOSTRING', str.slice(pi, i), strOffset + pi)); } inner = expr.slice(1, -1); if (inner.length) { + _ref2 = this.getLineAndColumnFromChunk(strOffset + i + 1), line = _ref2[0], column = _ref2[1]; nested = new Lexer().tokenize(inner, { - line: this.line, + line: line, + column: column, rewrite: false }); - nested.pop(); - if (((_ref2 = nested[0]) != null ? _ref2[0] : void 0) === 'TERMINATOR') { - nested.shift(); + popped = nested.pop(); + if (((_ref3 = nested[0]) != null ? _ref3[0] : void 0) === 'TERMINATOR') { + popped = nested.shift(); } if (len = nested.length) { if (len > 1) { - nested.unshift(['(', '(', this.line]); - nested.push([')', ')', this.line]); + nested.unshift(this.makeToken('(', '(', strOffset + i + 1, 0)); + nested.push(this.makeToken(')', ')', strOffset + i + 1 + inner.length, 0)); } tokens.push(['TOKENS', nested]); } } i += expr.length; pi = i + 1; } if ((i > pi && pi < str.length)) { - tokens.push(['NEOSTRING', str.slice(pi)]); + tokens.push(this.makeToken('NEOSTRING', str.slice(pi), strOffset + pi)); } if (regex) { return tokens; } if (!tokens.length) { - return this.token('STRING', '""'); + return this.token('STRING', '""', offsetInChunk, lexedLength); } if (tokens[0][0] !== 'NEOSTRING') { - tokens.unshift(['', '']); + tokens.unshift(this.makeToken('NEOSTRING', '', offsetInChunk)); } if (interpolated = tokens.length > 1) { - this.token('(', '('); + this.token('(', '(', offsetInChunk, 0); } for (i = _i = 0, _len = tokens.length; _i < _len; i = ++_i) { - _ref3 = tokens[i], tag = _ref3[0], value = _ref3[1]; + token = tokens[i]; + tag = token[0], value = token[1]; if (i) { - this.token('+', '+'); + if (i) { + plusToken = this.token('+', '+'); + } + locationToken = tag === 'TOKENS' ? value[0] : token; + plusToken[2] = { + first_line: locationToken[2].first_line, + first_column: locationToken[2].first_column, + last_line: locationToken[2].first_line, + last_column: locationToken[2].first_column + }; } if (tag === 'TOKENS') { (_ref4 = this.tokens).push.apply(_ref4, value); + } else if (tag === 'NEOSTRING') { + token[0] = 'STRING'; + token[1] = this.makeString(value, '"', heredoc); + this.tokens.push(token); } else { - this.token('STRING', this.makeString(value, '"', heredoc)); + this.error("Unexpected " + tag); } } if (interpolated) { - this.token(')', ')'); + rparen = this.makeToken(')', ')', offsetInChunk + lexedLength, 0); + rparen.stringEnd = true; + this.tokens.push(rparen); } return tokens; }; Lexer.prototype.pair = function(tag) { @@ -664,14 +719,54 @@ return this.pair(tag); } return this.ends.pop(); }; - Lexer.prototype.token = function(tag, value) { - return this.tokens.push([tag, value, this.line]); + Lexer.prototype.getLineAndColumnFromChunk = function(offset) { + var column, lineCount, lines, string; + if (offset === 0) { + return [this.chunkLine, this.chunkColumn]; + } + if (offset >= this.chunk.length) { + string = this.chunk; + } else { + string = this.chunk.slice(0, +(offset - 1) + 1 || 9e9); + } + lineCount = count(string, '\n'); + column = this.chunkColumn; + if (lineCount > 0) { + lines = string.split('\n'); + column = last(lines).length; + } else { + column += string.length; + } + return [this.chunkLine + lineCount, column]; }; + Lexer.prototype.makeToken = function(tag, value, offsetInChunk, length) { + var lastCharacter, locationData, token, _ref2, _ref3; + if (offsetInChunk == null) { + offsetInChunk = 0; + } + if (length == null) { + length = value.length; + } + locationData = {}; + _ref2 = this.getLineAndColumnFromChunk(offsetInChunk), locationData.first_line = _ref2[0], locationData.first_column = _ref2[1]; + lastCharacter = Math.max(0, length - 1); + _ref3 = this.getLineAndColumnFromChunk(offsetInChunk + lastCharacter), locationData.last_line = _ref3[0], locationData.last_column = _ref3[1]; + token = [tag, value, locationData]; + return token; + }; + + Lexer.prototype.token = function(tag, value, offsetInChunk, length) { + var token; + token = this.makeToken(tag, value, offsetInChunk, length); + this.tokens.push(token); + return token; + }; + Lexer.prototype.tag = function(index, tag) { var tok; return (tok = last(this.tokens, index)) && (tag ? tok[0] = tag : tok[0]); }; @@ -680,34 +775,57 @@ return (tok = last(this.tokens, index)) && (val ? tok[1] = val : tok[1]); }; Lexer.prototype.unfinished = function() { var _ref2; - return LINE_CONTINUER.test(this.chunk) || ((_ref2 = this.tag()) === '\\' || _ref2 === '.' || _ref2 === '?.' || _ref2 === 'UNARY' || _ref2 === 'MATH' || _ref2 === '+' || _ref2 === '-' || _ref2 === 'SHIFT' || _ref2 === 'RELATION' || _ref2 === 'COMPARE' || _ref2 === 'LOGIC' || _ref2 === 'THROW' || _ref2 === 'EXTENDS'); + return LINE_CONTINUER.test(this.chunk) || ((_ref2 = this.tag()) === '\\' || _ref2 === '.' || _ref2 === '?.' || _ref2 === '?::' || _ref2 === 'UNARY' || _ref2 === 'MATH' || _ref2 === '+' || _ref2 === '-' || _ref2 === 'SHIFT' || _ref2 === 'RELATION' || _ref2 === 'COMPARE' || _ref2 === 'LOGIC' || _ref2 === 'THROW' || _ref2 === 'EXTENDS'); }; + Lexer.prototype.removeNewlines = function(str) { + return str.replace(/^\s*\n\s*/, '').replace(/([^\\]|\\\\)\s*\n\s*$/, '$1'); + }; + Lexer.prototype.escapeLines = function(str, heredoc) { - return str.replace(MULTILINER, heredoc ? '\\n' : ''); + str = str.replace(/\\[^\S\n]*(\n|\\)\s*/g, function(escaped, character) { + if (character === '\n') { + return ''; + } else { + return escaped; + } + }); + if (heredoc) { + return str.replace(MULTILINER, '\\n'); + } else { + return str.replace(/\s*\n\s*/g, ' '); + } }; Lexer.prototype.makeString = function(body, quote, heredoc) { if (!body) { return quote + quote; } - body = body.replace(/\\([\s\S])/g, function(match, contents) { - if (contents === '\n' || contents === quote) { + body = body.replace(RegExp("\\\\(" + quote + "|\\\\)", "g"), function(match, contents) { + if (contents === quote) { return contents; } else { return match; } }); body = body.replace(RegExp("" + quote, "g"), '\\$&'); return quote + this.escapeLines(body, heredoc) + quote; }; - Lexer.prototype.error = function(message) { - throw SyntaxError("" + message + " on line " + (this.line + 1)); + Lexer.prototype.error = function(message, offset) { + var first_column, first_line, _ref2; + if (offset == null) { + offset = 0; + } + _ref2 = this.getLineAndColumnFromChunk(offset), first_line = _ref2[0], first_column = _ref2[1]; + return throwSyntaxError(message, { + first_line: first_line, + first_column: first_column + }); }; return Lexer; })(); @@ -747,35 +865,37 @@ exports.RESERVED = RESERVED.concat(JS_KEYWORDS).concat(COFFEE_KEYWORDS).concat(STRICT_PROSCRIBED); exports.STRICT_PROSCRIBED = STRICT_PROSCRIBED; + BOM = 65279; + IDENTIFIER = /^([$A-Za-z_\x7f-\uffff][$\w\x7f-\uffff]*)([^\n\S]*:(?!:))?/; NUMBER = /^0b[01]+|^0o[0-7]+|^0x[\da-f]+|^\d*\.?\d+(?:e[+-]?\d+)?/i; - HEREDOC = /^("""|''')([\s\S]*?)(?:\n[^\n\S]*)?\1/; + HEREDOC = /^("""|''')((?:\\[\s\S]|[^\\])*?)(?:\n[^\n\S]*)?\1/; - OPERATOR = /^(?:[-=]>|[-+*\/%<>&|^!?=]=|>>>=?|([-+:])\1|([&|<>])\2=?|\?\.|\.{2,3})/; + OPERATOR = /^(?:[-=]>|[-+*\/%<>&|^!?=]=|>>>=?|([-+:])\1|([&|<>])\2=?|\?(\.|::)|\.{2,3})/; WHITESPACE = /^[^\n\S]+/; - COMMENT = /^###([^#][\s\S]*?)(?:###[^\n\S]*|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/; + COMMENT = /^###([^#][\s\S]*?)(?:###[^\n\S]*|###$)|^(?:\s*#(?!##[^#]).*)+/; CODE = /^[-=]>/; MULTI_DENT = /^(?:\n[^\n\S]*)+/; - SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/; + SIMPLESTR = /^'[^\\']*(?:\\[\s\S][^\\']*)*'/; JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/; REGEX = /^(\/(?![\s=])[^[\/\n\\]*(?:(?:\\[\s\S]|\[[^\]\n\\]*(?:\\[\s\S][^\]\n\\]*)*])[^[\/\n\\]*)*\/)([imgy]{0,4})(?!\w)/; - HEREGEX = /^\/{3}([\s\S]+?)\/{3}([imgy]{0,4})(?!\w)/; + HEREGEX = /^\/{3}((?:\\?[\s\S])+?)\/{3}([imgy]{0,4})(?!\w)/; - HEREGEX_OMIT = /\s+(?:#.*)?/g; + HEREGEX_OMIT = /((?:\\\\)+)|\\(\s|\/)|\s+(?:#.*)?/g; MULTILINER = /\n/g; HEREDOC_INDENT = /\n+([^\n\S]*)/g; @@ -799,12 +919,12 @@ RELATION = ['IN', 'OF', 'INSTANCEOF']; BOOL = ['TRUE', 'FALSE']; - NOT_REGEX = ['NUMBER', 'REGEX', 'BOOL', 'NULL', 'UNDEFINED', '++', '--', ']']; + NOT_REGEX = ['NUMBER', 'REGEX', 'BOOL', 'NULL', 'UNDEFINED', '++', '--']; - NOT_SPACED_REGEX = NOT_REGEX.concat(')', '}', 'THIS', 'IDENTIFIER', 'STRING'); + NOT_SPACED_REGEX = NOT_REGEX.concat(')', '}', 'THIS', 'IDENTIFIER', 'STRING', ']'); CALLABLE = ['IDENTIFIER', 'STRING', 'REGEX', ')', ']', '}', '?', '::', '@', 'THIS', 'SUPER']; INDEXABLE = CALLABLE.concat('NUMBER', 'BOOL', 'NULL', 'UNDEFINED'); \ No newline at end of file