Sha256: 10bfc90fe8d2e4a2e7381e9ba5858fe6290211b1b6d319b66bca0b6f56806bba
Contents?: true
Size: 1.37 KB
Versions: 34
Compression:
Stored size: 1.37 KB
Contents
var SyntaxParseError = require('./error').SyntaxParseError;
var TAB = 9;
var N = 10;
var F = 12;
var R = 13;
var SPACE = 32;
var Tokenizer = function(str) {
this.str = str;
this.pos = 0;
};
Tokenizer.prototype = {
charCodeAt: function(pos) {
return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
},
charCode: function() {
return this.charCodeAt(this.pos);
},
nextCharCode: function() {
return this.charCodeAt(this.pos + 1);
},
nextNonWsCode: function(pos) {
return this.charCodeAt(this.findWsEnd(pos));
},
findWsEnd: function(pos) {
for (; pos < this.str.length; pos++) {
var code = this.str.charCodeAt(pos);
if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
break;
}
}
return pos;
},
substringToPos: function(end) {
return this.str.substring(this.pos, this.pos = end);
},
eat: function(code) {
if (this.charCode() !== code) {
this.error('Expect `' + String.fromCharCode(code) + '`');
}
this.pos++;
},
peek: function() {
return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
},
error: function(message) {
throw new SyntaxParseError(message, this.str, this.pos);
}
};
module.exports = Tokenizer;
Version data entries
34 entries across 18 versions & 4 rubygems