lib/gollum/public/gollum/livepreview/js/ace/lib/ace/tokenizer_dev.js in gollum-3.1.2 vs lib/gollum/public/gollum/livepreview/js/ace/lib/ace/tokenizer_dev.js in gollum-3.1.3
- old
+ new
@@ -27,63 +27,36 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ***** END LICENSE BLOCK ***** */
define(function(require, exports, module) {
+var BaseTokenizer = require("./tokenizer").Tokenizer;
+// tokenizing lines longer than this makes editor very slow
+var MAX_TOKEN_COUNT = 100000;
/*
* version of Tokenizer with additional logging
* and infinite loop checks
* can be used for developing/testing new modes
**/
-var Tokenizer = function(rules, flag) {
- flag = flag ? "g" + flag : "g";
- this.rules = rules;
- this.regExps = {};
- this.matchMappings = {};
- for ( var key in this.rules) {
- var rule = this.rules[key];
- var state = rule;
- var ruleRegExps = [];
- var matchTotal = 0;
- var mapping = this.matchMappings[key] = {};
+var Tokenizer = function(rules) {
+ BaseTokenizer.call(this, rules);
- for ( var i = 0; i < state.length; i++) {
-
- if (state[i].regex instanceof RegExp)
- state[i].regex = state[i].regex.toString().slice(1, -1);
-
- // Count number of matching groups. 2 extra groups from the full match
- // And the catch-all on the end (used to force a match);
- var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
-
- // Replace any backreferences and offset appropriately.
- var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
- return "\\" + (parseInt(digit, 10) + matchTotal + 1);
- });
-
- if (matchcount > 1 && state[i].token.length !== matchcount-1)
- throw new Error("For " + state[i].regex + " the matching groups and length of the token array don't match (rule #" + i + " of state " + key + ")");
-
- mapping[matchTotal] = {
- rule: i,
- len: matchcount
- };
- matchTotal += matchcount;
-
- ruleRegExps.push(adjustedregex);
- }
-
- this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
- }
-};
-
-(function() {
+ /**
+ * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
+ * @returns {Object}
+ **/
this.getLineTokens = function(line, startState) {
+ if (startState && typeof startState != "string") {
+ var stack = startState.slice(0);
+ startState = stack[0];
+ } else
+ var stack = [];
+
var currentState = startState || "start";
- var state = this.rules[currentState];
+ var state = this.states[currentState];
var mapping = this.matchMappings[currentState];
var re = this.regExps[currentState];
re.lastIndex = 0;
var match, tokens = [];
@@ -105,101 +78,106 @@
value: "",
state: currentState
};
initState();
- var maxRecur = 10000;
+ var maxRecur = 100000;
while (match = re.exec(line)) {
- var type = "default.text";
+ var type = mapping.defaultToken;
var rule = null;
- var value = [match[0]];
+ var value = match[0];
+ var index = re.lastIndex;
+ if (index - value.length > lastIndex) {
+ var skipped = line.substring(lastIndex, index - value.length);
+ if (token.type == type) {
+ token.value += skipped;
+ } else {
+ if (token.type)
+ tokens.push(token);
+ token = {type: type, value: skipped};
+ }
+ }
+
for (var i = 0; i < match.length-2; i++) {
if (match[i + 1] === undefined)
continue;
if (!maxRecur--) {
- throw "infinite" + mapping[i].rule + currentState
+ throw "infinite" + state[mapping[i]] + currentState
}
- rule = state[mapping[i].rule];
+ rule = state[mapping[i]];
- if (mapping[i].len > 1)
- value = match.slice(i+2, i+1+mapping[i].len);
-
- // compute token type
- if (typeof rule.token == "function")
- type = rule.token.apply(this, value);
+ if (rule.onMatch)
+ type = rule.onMatch(value, currentState, stack);
else
type = rule.token;
if (rule.next) {
- currentState = rule.next;
- state = this.rules[currentState];
- mapping = this.matchMappings[currentState];
- lastIndex = re.lastIndex;
+ if (typeof rule.next == "string")
+ currentState = rule.next;
+ else
+ currentState = rule.next(currentState, stack);
+ state = this.states[currentState];
+ if (!state) {
+ window.console && console.error && console.error(currentState, "doesn't exist");
+ currentState = "start";
+ state = this.states[currentState];
+ }
+ mapping = this.matchMappings[currentState];
+ lastIndex = index;
re = this.regExps[currentState];
+ re.lastIndex = index;
- if (re === undefined) {
- throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
- }
-
- re.lastIndex = lastIndex;
-
onStateChange();
}
break;
}
- if (value[0]) {
+ if (value) {
if (typeof type == "string") {
- value = [value.join("")];
- type = [type];
- }
- for (var i = 0; i < value.length; i++) {
- if (!value[i])
- continue;
-
- var mergeable = (!rule || rule.merge || type[i] === "text") && token.type === type[i];
-
- if (false && mergeable) {
- token.value += value[i];
+ if ((!rule || rule.merge !== false) && token.type === type) {
+ token.value += value;
} else {
- if (token.type) {
- token.stateTransitions = stateTransitions;
+ if (token.type)
tokens.push(token);
- initState()
- }
-
- token = {
- type: type[i],
- value: value[i],
- state: currentState,
- mergeable: mergeable
- };
+ token = {type: type, value: value};
}
+ } else {
+ if (token.type)
+ tokens.push(token);
+ token = {type: null, value: ""};
+ for (var i = 0; i < type.length; i++)
+ tokens.push(type[i]);
}
}
if (lastIndex == line.length)
break;
- lastIndex = re.lastIndex;
+ lastIndex = index;
+
+ if (tokens.length > MAX_TOKEN_COUNT) {
+ token.value += line.substr(lastIndex);
+ currentState = "start"
+ break;
+ }
}
- if (token.type) {
- token.stateTransitions = stateTransitions;
+ if (token.type)
tokens.push(token);
- }
return {
tokens : tokens,
- state : currentState
+ state : stack.length ? stack : currentState
};
};
-}).call(Tokenizer.prototype);
+};
+
+Tokenizer.prototype = BaseTokenizer.prototype;
exports.Tokenizer = Tokenizer;
});