You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
56 lines
1.3 KiB
56 lines
1.3 KiB
2 years ago
|
var SyntaxError = require('./SyntaxError');
|
||
|
|
|
||
|
|
var TAB = 9;
|
||
|
|
var N = 10;
|
||
|
|
var F = 12;
|
||
|
|
var R = 13;
|
||
|
|
var SPACE = 32;
|
||
|
|
|
||
|
|
var Tokenizer = function(str) {
|
||
|
|
this.str = str;
|
||
|
|
this.pos = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
Tokenizer.prototype = {
|
||
|
|
charCodeAt: function(pos) {
|
||
|
|
return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
|
||
|
|
},
|
||
|
|
charCode: function() {
|
||
|
|
return this.charCodeAt(this.pos);
|
||
|
|
},
|
||
|
|
nextCharCode: function() {
|
||
|
|
return this.charCodeAt(this.pos + 1);
|
||
|
|
},
|
||
|
|
nextNonWsCode: function(pos) {
|
||
|
|
return this.charCodeAt(this.findWsEnd(pos));
|
||
|
|
},
|
||
|
|
findWsEnd: function(pos) {
|
||
|
|
for (; pos < this.str.length; pos++) {
|
||
|
|
var code = this.str.charCodeAt(pos);
|
||
|
|
if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return pos;
|
||
|
|
},
|
||
|
|
substringToPos: function(end) {
|
||
|
|
return this.str.substring(this.pos, this.pos = end);
|
||
|
|
},
|
||
|
|
eat: function(code) {
|
||
|
|
if (this.charCode() !== code) {
|
||
|
|
this.error('Expect `' + String.fromCharCode(code) + '`');
|
||
|
|
}
|
||
|
|
|
||
|
|
this.pos++;
|
||
|
|
},
|
||
|
|
peek: function() {
|
||
|
|
return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
|
||
|
|
},
|
||
|
|
error: function(message) {
|
||
|
|
throw new SyntaxError(message, this.str, this.pos);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
module.exports = Tokenizer;
|