You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
57 lines
1.3 KiB
57 lines
1.3 KiB
2 years ago
|
'use strict';
|
||
|
|
|
||
|
|
const SyntaxError = require('./SyntaxError.cjs');
|
||
|
|
|
||
|
|
const TAB = 9;
|
||
|
|
const N = 10;
|
||
|
|
const F = 12;
|
||
|
|
const R = 13;
|
||
|
|
const SPACE = 32;
|
||
|
|
|
||
|
|
class Tokenizer {
|
||
|
|
constructor(str) {
|
||
|
|
this.str = str;
|
||
|
|
this.pos = 0;
|
||
|
|
}
|
||
|
|
charCodeAt(pos) {
|
||
|
|
return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
|
||
|
|
}
|
||
|
|
charCode() {
|
||
|
|
return this.charCodeAt(this.pos);
|
||
|
|
}
|
||
|
|
nextCharCode() {
|
||
|
|
return this.charCodeAt(this.pos + 1);
|
||
|
|
}
|
||
|
|
nextNonWsCode(pos) {
|
||
|
|
return this.charCodeAt(this.findWsEnd(pos));
|
||
|
|
}
|
||
|
|
findWsEnd(pos) {
|
||
|
|
for (; pos < this.str.length; pos++) {
|
||
|
|
const code = this.str.charCodeAt(pos);
|
||
|
|
if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return pos;
|
||
|
|
}
|
||
|
|
substringToPos(end) {
|
||
|
|
return this.str.substring(this.pos, this.pos = end);
|
||
|
|
}
|
||
|
|
eat(code) {
|
||
|
|
if (this.charCode() !== code) {
|
||
|
|
this.error('Expect `' + String.fromCharCode(code) + '`');
|
||
|
|
}
|
||
|
|
|
||
|
|
this.pos++;
|
||
|
|
}
|
||
|
|
peek() {
|
||
|
|
return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
|
||
|
|
}
|
||
|
|
error(message) {
|
||
|
|
throw new SyntaxError.SyntaxError(message, this.str, this.pos);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
exports.Tokenizer = Tokenizer;
|