You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

156 lines
4.7 KiB

#!/usr/bin/env node
//
var fs = require('fs'),
path = require('path'),
http = require('http'),
BufferStream = require('bufferstream'),
// http://www.ksu.ru/eng/departments/ktk/test/perl/lib/unicode/UCDFF301.html
keys = ['value', 'name', 'category', 'class',
'bidirectional_category', 'mapping', 'decimal_digit_value', 'digit_value',
'numeric_value', 'mirrored', 'unicode_name', 'comment', 'uppercase_mapping',
'lowercase_mapping', 'titlecase_mapping'],
systemfiles = [
"UnicodeData.txt"
],
refs = 0;
function zeroPadHex(text, finalLength) {
return ('0000' + text).slice(-1 * finalLength);
}
function escape(charValue) {
var hexadecimal = charValue.replace(/^0*/, ''); // is already in hexadecimal
if (hexadecimal.length <= 2) {
// shorthand: \x30
return '\\x' + zeroPadHex(hexadecimal, 2);
} else if (hexadecimal.length <= 4) {
// BMP unicode: \u2602
return '\\u' + zeroPadHex(hexadecimal, 4);
} else {
// Outside of BMP: \uD83D\uDE00
var astralCodePoint = parseInt(hexadecimal, 16);
var highSurrogate =
Math.floor((astralCodePoint - 0x10000) / 0x400) + 0xd800;
var lowSurrogate = ((astralCodePoint - 0x10000) % 0x400) + 0xdc00;
var highSurrogateString =
'\\u' + zeroPadHex(highSurrogate.toString(16), 4);
var lowSurrogateString =
'\\u' + zeroPadHex(lowSurrogate.toString(16), 4);
return '' + highSurrogateString + '' + lowSurrogateString;
}
}
function stringify(key, value) {
return key + ":" + JSON.stringify(value).replace(/\\\\(u|x)/g, "\\$1");
}
function create_index(categories, len) {
console.log("saving index.js …");
var filename = path.join(__dirname, "category", "index.js"),
cat, contents = 'module.exports = {';
for(var i = 0 ; i < len ; i++) {
cat = categories[i];
contents += "\n " + cat + ": require('./" + cat + "')";
if ((i + 1) !== len) contents += ',';
}
contents += '\n};';
fs.writeFileSync(filename, contents, {encoding:'utf8'});
}
function newFile(name, callback) {
var filename = path.join(__dirname, "category", name + ".js"),
file = fs.createWriteStream(filename, {encoding:'utf8'});
file.once('close', function () {
if (!--refs) {
console.log("done.");
callback();
}
});
refs++;
return file;
}
function parser(callback) {
var data = {},
buffer = new BufferStream({encoding:'utf8', size:'flexible'}),
resume = buffer.resume.bind(buffer);
buffer.split('\n', function (line) {
var v, c, char = {},
values = line.toString().split(';');
for(var i = 0 ; i < 15 ; i++)
char[keys[i]] = values[i];
v = parseInt(char.value, 16);
char.symbol = escape(char.value);
c = char.category;
if (!data[c]) {
data[c] = newFile(c, callback)
.on('drain', resume)
.once('open', function () {
console.log("saving data as %s.js …", c);
if (this.write('module.exports={' + stringify(v, char)))
buffer.resume();
});
buffer.pause();
} else if (!data[c].write("," + stringify(v, char))) {
buffer.pause();
}
});
buffer.on('end', function () {
var cat, categories = Object.keys(data),
len = categories.length;
for(var i = 0 ; i < len ; i++) {
cat = categories[i];
data[cat].end("};");
}
create_index(categories, len);
});
buffer.on('error', function (err) {
if (typeof err === 'string')
err = new Error(err);
throw err;
});
return buffer;
}
function read_file(success_cb, error_cb) {
var systemfile, sysfiles = systemfiles.slice(),
try_reading = function (success, error) {
systemfile = sysfiles.shift();
if (!systemfile) return error_cb();
console.log("try to read file %s …", systemfile);
fs.exists(systemfile, function (exists) {
if (!exists) {
console.error("%s not found.", systemfile);
return try_reading(success_cb, error_cb);
}
console.log("parsing …");
fs.createReadStream(systemfile, {encoding:'utf8'}).pipe(parser(success_cb));
});
};
try_reading(success_cb, error_cb);
}
// run
if (!module.parent) { // not required
read_file(process.exit, process.exit);
} else {
module.exports = {
escape:escape,
stringify:stringify,
newFile:newFile,
parser:parser,
read_file:read_file
};
}