diff --git a/Makefile b/Makefile index 2594ab966..89de88a3e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ test: @find test/{simple,system}/test-*.js | xargs -n 1 -t node +test-all: test + @find test/system/slow/test-*.js | xargs -n 1 -t node -.PHONY: test \ No newline at end of file +.PHONY: test diff --git a/Readme.md b/Readme.md index 9247cfcd2..2fb3acefc 100644 --- a/Readme.md +++ b/Readme.md @@ -15,6 +15,7 @@ A node.js module implementing the ## Sponsors * [Joyent](http://www.joyent.com/) +* [pinkbike.com](http://pinkbike.com/) I'm working on this driver because I need it for my own startup ([transloadit.com][transloadit]), but it's a big project (~100-200 hours) with @@ -192,10 +193,13 @@ parameter is provided which contains the information from the mysql OK packet. At this point the module is ready to be tried out, but a lot of things are yet to be done: -* Auto-cast mysql types into JS types +* Handle timeouts / reconnect +* Remaining mysql commands * Prepared Statements -* Charsets handling -* ... +* Packet's > 16 MB +* Compression +* Performance profiling +* ? ## License diff --git a/lib/mysql/auth.js b/lib/mysql/auth.js index 6077560de..08d0b285f 100644 --- a/lib/mysql/auth.js +++ b/lib/mysql/auth.js @@ -73,7 +73,7 @@ exports.randomInit = function(seed1, seed2) { max_value: 0x3FFFFFFF, max_value_dbl: 0x3FFFFFFF, seed1: seed1 % 0x3FFFFFFF, - seed2: seed2 % 0x3FFFFFFF, + seed2: seed2 % 0x3FFFFFFF }; }; @@ -115,18 +115,18 @@ exports.fmt32 = function(x){ if (b.length == 2) b = '00'+b; if (b.length == 3) b = '0'+b; return '' + a + '/' + b; -} +}; exports.xor32 = function(a,b){ return [a[0] ^ b[0], a[1] ^ b[1]]; -} +}; exports.add32 = function(a,b){ var w1 = a[1] + b[1], w2 = a[0] + b[0] + ((w1 & 0xFFFF0000) >> 16); return [w2 & 0xFFFF, w1 & 0xFFFF]; -} +}; exports.mul32 = function(a,b){ // based on this example of multiplying 32b ints using 16b @@ -135,11 +135,11 @@ exports.mul32 = function(a,b){ w2 = (((a[1] * b[1]) >> 16) & 0xFFFF) + ((a[0] * b[1]) & 0xFFFF) + (a[1] * b[0] & 0xFFFF); return [w2 & 0xFFFF, w1 & 0xFFFF]; -} +}; exports.and32 = function(a,b){ - return [a[0] & b[0], a[1] & b[1]] -} + return [a[0] & b[0], a[1] & b[1]]; +}; exports.shl32 = function(a,b){ // assume b is 16 or less @@ -147,7 +147,7 @@ exports.shl32 = function(a,b){ w2 = (a[0] << b) | ((w1 & 0xFFFF0000) >> 16); return [w2 & 0xFFFF, w1 & 0xFFFF]; -} +}; exports.int31Write = function(buffer, number, offset) { buffer[offset] = (number[0] >> 8) & 0x7F; diff --git a/lib/mysql/client.js b/lib/mysql/client.js index 66ac060c1..aa98d326d 100644 --- a/lib/mysql/client.js +++ b/lib/mysql/client.js @@ -8,9 +8,9 @@ var sys = require('sys'), Query = require('./query'), EventEmitter = require('events').EventEmitter; -function Client(config) { +function Client(properties) { if (!(this instanceof Client)) { - return new Client(config); + return new Client(properties); } EventEmitter.call(this); @@ -21,9 +21,10 @@ function Client(config) { this.password = null; this.database = ''; + this.typeCast = true; this.flags = Client.defaultFlags; this.maxPacketSize = 0x01000000; - this.charsetNumber = 8; + this.charsetNumber = Client.UTF8_UNICODE_CI; this.debug = false; this._greeting = null; @@ -31,10 +32,10 @@ function Client(config) { this._connection = null; this._parser = null; - for (var key in config) { - this[key] = config[key]; + for (var key in properties) { + this[key] = properties[key]; } -} +}; sys.inherits(Client, EventEmitter); module.exports = Client; @@ -69,7 +70,8 @@ Client.prototype.query = function(sql, params, cb) { cb = arguments[1]; } - var query = new Query(); + var query = new Query({typeCast: this.typeCast}); + if (cb) { var rows = [], fields = {}; query @@ -319,7 +321,7 @@ Client.CONNECT_WITH_DB = 8; Client.NO_SCHEMA = 16; Client.COMPRESS = 32; Client.ODBC = 64; -Client.LOCAL_FILES = 128 +Client.LOCAL_FILES = 128; Client.IGNORE_SPACE = 256; Client.PROTOCOL_41 = 512; Client.INTERACTIVE = 1024; @@ -379,6 +381,135 @@ Client.COM_STMT_RESET = 0x1a; Client.COM_SET_OPTION = 0x1b; Client.COM_STMT_FETCH = 0x1c; +// Collations / Charsets +Client.BIG5_CHINESE_CI = 1; +Client.LATIN2_CZECH_CS = 2; +Client.DEC8_SWEDISH_CI = 3; +Client.CP850_GENERAL_CI = 4; +Client.LATIN1_GERMAN1_CI = 5; +Client.HP8_ENGLISH_CI = 6; +Client.KOI8R_GENERAL_CI = 7; +Client.LATIN1_SWEDISH_CI = 8; +Client.LATIN2_GENERAL_CI = 9; +Client.SWE7_SWEDISH_CI = 10; +Client.ASCII_GENERAL_CI = 11; +Client.UJIS_JAPANESE_CI = 12; +Client.SJIS_JAPANESE_CI = 13; +Client.CP1251_BULGARIAN_CI = 14; +Client.LATIN1_DANISH_CI = 15; +Client.HEBREW_GENERAL_CI = 16; +Client.TIS620_THAI_CI = 18; +Client.EUCKR_KOREAN_CI = 19; +Client.LATIN7_ESTONIAN_CS = 20; +Client.LATIN2_HUNGARIAN_CI = 21; +Client.KOI8U_GENERAL_CI = 22; +Client.CP1251_UKRAINIAN_CI = 23; +Client.GB2312_CHINESE_CI = 24; +Client.GREEK_GENERAL_CI = 25; +Client.CP1250_GENERAL_CI = 26; +Client.LATIN2_CROATIAN_CI = 27; +Client.GBK_CHINESE_CI = 28; +Client.CP1257_LITHUANIAN_CI = 29; +Client.LATIN5_TURKISH_CI = 30; +Client.LATIN1_GERMAN2_CI = 31; +Client.ARMSCII8_GENERAL_CI = 32; +Client.UTF8_GENERAL_CI = 33; +Client.CP1250_CZECH_CS = 34; +Client.UCS2_GENERAL_CI = 35; +Client.CP866_GENERAL_CI = 36; +Client.KEYBCS2_GENERAL_CI = 37; +Client.MACCE_GENERAL_CI = 38; +Client.MACROMAN_GENERAL_CI = 39; +Client.CP852_GENERAL_CI = 40; +Client.LATIN7_GENERAL_CI = 41; +Client.LATIN7_GENERAL_CS = 42; +Client.MACCE_BIN = 43; +Client.CP1250_CROATIAN_CI = 44; +Client.LATIN1_BIN = 47; +Client.LATIN1_GENERAL_CI = 48; +Client.LATIN1_GENERAL_CS = 49; +Client.CP1251_BIN = 50; +Client.CP1251_GENERAL_CI = 51; +Client.CP1251_GENERAL_CS = 52; +Client.MACROMAN_BIN = 53; +Client.CP1256_GENERAL_CI = 57; +Client.CP1257_BIN = 58; +Client.CP1257_GENERAL_CI = 59; +Client.BINARY = 63; +Client.ARMSCII8_BIN = 64; +Client.ASCII_BIN = 65; +Client.CP1250_BIN = 66; +Client.CP1256_BIN = 67; +Client.CP866_BIN = 68; +Client.DEC8_BIN = 69; +Client.GREEK_BIN = 70; +Client.HEBREW_BIN = 71; +Client.HP8_BIN = 72; +Client.KEYBCS2_BIN = 73; +Client.KOI8R_BIN = 74; +Client.KOI8U_BIN = 75; +Client.LATIN2_BIN = 77; +Client.LATIN5_BIN = 78; +Client.LATIN7_BIN = 79; +Client.CP850_BIN = 80; +Client.CP852_BIN = 81; +Client.SWE7_BIN = 82; +Client.UTF8_BIN = 83; +Client.BIG5_BIN = 84; +Client.EUCKR_BIN = 85; +Client.GB2312_BIN = 86; +Client.GBK_BIN = 87; +Client.SJIS_BIN = 88; +Client.TIS620_BIN = 89; +Client.UCS2_BIN = 90; +Client.UJIS_BIN = 91; +Client.GEOSTD8_GENERAL_CI = 92; +Client.GEOSTD8_BIN = 93; +Client.LATIN1_SPANISH_CI = 94; +Client.CP932_JAPANESE_CI = 95; +Client.CP932_BIN = 96; +Client.EUCJPMS_JAPANESE_CI = 97; +Client.EUCJPMS_BIN = 98; +Client.CP1250_POLISH_CI = 99; +Client.UCS2_UNICODE_CI = 128; +Client.UCS2_ICELANDIC_CI = 129; +Client.UCS2_LATVIAN_CI = 130; +Client.UCS2_ROMANIAN_CI = 131; +Client.UCS2_SLOVENIAN_CI = 132; +Client.UCS2_POLISH_CI = 133; +Client.UCS2_ESTONIAN_CI = 134; +Client.UCS2_SPANISH_CI = 135; +Client.UCS2_SWEDISH_CI = 136; +Client.UCS2_TURKISH_CI = 137; +Client.UCS2_CZECH_CI = 138; +Client.UCS2_DANISH_CI = 139; +Client.UCS2_LITHUANIAN_CI = 140; +Client.UCS2_SLOVAK_CI = 141; +Client.UCS2_SPANISH2_CI = 142; +Client.UCS2_ROMAN_CI = 143; +Client.UCS2_PERSIAN_CI = 144; +Client.UCS2_ESPERANTO_CI = 145; +Client.UCS2_HUNGARIAN_CI = 146; +Client.UTF8_UNICODE_CI = 192; +Client.UTF8_ICELANDIC_CI = 193; +Client.UTF8_LATVIAN_CI = 194; +Client.UTF8_ROMANIAN_CI = 195; +Client.UTF8_SLOVENIAN_CI = 196; +Client.UTF8_POLISH_CI = 197; +Client.UTF8_ESTONIAN_CI = 198; +Client.UTF8_SPANISH_CI = 199; +Client.UTF8_SWEDISH_CI = 200; +Client.UTF8_TURKISH_CI = 201; +Client.UTF8_CZECH_CI = 202; +Client.UTF8_DANISH_CI = 203; +Client.UTF8_LITHUANIAN_CI = 204; +Client.UTF8_SLOVAK_CI = 205; +Client.UTF8_SPANISH2_CI = 206; +Client.UTF8_ROMAN_CI = 207; +Client.UTF8_PERSIAN_CI = 208; +Client.UTF8_ESPERANTO_CI = 209; +Client.UTF8_HUNGARIAN_CI = 210; + // Error numbers // from: http://dev.mysql.com/doc/refman/5.0/en/error-messages-server.html Client.ERROR_HASHCHK = 1000; diff --git a/lib/mysql/outgoing_packet.js b/lib/mysql/outgoing_packet.js index ea5a5b9ab..733be7e6c 100644 --- a/lib/mysql/outgoing_packet.js +++ b/lib/mysql/outgoing_packet.js @@ -7,7 +7,7 @@ function OutgoingPacket(size, num) { this.index = 0; this.writeNumber(3, size); this.writeNumber(1, num || 0); -} +}; module.exports = OutgoingPacket; OutgoingPacket.prototype.writeNumber = function(bytes, number) { diff --git a/lib/mysql/parser.js b/lib/mysql/parser.js index 078e867c5..84f867ae3 100644 --- a/lib/mysql/parser.js +++ b/lib/mysql/parser.js @@ -18,7 +18,7 @@ function Parser() { this._lengthCodedLength = null; this._lengthCodedStringLength = null; -} +}; sys.inherits(Parser, EventEmitter); module.exports = Parser; @@ -161,7 +161,7 @@ Parser.prototype.write = function(buffer) { break; case Parser.GREETING_FILLER_1: // 1 byte - 0x00 - advance() + advance(); break; case Parser.GREETING_SERVER_CAPABILITIES: if (packet.index == 0) { @@ -208,7 +208,7 @@ Parser.prototype.write = function(buffer) { if (packet.index == 0) { if (c === 0xff) { packet.type = Parser.ERROR_PACKET; - advance(Parser.ERROR_NUMBER) + advance(Parser.ERROR_NUMBER); break; } @@ -221,7 +221,7 @@ Parser.prototype.write = function(buffer) { // after the first OK PACKET, we are authenticated this.authenticated = true; packet.type = Parser.OK_PACKET; - advance(Parser.AFFECTED_ROWS) + advance(Parser.AFFECTED_ROWS); break; } } diff --git a/lib/mysql/query.js b/lib/mysql/query.js index dec386f07..abeae2548 100644 --- a/lib/mysql/query.js +++ b/lib/mysql/query.js @@ -5,9 +5,15 @@ var sys = require('sys'), Parser = require('./parser'), Client; -function Query() { +function Query(properties) { EventEmitter.call(this); -} + + this.typeCast = true; + + for (var key in properties) { + this[key] = properties[key]; + } +}; sys.inherits(Query, EventEmitter); module.exports = Query; @@ -31,7 +37,7 @@ Query.prototype._handlePacket = function(packet) { this._fields = []; } - this._fields.push(packet.name); + this._fields.push(packet); this.emit('field', packet); break; case Parser.EOF_PACKET: @@ -50,17 +56,42 @@ Query.prototype._handlePacket = function(packet) { field = this._fields[0]; this._rowIndex = 0; - row[field] = ''; + row[field.name] = ''; packet.on('data', function(buffer, remaining) { if (buffer) { - row[field] += buffer; + row[field.name] += buffer; } else { - row[field] = null; + row[field.name] = null; } if (remaining == 0) { self._rowIndex++; + if (self.typeCast) { + switch (field.fieldType) { + case Query.FIELD_TYPE_TIMESTAMP: + case Query.FIELD_TYPE_DATE: + case Query.FIELD_TYPE_DATETIME: + case Query.FIELD_TYPE_NEWDATE: + row[field.name] = new Date(row[field.name]+'Z'); + break; + case Query.FIELD_TYPE_TINY: + case Query.FIELD_TYPE_SHORT: + case Query.FIELD_TYPE_LONG: + case Query.FIELD_TYPE_LONGLONG: + case Query.FIELD_TYPE_INT24: + case Query.FIELD_TYPE_YEAR: + row[field.name] = parseInt(row[field.name], 10); + break; + case Query.FIELD_TYPE_DECIMAL: + case Query.FIELD_TYPE_FLOAT: + case Query.FIELD_TYPE_DOUBLE: + case Query.FIELD_TYPE_NEWDECIMAL: + row[field.name] = parseFloat(row[field.name]); + break; + } + } + if (self._rowIndex == self._fields.length) { delete self._row; delete self._rowIndex; @@ -69,9 +100,37 @@ Query.prototype._handlePacket = function(packet) { } field = self._fields[self._rowIndex]; - row[field] = ''; + row[field.name] = ''; } }); break; } }; + +Query.FIELD_TYPE_DECIMAL = 0x00; +Query.FIELD_TYPE_TINY = 0x01; +Query.FIELD_TYPE_SHORT = 0x02; +Query.FIELD_TYPE_LONG = 0x03; +Query.FIELD_TYPE_FLOAT = 0x04; +Query.FIELD_TYPE_DOUBLE = 0x05; +Query.FIELD_TYPE_NULL = 0x06; +Query.FIELD_TYPE_TIMESTAMP = 0x07; +Query.FIELD_TYPE_LONGLONG = 0x08; +Query.FIELD_TYPE_INT24 = 0x09; +Query.FIELD_TYPE_DATE = 0x0a; +Query.FIELD_TYPE_TIME = 0x0b; +Query.FIELD_TYPE_DATETIME = 0x0c; +Query.FIELD_TYPE_YEAR = 0x0d; +Query.FIELD_TYPE_NEWDATE = 0x0e; +Query.FIELD_TYPE_VARCHAR = 0x0f; +Query.FIELD_TYPE_BIT = 0x10; +Query.FIELD_TYPE_NEWDECIMAL = 0xf6; +Query.FIELD_TYPE_ENUM = 0xf7; +Query.FIELD_TYPE_SET = 0xf8; +Query.FIELD_TYPE_TINY_BLOB = 0xf9; +Query.FIELD_TYPE_MEDIUM_BLOB = 0xfa; +Query.FIELD_TYPE_LONG_BLOB = 0xfb; +Query.FIELD_TYPE_BLOB = 0xfc; +Query.FIELD_TYPE_VAR_STRING = 0xfd; +Query.FIELD_TYPE_STRING = 0xfe; +Query.FIELD_TYPE_GEOMETRY = 0xff; diff --git a/package.json b/package.json index c8d78ac8c..1c679f438 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { "name" : "mysql" -, "version": "0.5.0" +, "version": "0.6.0" , "dependencies": {"gently": ">=0.8.0"} , "main" : "./lib/mysql" } diff --git a/test/simple/test-client.js b/test/simple/test-client.js index 7ab26c949..6e03a3fe6 100644 --- a/test/simple/test-client.js +++ b/test/simple/test-client.js @@ -7,7 +7,7 @@ var StreamStub = GENTLY.stub('net', 'Stream'), for (var k in Parser) { ParserStub[k] = Parser[k]; -} +}; var Client = require('mysql/client'); @@ -16,7 +16,7 @@ function test(test) { gently = new Gently(); test(); gently.verify(test.name); -} +}; test(function constructor() { (function testDefaultProperties() { @@ -28,11 +28,12 @@ test(function constructor() { assert.strictEqual(client.password, null); assert.strictEqual(client.database, ''); + assert.strictEqual(client.typeCast, true); assert.strictEqual(client.debug, false); assert.strictEqual(client.flags, Client.defaultFlags); assert.strictEqual(client.maxPacketSize, 0x01000000); - assert.strictEqual(client.charsetNumber, 8); + assert.strictEqual(client.charsetNumber, Client.UTF8_UNICODE_CI); assert.strictEqual(client._greeting, null); assert.deepEqual(client._queue, []); @@ -139,6 +140,8 @@ test(function query() { QUERY, queryEmit = {}; + client.typeCast = 'super'; + (function testRegular() { gently.expect(client, 'format', function(sql, params) { assert.strictEqual(sql, SQL); @@ -146,9 +149,11 @@ test(function query() { return FORMATED_SQL; }); - gently.expect(QueryStub, 'new', function() { + gently.expect(QueryStub, 'new', function(properties) { QUERY = this; + assert.equal(properties.typeCast, client.typeCast); + var events = ['error', 'field', 'row', 'end']; gently.expect(QUERY, 'on', events.length, function (event, fn) { assert.equal(event, events.shift()); @@ -524,7 +529,7 @@ test(function _packetToUserObject() { length: 65, received: 65, number: 92, - foo: 'bar', + foo: 'bar' }; var ok = Client._packetToUserObject(PACKET); diff --git a/test/simple/test-query.js b/test/simple/test-query.js index 58a2c1e63..a23acfb98 100644 --- a/test/simple/test-query.js +++ b/test/simple/test-query.js @@ -15,6 +15,8 @@ function test(test) { test(function constructor() { assert.ok(query instanceof EventEmitter); + assert.strictEqual(query.typeCast, true); + assert.equal(new Query({foo: 'bar'}).foo, 'bar'); }); test(function _handlePacket() { @@ -60,7 +62,7 @@ test(function _handlePacket() { query._handlePacket(PACKET); - assert.deepEqual(query._fields, [PACKET.name]); + assert.strictEqual(query._fields[0], PACKET); })(); (function testEofPacket() { @@ -78,7 +80,7 @@ test(function _handlePacket() { })(); (function testRowPacket() { - query._fields = ['a', 'b']; + query._fields = [{name: 'a', fieldType: -1}, {name: 'b', fieldType: -1}]; var PACKET = new EventEmitter(); PACKET.type = Parser.ROW_DATA_PACKET; @@ -100,4 +102,44 @@ test(function _handlePacket() { query._handlePacket(PACKET); })(); + + function typeCast(type, strValue) { + query._fields = [{name: 'my_field', fieldType: type}]; + + var PACKET = new EventEmitter(), r; + PACKET.type = Parser.ROW_DATA_PACKET; + + gently.expect(PACKET, 'on', function (event, fn) { + assert.equal(event, 'data'); + + gently.expect(query, 'emit', function (event, row) { + assert.equal(event, 'row'); + r = row.my_field; + }); + + fn(new Buffer(strValue), 0); + }); + + query._handlePacket(PACKET); + return r; + } + + assert.deepEqual(typeCast(Query.FIELD_TYPE_TIMESTAMP, '2010-10-05 06:23:42'), new Date('2010-10-05 06:23:42Z')); + + assert.deepEqual(typeCast(Query.FIELD_TYPE_TIMESTAMP, '2010-10-05'), new Date('2010-10-05Z')); + assert.deepEqual(typeCast(Query.FIELD_TYPE_DATE, '2010-10-05'), new Date('2010-10-05Z')); + assert.deepEqual(typeCast(Query.FIELD_TYPE_DATETIME, '2010-10-05'), new Date('2010-10-05Z')); + assert.deepEqual(typeCast(Query.FIELD_TYPE_NEWDATE, '2010-10-05'), new Date('2010-10-05Z')); + + assert.strictEqual(typeCast(Query.FIELD_TYPE_TINY, '08'), 8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_SHORT, '08'), 8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_LONG, '08'), 8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_LONGLONG, '08'), 8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_INT24, '08'), 8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_YEAR, '08'), 8); + + assert.strictEqual(typeCast(Query.FIELD_TYPE_DECIMAL, '2.8'), 2.8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_FLOAT, '2.8'), 2.8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_DOUBLE, '2.8'), 2.8); + assert.strictEqual(typeCast(Query.FIELD_TYPE_NEWDECIMAL, '2.8'), 2.8); }); diff --git a/test/system/test-client-query-column-order.js b/test/system/slow/test-client-query-column-order.js similarity index 95% rename from test/system/test-client-query-column-order.js rename to test/system/slow/test-client-query-column-order.js index 19161d947..bdd33c5ac 100644 --- a/test/system/test-client-query-column-order.js +++ b/test/system/slow/test-client-query-column-order.js @@ -1,5 +1,5 @@ -require('../common'); -var REPEATS = 1000, +require('../../common'); +var REPEATS = 500, Client = require('mysql').Client, client = Client(TEST_CONFIG), fs = require('fs'), diff --git a/test/system/test-client-query.js b/test/system/test-client-query.js index 54e90b678..6a055de00 100644 --- a/test/system/test-client-query.js +++ b/test/system/test-client-query.js @@ -62,6 +62,8 @@ var query = client.query( assert.equal(results.length, 2); assert.equal(results[1].title, 'another entry'); + assert.ok(typeof results[1].id == 'number'); + assert.ok(results[1].created instanceof Date); client.end(); }) );