From ec00c6a516fd19c1f6d78a82656c8b19c24ccb6c Mon Sep 17 00:00:00 2001 From: Aaron Oneal Date: Wed, 31 Dec 2014 19:38:08 -0800 Subject: [PATCH] Support SQL blob marshaling using ArrayBuffer and Base64. Web SQL doesn't actually support storing binary data in SQL blobs using the SQLite blob methods. It serializes binary data using strings with the SQLite text methods which results in data not being stored as a blob. This can be problematic in Cordova if you need to work with existing SQLite databases. This change adds more robust support for binary serialization. ArrayBuffer values can be used in statements and these will be converted to a `sqlblob` URI with base64 data that is unpacked on the native side and stored as a proper binary blob. When reading a blob back from the native side, the previous behavior was to return the blob as base64. Now, the blob is returned as a `sqlblob` object which can then be unpacked in JavaScript to base64, ArrayBuffer, or binary string. tx.executeSql("SELECT foo FROM test_table", [], function(tx, res) { var blob = res.rows.item(0).foo; if (blob.isBlob) { blob.toBase64(); blob.toBinaryString(); blob.toArrayBuffer(); } }); Tests were added to verify the behavior and to demonstrate usage. Only iOS and Android were updated. Windows Phone did not previously have blob support; saving for a future update. --- SQLitePlugin.coffee.md | 115 ++++++++++++- src/android/org/pgsqlite/SQLitePlugin.java | 4 +- src/ios/SQLitePlugin.h | 8 +- src/ios/SQLitePlugin.m | 189 ++++----------------- test-www/www/index.html | 134 ++++++++++++++- 5 files changed, 285 insertions(+), 165 deletions(-) diff --git a/SQLitePlugin.coffee.md b/SQLitePlugin.coffee.md index 228e808d2..42364712c 100644 --- a/SQLitePlugin.coffee.md +++ b/SQLitePlugin.coffee.md @@ -17,6 +17,65 @@ License for common Javascript: MIT or Apache ### global(s): txLocks = {} + +### base64 conversion + + # Adapted from: + # base64-arraybuffer + # https://github.com/niklasvh/base64-arraybuffer + # Copyright (c) 2012 Niklas von Hertzen + # Licensed under the MIT license. + + BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + + arrayBufferToBase64 = (arraybuffer) -> + bytes = new Uint8Array(arraybuffer) + len = bytes.length + base64 = "" + + i = 0 + while i < len + base64 += BASE64_CHARS[bytes[i] >> 2] + base64 += BASE64_CHARS[((bytes[i] & 3) << 4) | (bytes[i + 1] >> 4)] + base64 += BASE64_CHARS[((bytes[i + 1] & 15) << 2) | (bytes[i + 2] >> 6)] + base64 += BASE64_CHARS[bytes[i + 2] & 63] + i += 3 + + if (len % 3) is 2 + base64 = base64.substring(0, base64.length - 1) + "=" + else if (len % 3) is 1 + base64 = base64.substring(0, base64.length - 2) + "==" + + return base64 + + # This direct conversion should be faster than atob() and array copy + base64ToArrayBuffer = (base64) -> + bufferLength = base64.length * 0.75 + len = base64.length + p = 0 + + if base64[base64.length - 1] is "=" + bufferLength-- + + if base64[base64.length - 2] is "=" + bufferLength-- + + arraybuffer = new ArrayBuffer(bufferLength) + bytes = new Uint8Array(arraybuffer) + + i = 0 + while i < len + encoded1 = BASE64_CHARS.indexOf(base64[i]) + encoded2 = BASE64_CHARS.indexOf(base64[i+1]) + encoded3 = BASE64_CHARS.indexOf(base64[i+2]) + encoded4 = BASE64_CHARS.indexOf(base64[i+3]) + + bytes[p++] = (encoded1 << 2) | (encoded2 >> 4) + bytes[p++] = ((encoded2 & 15) << 4) | (encoded3 >> 2) + bytes[p++] = ((encoded3 & 3) << 6) | (encoded4 & 63) + i += 4 + + return arraybuffer; ### utility function(s): @@ -43,6 +102,50 @@ License for common Javascript: MIT or Apache return sqlError + # Converts JS types to something that can be marshaled to native. + # + # Cordova handles the basics, but this will convert ArrayBuffer + # instances to a format that the native side will know to treat as + # binary. That is, parse and convert back from base64 to bind as + # a blob. + # + # The specific format is similar to a data URI. + # + # Data URI is not used directly because databases may want to work + # with them as strings. + convertJsToNativeParam = (param) -> + if typeof ArrayBuffer is "undefined" + return param + + if param instanceof ArrayBuffer + return "sqlblob:;base64," + arrayBufferToBase64(param) + else + return param + + # Converts native types back to JS. + # + # Web SQL does not support Blob binding but the plugin does. + # + # The Cordova layer generally URL or base64 encodes binary data + # to get it to and from the native side. The plugin does this + # itself so it can mark the data as a blob. + # + # When JavaScript reads the returned blob we have the option of + # splitting out the base64 or converting to an ArrayBuffer. + # + # Below makes assumptions about the format based on what the + # native layer passes. + convertNativeToJsParam = (param) -> + if typeof param is "string" && param.indexOf("sqlblob:;base64,") is 0 + base64string = param.slice(16) + blob = + toArrayBuffer: () -> base64ToArrayBuffer(base64string) + toBinaryString: () -> atob(base64string) + toBase64: () -> base64string + isBlob: true + return blob + return param + nextTick = window.setImmediate || (fun) -> window.setTimeout(fun, 0) return @@ -235,6 +338,12 @@ License for common Javascript: MIT or Apache return rows = response.rows || [] + + # Convert parameters if needed + for row in rows + for own colName, colValue of row + row[colName] = convertNativeToJsParam(colValue) + payload = rows: item: (i) -> @@ -270,7 +379,7 @@ License for common Javascript: MIT or Apache try if didSucceed tx.handleStatementSuccess batchExecutes[index].success, response - else + else tx.handleStatementFailure batchExecutes[index].error, toSQLError(response) catch err txFailure = toSQLError(err) unless txFailure @@ -305,7 +414,7 @@ License for common Javascript: MIT or Apache tropts.push qid: qid sql: request.sql - params: request.params + params: (convertJsToNativeParam param for param in request.params) i++ @@ -415,7 +524,7 @@ License for common Javascript: MIT or Apache new SQLitePlugin openargs, okcb, errorcb deleteDb: (databaseName, success, error) -> - delete SQLitePlugin::openDBs[databaseName] + delete SQLitePlugin::openDBs[databaseName] onError = (errMessage) -> if !!error then error new Error("error while trying to delete database: " + errMessage) cordova.exec success, onError, "SQLitePlugin", "delete", [{ path: databaseName }] diff --git a/src/android/org/pgsqlite/SQLitePlugin.java b/src/android/org/pgsqlite/SQLitePlugin.java index b2c2e2601..b4b9d572b 100755 --- a/src/android/org/pgsqlite/SQLitePlugin.java +++ b/src/android/org/pgsqlite/SQLitePlugin.java @@ -732,7 +732,7 @@ private void bindPostHoneycomb(JSONObject row, String key, Cursor cur, int i) th row.put(key, cur.getDouble(i)); break; case Cursor.FIELD_TYPE_BLOB: - row.put(key, new String(Base64.encode(cur.getBlob(i), Base64.DEFAULT))); + row.put(key, "sqlblob:;base64,".concat(new String(Base64.encode(cur.getBlob(i), Base64.DEFAULT)))); break; case Cursor.FIELD_TYPE_STRING: default: /* (not expected) */ @@ -755,7 +755,7 @@ private void bindPreHoneycomb(JSONObject row, String key, Cursor cursor, int i) } else if (cursorWindow.isFloat(pos, i)) { row.put(key, cursor.getDouble(i)); } else if (cursorWindow.isBlob(pos, i)) { - row.put(key, new String(Base64.encode(cursor.getBlob(i), Base64.DEFAULT))); + row.put(key, "sqlblob:;base64,".concat(new String(Base64.encode(cursor.getBlob(i), Base64.DEFAULT)))); } else { // string row.put(key, cursor.getString(i)); } diff --git a/src/ios/SQLitePlugin.h b/src/ios/SQLitePlugin.h index 073af33df..e5b902ffc 100755 --- a/src/ios/SQLitePlugin.h +++ b/src/ios/SQLitePlugin.h @@ -12,6 +12,7 @@ #import #import +#import #import "AppDelegate.h" @@ -56,9 +57,6 @@ typedef int WebSQLError; +(int)mapSQLiteErrorCode:(int)code; -// LIBB64 -+(id) getBlobAsBase64String:(const char*) blob_chars - withlength: (int) blob_length; -// LIBB64---END - ++(NSString*)getBlobAsBase64String:(const char*) blob_chars + withlength:(int) blob_length; @end diff --git a/src/ios/SQLitePlugin.m b/src/ios/SQLitePlugin.m index 8f092fe48..bc8bd31a0 100755 --- a/src/ios/SQLitePlugin.m +++ b/src/ios/SQLitePlugin.m @@ -9,128 +9,6 @@ #import "SQLitePlugin.h" #include - -//LIBB64 -typedef enum -{ - step_A, step_B, step_C -} base64_encodestep; - -typedef struct -{ - base64_encodestep step; - char result; - int stepcount; -} base64_encodestate; - -static void base64_init_encodestate(base64_encodestate* state_in) -{ - state_in->step = step_A; - state_in->result = 0; - state_in->stepcount = 0; -} - -static char base64_encode_value(char value_in) -{ - static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - if (value_in > 63) return '='; - return encoding[(int)value_in]; -} - -static int base64_encode_block(const char* plaintext_in, - int length_in, - char* code_out, - base64_encodestate* state_in, - int line_length) -{ - const char* plainchar = plaintext_in; - const char* const plaintextend = plaintext_in + length_in; - char* codechar = code_out; - char result; - char fragment; - - result = state_in->result; - - switch (state_in->step) - { - while (1) - { - case step_A: - if (plainchar == plaintextend) - { - state_in->result = result; - state_in->step = step_A; - return codechar - code_out; - } - fragment = *plainchar++; - result = (fragment & 0x0fc) >> 2; - *codechar++ = base64_encode_value(result); - result = (fragment & 0x003) << 4; - case step_B: - if (plainchar == plaintextend) - { - state_in->result = result; - state_in->step = step_B; - return codechar - code_out; - } - fragment = *plainchar++; - result |= (fragment & 0x0f0) >> 4; - *codechar++ = base64_encode_value(result); - result = (fragment & 0x00f) << 2; - case step_C: - if (plainchar == plaintextend) - { - state_in->result = result; - state_in->step = step_C; - return codechar - code_out; - } - fragment = *plainchar++; - result |= (fragment & 0x0c0) >> 6; - *codechar++ = base64_encode_value(result); - result = (fragment & 0x03f) >> 0; - *codechar++ = base64_encode_value(result); - - if(line_length > 0) - { - ++(state_in->stepcount); - if (state_in->stepcount == line_length/4) - { - *codechar++ = '\n'; - state_in->stepcount = 0; - } - } - } - } - /* control should not reach here */ - return codechar - code_out; -} - -static int base64_encode_blockend(char* code_out, - base64_encodestate* state_in) -{ - char* codechar = code_out; - - switch (state_in->step) - { - case step_B: - *codechar++ = base64_encode_value(state_in->result); - *codechar++ = '='; - *codechar++ = '='; - break; - case step_C: - *codechar++ = base64_encode_value(state_in->result); - *codechar++ = '='; - break; - case step_A: - break; - } - *codechar++ = '\n'; - - return codechar - code_out; -} - -//LIBB64---END - static void sqlite_regexp(sqlite3_context* context, int argc, sqlite3_value** values) { int ret; regex_t regex; @@ -217,7 +95,7 @@ -(void)open: (CDVInvokedUrlCommand*)command // const char *key = [@"your_key_here" UTF8String]; // if(key != NULL) sqlite3_key(db, key, strlen(key)); - sqlite3_create_function(db, "regexp", 2, SQLITE_ANY, NULL, &sqlite_regexp, NULL, NULL); + sqlite3_create_function(db, "regexp", 2, SQLITE_ANY, NULL, &sqlite_regexp, NULL, NULL); // Attempt to read the SQLite master table (test for SQLCipher version): if(sqlite3_exec(db, (const char*)"SELECT count(*) FROM sqlite_master;", NULL, NULL, NULL) == SQLITE_OK) { @@ -427,10 +305,8 @@ -(CDVPluginResult*) executeSqlWithDict: (NSMutableDictionary*)options andArgs: ( #endif break; case SQLITE_BLOB: - //LIBB64 columnValue = [SQLitePlugin getBlobAsBase64String: sqlite3_column_blob(statement, i) - withlength: sqlite3_column_bytes(statement, i) ]; - //LIBB64---END + withLength: sqlite3_column_bytes(statement, i)]; break; case SQLITE_FLOAT: columnValue = [NSNumber numberWithFloat: sqlite3_column_double(statement, i)]; @@ -506,9 +382,25 @@ -(void)bindStatement:(sqlite3_stmt *)statement withArg:(NSObject *)arg atIndex:( } else { stringArg = [arg description]; // convert to text } - - NSData *data = [stringArg dataUsingEncoding:NSUTF8StringEncoding]; - sqlite3_bind_text(statement, argIndex, data.bytes, data.length, SQLITE_TRANSIENT); + + // If the string is a sqlblob URI then decode it and store the binary directly. + // + // A sqlblob URI is formatted similar to a data URI which makes it easy to convert: + // sqlblob:[][;charset=][;base64], + // + // The reason the `sqlblob` prefix is used instead of `data` is because + // applications may want to use data URI strings directly, so the + // `sqlblob` prefix disambiguates the desired behavior. + NSRange base64Range; + + if([stringArg hasPrefix:@"sqlblob:"] && (base64Range = [stringArg rangeOfString:@";base64,"]).location != NSNotFound) { + NSData *data = [NSData dataFromBase64String:[stringArg substringFromIndex:base64Range.location + 8]]; + sqlite3_bind_blob(statement, argIndex, data.bytes, data.length, SQLITE_TRANSIENT); + } + else { + NSData *data = [stringArg dataUsingEncoding:NSUTF8StringEncoding]; + sqlite3_bind_text(statement, argIndex, data.bytes, data.length, SQLITE_TRANSIENT); + } } } @@ -574,32 +466,21 @@ +(int)mapSQLiteErrorCode:(int)code } } -+(id) getBlobAsBase64String:(const char*) blob_chars - withlength: (int) blob_length ++(NSString*)getBlobAsBase64String:(const char*)blob_chars + withLength:(int)blob_length { - base64_encodestate b64state; - - base64_init_encodestate(&b64state); - - //2* ensures 3 bytes -> 4 Base64 characters + null for NSString init - char* code = malloc (2*blob_length*sizeof(char)); - - int codelength; - int endlength; - - codelength = base64_encode_block(blob_chars,blob_length,code,&b64state,0); - - endlength = base64_encode_blockend(&code[codelength], &b64state); - - //Adding in a null in order to use initWithUTF8String, expecting null terminated char* string - code[codelength+endlength] = '\0'; - - NSString* result = [NSString stringWithUTF8String: code]; - - free(code); - - return result; + size_t outputLength = 0; + char* outputBuffer = CDVNewBase64Encode(blob_chars, blob_length, true, &outputLength); + + NSString* result = [[NSString alloc] initWithBytesNoCopy:outputBuffer + length:outputLength + encoding:NSASCIIStringEncoding + freeWhenDone:YES]; +#if !__has_feature(objc_arc) + [result autorelease]; +#endif + + return [NSString stringWithFormat:@"sqlblob:;base64,%@", result]; } - @end diff --git a/test-www/www/index.html b/test-www/www/index.html index bc40aed59..484949db0 100755 --- a/test-www/www/index.html +++ b/test-www/www/index.html @@ -591,7 +591,139 @@ }); }); }); + + // Web SQL does not support true blob serialization, so this is a Plugin only supported scenario + if (typeof ArrayBuffer !== "undefined" && !isWebSql) test(suiteName + "ArrayBuffer to SQL Blob and back as Base64", function() { + var db = openDatabase("Blob-test.db", "1.0", "Demo", DEFAULT_SIZE); + ok(!!db, "db object"); + stop(2); + + db.transaction(function(tx) { + ok(!!tx, "tx object"); + + var buffer = new ArrayBuffer(5); + var view = new Uint8Array(buffer); + view[0] = 'h'.charCodeAt(); + view[1] = 'e'.charCodeAt(); + view[2] = 'l'.charCodeAt(); + view[3] = 'l'.charCodeAt(); + view[4] = 'o'.charCodeAt(); + + tx.executeSql('DROP TABLE IF EXISTS test_table'); + tx.executeSql('CREATE TABLE IF NOT EXISTS test_table (foo blob)'); + tx.executeSql('INSERT INTO test_table VALUES (?)', [view.buffer], function(tx, res) { + strictEqual(res.rowsAffected, 1, "res.rowsAffected === 1"); + tx.executeSql("SELECT * FROM test_table", [], function(tx, res) { + var foo = res.rows.item(0).foo; + ok(foo.isBlob, "check result is a blob"); + ok(!!foo.toBase64, "check result converts to base64"); + if(!!foo.toBase64) { + var base64 = foo.toBase64() + strictEqual(base64, "aGVsbG8=", "check blob is returned as base64 encode of bytes"); + strictEqual(atob(base64), "hello", "check base64 decode is accurate"); + } + start(1); + }); + start(1); + }); + start(1); + }, function(err) { + ok(false, err.message); + start(3); + }); + }); + + // Web SQL does not support true blob serialization, so this is a Plugin only supported scenario + if (typeof ArrayBuffer !== "undefined" && !isWebSql) test(suiteName + "ArrayBuffer to SQL Blob and back as binary string", function() { + var db = openDatabase("Blob-test.db", "1.0", "Demo", DEFAULT_SIZE); + ok(!!db, "db object"); + stop(2); + db.transaction(function(tx) { + ok(!!tx, "tx object"); + + var buffer = new ArrayBuffer(5); + var view = new Uint8Array(buffer); + view[0] = 'h'.charCodeAt(); + view[1] = 'e'.charCodeAt(); + view[2] = 'l'.charCodeAt(); + view[3] = 'l'.charCodeAt(); + view[4] = 'o'.charCodeAt(); + + tx.executeSql('DROP TABLE IF EXISTS test_table'); + tx.executeSql('CREATE TABLE IF NOT EXISTS test_table (foo blob)'); + tx.executeSql('INSERT INTO test_table VALUES (?)', [view.buffer], function(tx, res) { + strictEqual(res.rowsAffected, 1, "res.rowsAffected === 1"); + tx.executeSql("SELECT * FROM test_table", [], function(tx, res) { + var foo = res.rows.item(0).foo; + ok(foo.isBlob, "check result is a blob"); + ok(!!foo.toBinaryString, "check result converts to binary string"); + if(!!foo.toBinaryString) { + var binary = foo.toBinaryString() + strictEqual(binary, "hello", "check blob is returned as string encode of bytes"); + } + start(1); + }); + start(1); + }); + start(1); + }, function(err) { + ok(false, err.message); + start(3); + }); + }); + + // Web SQL does not support true blob serialization, so this is a Plugin only supported scenario + if (typeof ArrayBuffer !== "undefined" && !isWebSql) test(suiteName + "ArrayBuffer to SQL Blob and back as ArrayBuffer", function() { + var db = openDatabase("Blob-test.db", "1.0", "Demo", DEFAULT_SIZE); + ok(!!db, "db object"); + stop(2); + + db.transaction(function(tx) { + ok(!!tx, "tx object"); + + var buffer = new ArrayBuffer(5); + var view = new Uint8Array(buffer); + view[0] = 'h'.charCodeAt(); + view[1] = 'e'.charCodeAt(); + view[2] = 'l'.charCodeAt(); + view[3] = 'l'.charCodeAt(); + view[4] = 'o'.charCodeAt(); + + tx.executeSql('DROP TABLE IF EXISTS test_table'); + tx.executeSql('CREATE TABLE IF NOT EXISTS test_table (foo blob)'); + tx.executeSql('INSERT INTO test_table VALUES (?)', [view.buffer], function(tx, res) { + strictEqual(res.rowsAffected, 1, "res.rowsAffected === 1"); + tx.executeSql("SELECT * FROM test_table", [], function(tx, res) { + var foo = res.rows.item(0).foo; + ok(foo.isBlob, "check result is a blob"); + ok(!!foo.toArrayBuffer, "check result converts to arraybuffer"); + + if(!!foo.toArrayBuffer) { + var arrayBuffer = foo.toArrayBuffer(); + ok(arrayBuffer instanceof ArrayBuffer, "check arraybuffer type is valid"); + if(arrayBuffer instanceof ArrayBuffer) { + strictEqual(arrayBuffer.byteLength, 5, "check result is correct length: " + JSON.stringify(arrayBuffer)); + + var resultView = new Uint8Array(arrayBuffer) + ok(resultView[0] == 'h'.charCodeAt(), "check resultView[0] == 'h'"); + ok(resultView[1] == 'e'.charCodeAt(), "check resultView[1] == 'e'"); + ok(resultView[2] == 'l'.charCodeAt(), "check resultView[2] == 'l'"); + ok(resultView[3] == 'l'.charCodeAt(), "check resultView[3] == 'l'"); + ok(resultView[4] == 'o'.charCodeAt(), "check resultView[4] == 'o'"); + } + } + start(1); + }); + start(1); + }); + start(1); + }, function(err) { + ok(false, err.message); + start(3); + }); + }); + // This test shows that the plugin does not throw an error when trying to serialize // an unsupported parameter type. Blob becomes an empty dictionary on iOS, for example, // and so this verifies the type is converted to a string and continues. Web SQL does @@ -901,7 +1033,7 @@ }); }); } - + if (!/MSIE/.test(navigator.userAgent)) test(suiteName + ' returns unicode correctly', function () { stop();