diff --git a/SQLBlob.coffee.md b/SQLBlob.coffee.md new file mode 100644 index 000000000..f49ea75c5 --- /dev/null +++ b/SQLBlob.coffee.md @@ -0,0 +1,295 @@ +# SQLBlob in Markdown (litcoffee) + +## Top-level objects + +### Root window object + + root = @ + +### Base64 conversion + + # Adapted from: base64-arraybuffer + # https://github.com/niklasvh/base64-arraybuffer + # Copyright (c) 2012 Niklas von Hertzen + # Licensed under the MIT license. + + BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + + arrayBufferToBase64 = (arraybuffer) -> + bytes = new Uint8Array(arraybuffer) + len = bytes.length + base64 = "" + + i = 0 + while i < len + base64 += BASE64_CHARS[bytes[i] >> 2] + base64 += BASE64_CHARS[((bytes[i] & 3) << 4) | (bytes[i + 1] >> 4)] + base64 += BASE64_CHARS[((bytes[i + 1] & 15) << 2) | (bytes[i + 2] >> 6)] + base64 += BASE64_CHARS[bytes[i + 2] & 63] + i += 3 + + if (len % 3) is 2 + base64 = base64.substring(0, base64.length - 1) + "=" + else if (len % 3) is 1 + base64 = base64.substring(0, base64.length - 2) + "==" + + return base64 + + # This direct conversion should be faster than atob() and array copy + base64ToArrayBuffer = (base64) -> + bufferLength = base64.length * 0.75 + len = base64.length + p = 0 + + if base64[base64.length - 1] is "=" + bufferLength-- + + if base64[base64.length - 2] is "=" + bufferLength-- + + arraybuffer = new ArrayBuffer(bufferLength) + bytes = new Uint8Array(arraybuffer) + + i = 0 + while i < len + encoded1 = BASE64_CHARS.indexOf(base64[i]) + encoded2 = BASE64_CHARS.indexOf(base64[i+1]) + encoded3 = BASE64_CHARS.indexOf(base64[i+2]) + encoded4 = BASE64_CHARS.indexOf(base64[i+3]) + + bytes[p++] = (encoded1 << 2) | (encoded2 >> 4) + bytes[p++] = ((encoded2 & 15) << 4) | (encoded3 >> 2) + bytes[p++] = ((encoded3 & 3) << 6) | (encoded4 & 63) + i += 4 + + return arraybuffer + +### Binary string conversion + +Binary string is a JavaScript term that basically means treat the string as a byte array. + + # Each byte from the buffer is transferred into a JavaScript UCS-2 string. + arrayBufferToBinaryString = (buffer) -> + binary = "" + bytes = new Uint8Array(buffer) + length = bytes.byteLength + i = 0 + while i < length + # code will be < 256 but expanded to 2 bytes + # when stored in a JavaScript string + binary += String.fromCharCode(bytes[i]) + ++i + return binary + + # The binary string is UCS-2 encoded, but all characters are required to be < 256. + # That is, the string must represent a series of bytes. Because UCS-2 is a 2 byte + # format, it's possible a multi-byte character was added and that this isn't + # a true "binary string", so the method throws if encountered. + binaryStringToArrayBuffer = (binary) -> + length = binary.length + buffer = new ArrayBuffer(length) + bytes = new Uint8Array(buffer) + i = 0 + while i < length + code = binary.charCodeAt(i) + if code > 255 + throw new Error("a multibyte character was encountered in the provided string which indicates it was not encoded as a binary string") + bytes[i] = code + ++i + return bytes.buffer + +### SQLBlob + +#### Summary + +SQLite determines column types for data rows during insert based on the type used to bind the value to the statement, not the type specified in the table schema. The latter is only an informational tag to indicate expected binding. + +Browser implementations convert between JS and SQL types when binding statements: + +* JS number -> SQLite integer | float +* JS UCS-2 string -> SQLite text + +Values never serialize to SQLite blobs and at best will be serialized as text. Some implementations serialize text using UTF-8 and some UTF-16. + +In a Web SQL environment, it's up to the caller to determine how to encode binary into a string format that can be serialized and deserialized given those conversions. + +One can use a "binary string" which is a JS UCS-2 string where every real byte of data has an extra byte of padding internally, but unfortunately not all browser implementations support all Unicode values from end to end. Some interpret `\u0000` as a null terminator when reading back the string from their internal SQLite implementations which results in truncated data. + +Additionally, Cordova invokes `JSON.stringify()` before sending data to the native layer. See the [JSON spec](http://json.org/) for what that does to strings; generally any Unicode character is allowed except `"` and `\` which will be escaped. A JSON parser like `JSON.parse()` will handle unescaping. + +Even though browser Web SQL implementations only persist binary data as text, it is useful in Cordova scenarios to be able to consume or produce databases that conform to external specifications that make use of blobs and raw binary data. + +The `SQLBlob` type below abstracts these problems by making it: + +1. Easy to serialize binary data to and from an ArrayBuffer, base64 string, or binary string. +2. Possible to recognize binary data in the Cordova plugin so it can be stored as a binary blob. +3. Possible to write the same code for persisting binary data whether using the Cordova plugin or Web SQL in a browser. + +#### SQLBlob object is defined by a constructor function and prototype member functions + + class SQLBlob + # This prefix allows a Cordova native SQL plugin to recognize binary data. + SQLBLOB_URL_PREFIX = "sqlblob:" + DATA_URL_PREFIX = "data:" + SQLBLOB_URL_BASE64_ENCODING = ";base64" + SQLBLOB_URL_BASE64_PREFIX = SQLBLOB_URL_PREFIX + SQLBLOB_URL_BASE64_ENCODING + + # The blob's value is internally and externally represented as + # a SQLBlob URL which is nearly identical to a data URL: + # sqlblob:[][;charset=][;base64], + # + # If ";base64" is part of the URL then data is base64 encoded, + # otherwise it is URL encoded (percent encoded UTF-8). + # + # The former is generally better for binary and the latter for text. + # There is an encoding option to specify the default representation. + # "auto": Prefer base64 for ArrayBuffer and BinaryString, pass through encoding for URLs + # "base64": Always base64 encode + # "url": Always url encode + + @_value # the sqlblob URL with base64 or url encoded data + @_commaIndex # the comma index in the sqlblob URL separating the prefix and data regions + @_options # options like encoding + + constructor: (obj, options = { encoding: "auto" }) -> + @_options = options + + if options.encoding isnt "auto" and options.encoding isnt "url" and options.encoding isnt "base64" + throw new Error("Unknown encoding (must be 'auto', 'url', or 'base64'): " + options.encoding) + + # allow null or undefined as a passthrough + if !obj + @_value = obj + return + + if obj instanceof ArrayBuffer + if options.encoding is "base64" or options.encoding is "auto" + @_value = SQLBLOB_URL_BASE64_PREFIX + "," + arrayBufferToBase64(obj) + @_commaIndex = SQLBLOB_URL_BASE64_PREFIX.length; + else if options.encoding is "url" + # convert to percent encoded UTF-8 (good for most text, not so good for binary) + @_value = SQLBLOB_URL_PREFIX + "," + encodeURIComponent(arrayBufferToBinaryString(obj)); + @_commaIndex = SQLBLOB_URL_PREFIX.length; + else if typeof obj is "string" + # Decode SQLBlob or Data URL if detected. + # Slice is faster than indexOf. + startsWithSqlBlob = obj.slice(0, SQLBLOB_URL_PREFIX.length) is SQLBLOB_URL_PREFIX + startsWithData = obj.slice(0, DATA_URL_PREFIX.length) is DATA_URL_PREFIX + + # verify supported format + if not startsWithSqlBlob and not startsWithData + throw new Error("Only 'sqlblob' and 'data' URI strings are supported") + + # convert data to sqlblob + if startsWithData + obj = SQLBLOB_URL_PREFIX + obj.slice(DATA_URL_PREFIX.length) + + # find comma dividing prefix and data regions + @_commaIndex = commaIndex = obj.indexOf(",") + throw new Error("Missing comma in SQLBlob URL") if commaIndex is -1 + + # test for base64 + isBase64 = obj.slice(0, commaIndex).indexOf(SQLBLOB_URL_BASE64_ENCODING) isnt -1 + + # assign value + if options.encoding is "auto" + @_value = obj # save the sqlblob verbatim + else if options.encoding is "base64" + if isBase64 + @_value = obj # save the sqlblob verbatim + else + # take the percent encoded UTF-8, unescape it to get a byte string, then base64 encode + prefix = obj.slice(0, commaIndex) + SQLBLOB_URL_BASE64_ENCODING + "," + @_commaIndex = prefix.length - 1; + data = obj.slice(commaIndex + 1) + # use unescape here to decode to binary rather than interpret the bytes as UTF-8 + @_value = prefix + window.btoa(unescape(data)) + else if options.encoding is "url" + if not isBase64 + @_value = obj # save the url encoded sqlblob verbatim + else + # decode the base64 to binary, escape to convert bytes back into percent encoding + prefix = obj.slice(0, commaIndex + 1).replace(SQLBLOB_URL_BASE64_ENCODING, "") + @_commaIndex = prefix.length - 1; + data = obj.slice(commaIndex + 1) + @_value = prefix + encodeURIComponent(window.atob(data)) + else + throw new Error("unsupported object type (must be ArrayBuffer or string): " + typeof obj) + # TODO: Blob with FileReader + + return + + Object.defineProperties @prototype, + isBase64: + get: -> @_value.slice(0, @_commaIndex).indexOf(SQLBLOB_URL_BASE64_ENCODING) isnt -1 + + toString: () -> + return @_value # already string + + # This is for JavaScript automatic type conversion and used + # by Web SQL for serialization. + valueOf: () -> + return @_value + + toJSON: () -> + return @_value + + toArrayBuffer: () -> + return @_value if !@_value + + data = @_value.slice(@_commaIndex + 1) + + if @isBase64 + return base64ToArrayBuffer(data) + else + return binaryStringToArrayBuffer(unescape(data)) + + toBase64: () -> + return @_value if !@_value + + data = @_value.slice(@_commaIndex + 1) + + if @isBase64 + return data + else + return window.btoa(unescape(data)) + + toBinaryString: () -> + return @_value if !@_value + + data = @_value.slice(@_commaIndex + 1) + + if @isBase64 + return window.atob(data) + else + return unescape(data) + + toUnicodeString: () -> + return @_value if !@_value + + data = @_value.slice(@_commaIndex + 1) + + if @isBase64 + return decodeURIComponent(escape(window.atob(data))) + else + return decodeURIComponent(data) + + @createFromBase64: (base64, options) -> + return new SQLBlob(SQLBLOB_URL_BASE64_PREFIX + "," + base64, options) + + # All character codes must be < 256 as the string is used in place of a byte array. + @createFromBinaryString: (binary, options = { encoding: "auto" }) -> + if options.encoding is "base64" or options.encoding is "auto" + return new SQLBlob(SQLBLOB_URL_BASE64_PREFIX + "," + window.btoa(binary), options) + else if options.encoding is "url" + return new SQLBlob(SQLBLOB_URL_PREFIX + "," + encodeURIComponent(binary), options) + + # Unicode chars are converted to UTF-8 and percent encoded. If "url" encoding is not + # specified as an option, then the constructor used below will complete the + # conversion of the UTF-8 encoded string to base64. + @createFromUnicodeString: (text, options = { encoding: "auto" }) -> + return new SQLBlob(SQLBLOB_URL_PREFIX + "," + encodeURIComponent(text), options) + +### Exported API + + root.SQLBlob = SQLBlob \ No newline at end of file diff --git a/SQLitePlugin.coffee.md b/SQLitePlugin.coffee.md index 8e57745d7..af6c7a78c 100644 --- a/SQLitePlugin.coffee.md +++ b/SQLitePlugin.coffee.md @@ -320,7 +320,8 @@ License for common Javascript: MIT or Apache tropts.push qid: qid sql: request.sql - params: request.params + # only primitives are supported by Web SQL so call valueOf + params: ((if p then p.valueOf() else p) for p in request.params) i++ diff --git a/bin/test.sh b/bin/test.sh index 11e7596e6..fda37cd3b 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -34,7 +34,7 @@ if [[ $? != 0 ]]; then # run from the bin/ directory fi # compile coffeescript -coffee --no-header -cl -o ../www ../SQLitePlugin.coffee.md +coffee --no-header -cl -o ../www ../SQLitePlugin.coffee.md ../SQLBlob.coffee.md if [[ $? != 0 ]]; then echo "coffeescript compilation failed" diff --git a/plugin.xml b/plugin.xml index 74ef146a1..7be5f406c 100644 --- a/plugin.xml +++ b/plugin.xml @@ -18,6 +18,10 @@ + + + + diff --git a/src/android/org/pgsqlite/SQLitePlugin.java b/src/android/org/pgsqlite/SQLitePlugin.java index b2c2e2601..425eecc67 100755 --- a/src/android/org/pgsqlite/SQLitePlugin.java +++ b/src/android/org/pgsqlite/SQLitePlugin.java @@ -28,6 +28,7 @@ import org.apache.cordova.CallbackContext; import org.apache.cordova.CordovaPlugin; +import org.apache.http.util.EncodingUtils; import org.json.JSONArray; import org.json.JSONException; @@ -633,7 +634,28 @@ private void bindArgsToStatement(SQLiteStatement myStatement, JSONArray sqlArgs) } else if (sqlArgs.isNull(i)) { myStatement.bindNull(i + 1); } else { - myStatement.bindString(i + 1, sqlArgs.getString(i)); + String text = sqlArgs.getString(i); + + int commaIndex; + if(text.startsWith("sqlblob:") && (commaIndex = text.indexOf(',')) != -1) { + String[] mimeParts = text.substring(0, commaIndex).split(";"); + String contentType = mimeParts.length > 0 ? mimeParts[0] : null; + boolean base64 = false; + for (int j = 1; j < mimeParts.length; ++j) { + if ("base64".equalsIgnoreCase(mimeParts[j])) { + base64 = true; + } + } + String dataText = text.substring(commaIndex + 1); + + byte[] data = base64 ? Base64.decode(dataText, Base64.DEFAULT) : + EncodingUtils.getBytes(dataText, "UTF-8"); + + myStatement.bindBlob(i + 1, data); + } + else { + myStatement.bindString(i + 1, text); + } } } } @@ -732,7 +754,7 @@ private void bindPostHoneycomb(JSONObject row, String key, Cursor cur, int i) th row.put(key, cur.getDouble(i)); break; case Cursor.FIELD_TYPE_BLOB: - row.put(key, new String(Base64.encode(cur.getBlob(i), Base64.DEFAULT))); + row.put(key, "sqlblob:;base64,".concat(new String(Base64.encode(cur.getBlob(i), Base64.DEFAULT)))); break; case Cursor.FIELD_TYPE_STRING: default: /* (not expected) */ @@ -755,7 +777,7 @@ private void bindPreHoneycomb(JSONObject row, String key, Cursor cursor, int i) } else if (cursorWindow.isFloat(pos, i)) { row.put(key, cursor.getDouble(i)); } else if (cursorWindow.isBlob(pos, i)) { - row.put(key, new String(Base64.encode(cursor.getBlob(i), Base64.DEFAULT))); + row.put(key, "sqlblob:;base64,".concat(new String(Base64.encode(cursor.getBlob(i), Base64.DEFAULT)))); } else { // string row.put(key, cursor.getString(i)); } diff --git a/src/ios/SQLitePlugin.h b/src/ios/SQLitePlugin.h index 073af33df..e5b902ffc 100755 --- a/src/ios/SQLitePlugin.h +++ b/src/ios/SQLitePlugin.h @@ -12,6 +12,7 @@ #import #import +#import #import "AppDelegate.h" @@ -56,9 +57,6 @@ typedef int WebSQLError; +(int)mapSQLiteErrorCode:(int)code; -// LIBB64 -+(id) getBlobAsBase64String:(const char*) blob_chars - withlength: (int) blob_length; -// LIBB64---END - ++(NSString*)getBlobAsBase64String:(const char*) blob_chars + withlength:(int) blob_length; @end diff --git a/src/ios/SQLitePlugin.m b/src/ios/SQLitePlugin.m index 8f092fe48..f6224fa48 100755 --- a/src/ios/SQLitePlugin.m +++ b/src/ios/SQLitePlugin.m @@ -9,128 +9,6 @@ #import "SQLitePlugin.h" #include - -//LIBB64 -typedef enum -{ - step_A, step_B, step_C -} base64_encodestep; - -typedef struct -{ - base64_encodestep step; - char result; - int stepcount; -} base64_encodestate; - -static void base64_init_encodestate(base64_encodestate* state_in) -{ - state_in->step = step_A; - state_in->result = 0; - state_in->stepcount = 0; -} - -static char base64_encode_value(char value_in) -{ - static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - if (value_in > 63) return '='; - return encoding[(int)value_in]; -} - -static int base64_encode_block(const char* plaintext_in, - int length_in, - char* code_out, - base64_encodestate* state_in, - int line_length) -{ - const char* plainchar = plaintext_in; - const char* const plaintextend = plaintext_in + length_in; - char* codechar = code_out; - char result; - char fragment; - - result = state_in->result; - - switch (state_in->step) - { - while (1) - { - case step_A: - if (plainchar == plaintextend) - { - state_in->result = result; - state_in->step = step_A; - return codechar - code_out; - } - fragment = *plainchar++; - result = (fragment & 0x0fc) >> 2; - *codechar++ = base64_encode_value(result); - result = (fragment & 0x003) << 4; - case step_B: - if (plainchar == plaintextend) - { - state_in->result = result; - state_in->step = step_B; - return codechar - code_out; - } - fragment = *plainchar++; - result |= (fragment & 0x0f0) >> 4; - *codechar++ = base64_encode_value(result); - result = (fragment & 0x00f) << 2; - case step_C: - if (plainchar == plaintextend) - { - state_in->result = result; - state_in->step = step_C; - return codechar - code_out; - } - fragment = *plainchar++; - result |= (fragment & 0x0c0) >> 6; - *codechar++ = base64_encode_value(result); - result = (fragment & 0x03f) >> 0; - *codechar++ = base64_encode_value(result); - - if(line_length > 0) - { - ++(state_in->stepcount); - if (state_in->stepcount == line_length/4) - { - *codechar++ = '\n'; - state_in->stepcount = 0; - } - } - } - } - /* control should not reach here */ - return codechar - code_out; -} - -static int base64_encode_blockend(char* code_out, - base64_encodestate* state_in) -{ - char* codechar = code_out; - - switch (state_in->step) - { - case step_B: - *codechar++ = base64_encode_value(state_in->result); - *codechar++ = '='; - *codechar++ = '='; - break; - case step_C: - *codechar++ = base64_encode_value(state_in->result); - *codechar++ = '='; - break; - case step_A: - break; - } - *codechar++ = '\n'; - - return codechar - code_out; -} - -//LIBB64---END - static void sqlite_regexp(sqlite3_context* context, int argc, sqlite3_value** values) { int ret; regex_t regex; @@ -217,7 +95,7 @@ -(void)open: (CDVInvokedUrlCommand*)command // const char *key = [@"your_key_here" UTF8String]; // if(key != NULL) sqlite3_key(db, key, strlen(key)); - sqlite3_create_function(db, "regexp", 2, SQLITE_ANY, NULL, &sqlite_regexp, NULL, NULL); + sqlite3_create_function(db, "regexp", 2, SQLITE_ANY, NULL, &sqlite_regexp, NULL, NULL); // Attempt to read the SQLite master table (test for SQLCipher version): if(sqlite3_exec(db, (const char*)"SELECT count(*) FROM sqlite_master;", NULL, NULL, NULL) == SQLITE_OK) { @@ -427,10 +305,8 @@ -(CDVPluginResult*) executeSqlWithDict: (NSMutableDictionary*)options andArgs: ( #endif break; case SQLITE_BLOB: - //LIBB64 columnValue = [SQLitePlugin getBlobAsBase64String: sqlite3_column_blob(statement, i) - withlength: sqlite3_column_bytes(statement, i) ]; - //LIBB64---END + withLength: sqlite3_column_bytes(statement, i)]; break; case SQLITE_FLOAT: columnValue = [NSNumber numberWithFloat: sqlite3_column_double(statement, i)]; @@ -506,9 +382,25 @@ -(void)bindStatement:(sqlite3_stmt *)statement withArg:(NSObject *)arg atIndex:( } else { stringArg = [arg description]; // convert to text } - - NSData *data = [stringArg dataUsingEncoding:NSUTF8StringEncoding]; - sqlite3_bind_text(statement, argIndex, data.bytes, data.length, SQLITE_TRANSIENT); + + // If the string is a sqlblob URI then decode it and store the binary directly. + // + // A sqlblob URI is formatted similar to a data URI which makes it easy to convert: + // sqlblob:[][;charset=][;base64], + // + // The reason the `sqlblob` prefix is used instead of `data` is because + // applications may want to use data URI strings directly, so the + // `sqlblob` prefix disambiguates the desired behavior. + if([stringArg hasPrefix:@"sqlblob:"]) { + // convert to data URI, decode, store as blob + stringArg = [stringArg stringByReplacingCharactersInRange:NSMakeRange(0,7) withString:@"data"]; + NSData *data = [NSData dataWithContentsOfURL: [NSURL URLWithString:stringArg]]; + sqlite3_bind_blob(statement, argIndex, data.bytes, data.length, SQLITE_TRANSIENT); + } + else { + NSData *data = [stringArg dataUsingEncoding:NSUTF8StringEncoding]; + sqlite3_bind_text(statement, argIndex, data.bytes, data.length, SQLITE_TRANSIENT); + } } } @@ -574,32 +466,21 @@ +(int)mapSQLiteErrorCode:(int)code } } -+(id) getBlobAsBase64String:(const char*) blob_chars - withlength: (int) blob_length ++(NSString*)getBlobAsBase64String:(const char*)blob_chars + withLength:(int)blob_length { - base64_encodestate b64state; - - base64_init_encodestate(&b64state); - - //2* ensures 3 bytes -> 4 Base64 characters + null for NSString init - char* code = malloc (2*blob_length*sizeof(char)); - - int codelength; - int endlength; - - codelength = base64_encode_block(blob_chars,blob_length,code,&b64state,0); - - endlength = base64_encode_blockend(&code[codelength], &b64state); - - //Adding in a null in order to use initWithUTF8String, expecting null terminated char* string - code[codelength+endlength] = '\0'; - - NSString* result = [NSString stringWithUTF8String: code]; - - free(code); - - return result; + size_t outputLength = 0; + char* outputBuffer = CDVNewBase64Encode(blob_chars, blob_length, true, &outputLength); + + NSString* result = [[NSString alloc] initWithBytesNoCopy:outputBuffer + length:outputLength + encoding:NSASCIIStringEncoding + freeWhenDone:YES]; +#if !__has_feature(objc_arc) + [result autorelease]; +#endif + + return [NSString stringWithFormat:@"sqlblob:;base64,%@", result]; } - @end diff --git a/test-www/www/index.html b/test-www/www/index.html index 4ab0ab854..6f4532990 100755 --- a/test-www/www/index.html +++ b/test-www/www/index.html @@ -10,6 +10,7 @@ +