From 15360d50abd6645a48f93079d96a34755ddbe17c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 4 Feb 2015 14:43:19 -0800 Subject: [PATCH] emscripten_idb_* API; #3169 --- .../docs/api_reference/emscripten.h.rst | 113 ++++++++++---- src/library_idbstore.js | 138 ++++++++++++++++++ src/modules.js | 1 + system/include/emscripten/emscripten.h | 10 +- tests/idbstore.c | 69 +++++++++ tests/test_browser.py | 6 + 6 files changed, 310 insertions(+), 27 deletions(-) create mode 100644 src/library_idbstore.js create mode 100644 tests/idbstore.c diff --git a/site/source/docs/api_reference/emscripten.h.rst b/site/source/docs/api_reference/emscripten.h.rst index 6af650e89..f14777111 100644 --- a/site/source/docs/api_reference/emscripten.h.rst +++ b/site/source/docs/api_reference/emscripten.h.rst @@ -454,7 +454,7 @@ Functions In addition to fetching the URL from the network, the contents are prepared so that the data is usable in ``IMG_Load`` and so forth (we asynchronously do the work to make the browser decode the image or audio etc.). - When file is ready the ``onload`` callback will be called. If any error occurs ``onerror`` will be called. The callbacks are called with the file as their argument. + When the file is ready the ``onload`` callback will be called. If any error occurs ``onerror`` will be called. The callbacks are called with the file as their argument. :param const char* url: The URL to load. :param const char* file: The name of the file created and loaded from the URL. If the file already exists it will be overwritten. @@ -476,7 +476,7 @@ Functions Instead of writing to a file, this function writes to a buffer directly in memory. This avoids the overhead of using the emulated file system; note however that since files are not used, it cannot do the 'prepare' stage to set things up for ``IMG_Load`` and so forth (``IMG_Load`` etc. work on files). - When file is ready then the ``onload`` callback will be called. If any error occurred ``onerror`` will be called. The callbacks are called with the file as their argument. + When the file is ready then the ``onload`` callback will be called. If any error occurred ``onerror`` will be called. :param url: The URL of the file to load. :type url: const char* @@ -578,30 +578,6 @@ Functions :param int handle: A handle to request to be aborted. - -.. c:function:: int emscripten_async_prepare(const char* file, em_str_callback_func onload, em_str_callback_func onerror) - - Prepares a file asynchronously. - - This does just the preparation part of :c:func:`emscripten_async_wget`. That is, it works on file data already present and performs any required asynchronous operations (for example, decoding images for use in ``IMG_Load``, decoding audio for use in ``Mix_LoadWAV``, etc.). - - Once the operations are complete (the file is prepared), the ``onload`` callback will be called. If any error occurs ``onerror`` will be called. The callbacks are called with the file as their argument. - - :param file: The name of the file to prepare. - :type file: const char* - :param em_str_callback_func onload: Callback on successful preparation of the file. The callback function parameter value is: - - - *(const char*)* : The name of the ``file`` that was prepared. - - :param em_str_callback_func onerror: Callback in the event of failure. The callback function parameter value is: - - - *(const char*)* : The name of the ``file`` for which the prepare failed. - - :return: 0 if successful, -1 if the file does not exist - :rtype: int - - - .. c:function:: void emscripten_async_prepare_data(char* data, int size, const char *suffix, void *arg, em_async_prepare_data_onload_func onload, em_arg_callback_func onerror) Prepares a buffer of data asynchronously. This is a "data" version of :c:func:`emscripten_async_prepare`, which receives raw data as input instead of a filename (this can prevent the need to write data to a file first). @@ -624,6 +600,91 @@ Functions - *(void*)* : A pointer to ``arg`` (user defined data). +Emscripten Asynchronous IndexedDB API +===================================== + + IndexedDB is a browser API that lets you store data persistently, that is, you can save data there and load it later when the user re-visits the web page. IDBFS provides one way to use IndexedDB, through the Emscripten filesystem layer. The ``emscripten_idb_*`` methods listed here provide an alternative API, directly to IndexedDB, thereby avoiding the overhead of the filesystem layer. + +.. c:function:: void emscripten_idb_async_load(const char *db_name, const char *file_id, void* arg, em_async_wget_onload_func onload, em_arg_callback_func onerror) + + Loads data from local IndexedDB storage asynchronously. This allows use of persistent data, without the overhead of the filesystem layer. + + When the data is ready then the ``onload`` callback will be called. If any error occurred ``onerror`` will be called. + + :param db_name: The IndexedDB database from which to load. + :param file_id: The identifier of the data to load. + :param void* arg: User-defined data that is passed to the callbacks, untouched by the API itself. This may be be used by a callback to identify the associated call. + :param em_async_wget_onload_func onload: Callback on successful load of the URL into the buffer. The callback function parameter values are: + + - *(void*)* : A pointer to ``arg`` (user defined data). + - *(void*)* : A pointer to a buffer with the data. Note that, as with the worker API, the data buffer only lives during the callback; it must be used or copied during that time. + - *(int)* : The size of the buffer, in bytes. + + :param em_arg_callback_func onerror: Callback in the event of failure. The callback function parameter values are: + + - *(void*)* : A pointer to ``arg`` (user defined data). + +.. c:function:: void emscripten_idb_async_store(const char *db_name, const char *file_id, void* ptr, int num, void* arg, em_arg_callback_func onstore, em_arg_callback_func onerror); + + Stores data to local IndexedDB storage asynchronously. This allows use of persistent data, without the overhead of the filesystem layer. + + When the data has been stored then the ``onstore`` callback will be called. If any error occurred ``onerror`` will be called. + + :param db_name: The IndexedDB database from which to load. + :param file_id: The identifier of the data to load. + :param ptr: A pointer to the data to store. + :param num: How many bytes to store. + :param void* arg: User-defined data that is passed to the callbacks, untouched by the API itself. This may be be used by a callback to identify the associated call. + :param em_async_wget_onload_func onload: Callback on successful load of the URL into the buffer. The callback function parameter values are: + + - *(void*)* : A pointer to ``arg`` (user defined data). + + :param em_arg_callback_func onerror: Callback in the event of failure. The callback function parameter values are: + + - *(void*)* : A pointer to ``arg`` (user defined data). + +.. c:function:: void emscripten_idb_async_delete(const char *db_name, const char *file_id, void* arg, em_arg_callback_func ondelete, em_arg_callback_func onerror) + + Deletes data from local IndexedDB storage asynchronously. + + When the data has been deleted then the ``ondelete`` callback will be called. If any error occurred ``onerror`` will be called. + + :param db_name: The IndexedDB database. + :param file_id: The identifier of the data. + :param void* arg: User-defined data that is passed to the callbacks, untouched by the API itself. This may be be used by a callback to identify the associated call. + :param em_arg_callback_func ondelete: Callback on successful delete + + - *(void*)* : A pointer to ``arg`` (user defined data). + + :param em_arg_callback_func onerror: Callback in the event of failure. The callback function parameter values are: + + - *(void*)* : A pointer to ``arg`` (user defined data). + + + + +.. c:function:: int emscripten_async_prepare(const char* file, em_str_callback_func onload, em_str_callback_func onerror) + + Prepares a file asynchronously. + + This does just the preparation part of :c:func:`emscripten_async_wget`. That is, it works on file data already present and performs any required asynchronous operations (for example, decoding images for use in ``IMG_Load``, decoding audio for use in ``Mix_LoadWAV``, etc.). + + Once the operations are complete (the file is prepared), the ``onload`` callback will be called. If any error occurs ``onerror`` will be called. The callbacks are called with the file as their argument. + + :param file: The name of the file to prepare. + :type file: const char* + :param em_str_callback_func onload: Callback on successful preparation of the file. The callback function parameter value is: + + - *(const char*)* : The name of the ``file`` that was prepared. + + :param em_str_callback_func onerror: Callback in the event of failure. The callback function parameter value is: + + - *(const char*)* : The name of the ``file`` for which the prepare failed. + + :return: 0 if successful, -1 if the file does not exist + :rtype: int + + Compiling ================ diff --git a/src/library_idbstore.js b/src/library_idbstore.js new file mode 100644 index 000000000..1a4486ee7 --- /dev/null +++ b/src/library_idbstore.js @@ -0,0 +1,138 @@ +var LibraryIDBStore = { + // A simple IDB-backed storage mechanism. Suitable for saving and loading large files asynchronously. This does + // *NOT* use the emscripten filesystem, intentionally, to avoid overhead. It lets you application define whatever + // filesystem-like layer you want, with the overhead 100% controlled by you. At the extremes, you could either + // just store large files, with almost no extra code; or you could implement a file b-tree using posix-compliant + // filesystem on top. + $IDBStore: { + indexedDB: function() { + if (typeof indexedDB !== 'undefined') return indexedDB; + var ret = null; + if (typeof window === 'object') ret = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB; + assert(ret, 'IDBStore used, but indexedDB not supported'); + return ret; + }, + DB_VERSION: 22, + DB_STORE_NAME: 'FILE_DATA', + dbs: {}, + getDB: function(name, callback) { + // check the cache first + var db = IDBStore.dbs[name]; + if (db) { + return callback(null, db); + } + var req; + try { + req = IDBStore.indexedDB().open(name, IDBStore.DB_VERSION); + } catch (e) { + return callback(e); + } + req.onupgradeneeded = function(e) { + var db = e.target.result; + var transaction = e.target.transaction; + var fileStore; + if (db.objectStoreNames.contains(IDBStore.DB_STORE_NAME)) { + fileStore = transaction.objectStore(IDBStore.DB_STORE_NAME); + } else { + fileStore = db.createObjectStore(IDBStore.DB_STORE_NAME); + } + }; + req.onsuccess = function() { + db = req.result; + // add to the cache + IDBStore.dbs[name] = db; + callback(null, db); + }; + req.onerror = function(e) { + callback(this.error); + e.preventDefault(); + }; + }, + getStore: function(dbName, type, callback) { + IDBStore.getDB(dbName, function(error, db) { + var transaction = db.transaction([IDBStore.DB_STORE_NAME], type); + transaction.onerror = function(e) { + callback(this.error); + e.preventDefault(); + }; + var store = transaction.objectStore(IDBStore.DB_STORE_NAME); + callback(null, store); + }); + }, + // External API + getFile: function(dbName, id, callback) { + IDBStore.getStore(dbName, 'readonly', function(err, store) { + var req = store.get(id); + req.onsuccess = function(event) { + var result = event.target.result; + if (!result) { + return callback('file ' + id + ' not found'); + } else { + return callback(null, result); + } + }; + req.onerror = function(error) { + callback(error); + }; + }); + }, + setFile: function(dbName, id, data, callback) { + IDBStore.getStore(dbName, 'readwrite', function(err, store) { + var req = store.put(data, id); + req.onsuccess = function(event) { + callback(); + }; + req.onerror = function(error) { + errback(error); + }; + }); + }, + deleteFile: function(dbName, id, callback) { + IDBStore.getStore(dbName, 'readwrite', function(err, store) { + var req = store.delete(id); + req.onsuccess = function(event) { + callback(); + }; + req.onerror = function(error) { + errback(error); + }; + }); + }, + }, + + emscripten_idb_async_load: function(db, id, arg, onload, onerror) { + IDBStore.getFile(Pointer_stringify(db), Pointer_stringify(id), function(error, byteArray) { + if (error) { + if (onerror) Runtime.dynCall('vi', onerror, [arg]); + return; + } + var buffer = _malloc(byteArray.length); + HEAPU8.set(byteArray, buffer); + Runtime.dynCall('viii', onload, [arg, buffer, byteArray.length]); + _free(buffer); + }); + }, + emscripten_idb_async_store: function(db, id, ptr, num, arg, onstore, onerror) { + // note that we copy the data here, as these are async operatins - changes to HEAPU8 meanwhile should not affect us! + IDBStore.setFile(Pointer_stringify(db), Pointer_stringify(id), new Uint8Array(HEAPU8.subarray(ptr, ptr+num)), function(error) { + if (error) { + if (onerror) Runtime.dynCall('vi', onerror, [arg]); + return; + } + if (onstore) Runtime.dynCall('vi', onstore, [arg]); + }); + }, + emscripten_idb_async_delete: function(db, id, arg, ondelete, onerror) { + IDBStore.deleteFile(Pointer_stringify(db), Pointer_stringify(id), function(error) { + if (error) { + if (onerror) Runtime.dynCall('vi', onerror, [arg]); + return; + } + if (ondelete) Runtime.dynCall('vi', ondelete, [arg]); + }); + }, +}; + +autoAddDeps(LibraryIDBStore, '$IDBStore'); +mergeInto(LibraryManager.library, LibraryIDBStore); + diff --git a/src/modules.js b/src/modules.js index d05115b6c..d47a42e73 100644 --- a/src/modules.js +++ b/src/modules.js @@ -458,6 +458,7 @@ var LibraryManager = { 'library_glew.js', 'library_html5.js', 'library_signals.js', + 'library_idbstore.js', 'library_async.js' ]).concat(additionalLibraries); diff --git a/system/include/emscripten/emscripten.h b/system/include/emscripten/emscripten.h index c244389b5..0f4ea01d0 100644 --- a/system/include/emscripten/emscripten.h +++ b/system/include/emscripten/emscripten.h @@ -144,7 +144,7 @@ static inline double emscripten_get_now(void) { float emscripten_random(void); - +// wget void emscripten_wget(const char* url, const char* file); void emscripten_async_wget(const char* url, const char* file, em_str_callback_func onload, em_str_callback_func onerror); @@ -165,6 +165,14 @@ int emscripten_async_wget2_data(const char* url, const char* requesttype, const void emscripten_async_wget2_abort(int handle); +// IDB + +void emscripten_idb_async_load(const char *db_name, const char *file_id, void* arg, em_async_wget_onload_func onload, em_arg_callback_func onerror); +void emscripten_idb_async_store(const char *db_name, const char *file_id, void* ptr, int num, void* arg, em_arg_callback_func onstore, em_arg_callback_func onerror); +void emscripten_idb_async_delete(const char *db_name, const char *file_id, void* arg, em_arg_callback_func ondelete, em_arg_callback_func onerror); + +// other async utilities + int emscripten_async_prepare(const char* file, em_str_callback_func onload, em_str_callback_func onerror); typedef void (*em_async_prepare_data_onload_func)(void*, const char*); diff --git a/tests/idbstore.c b/tests/idbstore.c new file mode 100644 index 000000000..e6b25d265 --- /dev/null +++ b/tests/idbstore.c @@ -0,0 +1,69 @@ +#include +#include +#include + +#include + +#define DB "THE_DB" + +int expected; +int result; + +void ok(void* arg) +{ + assert(expected == (int)arg); + REPORT_RESULT(); +} + +void onerror(void* arg) +{ + assert(expected == (int)arg); + result = 999; + REPORT_RESULT(); +} + +void onload(void* arg, void* ptr, int num) +{ + assert(expected == (int)arg); + printf("loaded %s\n", ptr); + assert(num == strlen(SECRET)+1); + assert(strcmp(ptr, SECRET) == 0); + result = 1; + REPORT_RESULT(); +} + +void onbadload(void* arg, void* ptr, int num) +{ + printf("load failed, surprising\n"); + result = 999; + REPORT_RESULT(); +} + +void test() { + result = STAGE; +#if STAGE == 0 + expected = 12; + emscripten_idb_async_store(DB, "the_secret", SECRET, strlen(SECRET)+1, (void*)expected, ok, onerror); + printf("storing %s\n", SECRET); +#elif STAGE == 1 + expected = 31; + emscripten_idb_async_load(DB, "the_secret", (void*)expected, onload, onerror); +#elif STAGE == 2 + expected = 44; + emscripten_idb_async_delete(DB, "the_secret", (void*)expected, ok, onerror); + printf("deleting the_secret\n"); +#elif STAGE == 3 + expected = 55; + emscripten_idb_async_load(DB, "the_secret", (void*)expected, onbadload, ok); + printf("loading, should fail as we deleted\n"); +#else + assert(0); +#endif +} + +int main() { + test(); + emscripten_exit_with_live_runtime(); + return 0; +} + diff --git a/tests/test_browser.py b/tests/test_browser.py index 4633e5aff..deedc65b4 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -1161,6 +1161,12 @@ keydown(100);keyup(100); // trigger the end self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=mode + ['-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_test', '_success']''']) self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=mode + ['-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_test', '_success']''']) + def test_idbstore(self): + secret = str(time.time()) + for stage in [0, 1, 2, 3, 0, 1, 2, 0, 0, 1]: + self.clear() + self.btest(path_from_root('tests', 'idbstore.c'), str(stage), force_c=True, args=['-DSTAGE=' + str(stage), '-DSECRET=\"' + secret + '\"']) + def test_force_exit(self): self.btest('force_exit.c', force_c=True, expected='17')