From a2cd9284f7aa29863b95780bb9276e69b6f5d959 Mon Sep 17 00:00:00 2001 From: Timothy Farrell Date: Thu, 26 Jul 2018 16:28:00 -0500 Subject: [PATCH] Split attachmentProxy into its own package --- packages/pouchdb-attachmentproxy/README.md | 79 ++++++ packages/pouchdb-attachmentproxy/package.json | 27 ++ .../pouchdb-attachmentproxy/spec/index.html | 15 ++ .../spec/proxy.spec.js | 230 ++++++++++++++++++ packages/pouchdb-attachmentproxy/src/index.js | 2 + packages/pouchdb-attachmentproxy/src/proxy.js | 97 ++++++++ packages/pouchdb-attachmentproxy/src/utils.js | 13 + .../webpack.test-config.js | 31 +++ 8 files changed, 494 insertions(+) create mode 100644 packages/pouchdb-attachmentproxy/README.md create mode 100644 packages/pouchdb-attachmentproxy/package.json create mode 100644 packages/pouchdb-attachmentproxy/spec/index.html create mode 100644 packages/pouchdb-attachmentproxy/spec/proxy.spec.js create mode 100644 packages/pouchdb-attachmentproxy/src/index.js create mode 100644 packages/pouchdb-attachmentproxy/src/proxy.js create mode 100644 packages/pouchdb-attachmentproxy/src/utils.js create mode 100644 packages/pouchdb-attachmentproxy/webpack.test-config.js diff --git a/packages/pouchdb-attachmentproxy/README.md b/packages/pouchdb-attachmentproxy/README.md new file mode 100644 index 0000000..3d7adbc --- /dev/null +++ b/packages/pouchdb-attachmentproxy/README.md @@ -0,0 +1,79 @@ +# PouchDB Attachment Proxy + +PouchDB Attachment Proxy is a PouchDB plugin that intercepts and allows you to redirect document +attachments in a PouchDB database. The intention is to use the PouchDB API to manage all attachment +metadata without having the raw data bogging down your PouchDB database. + +## Overview + +Attachment Proxy works by intercepting attachments bound for the pouchdb backend and passing them to +a custom `save` function that will return a sentry blob to be saved in the pouchdb document. + +When an attachment is requested, the sentry blob will be provided to a custom `getFn` function to be +converted into desired document. + +Similar to `getFn`, the sentry blob is provided to `remove` when an attachment or its containing +document is deleted. + +Each function is expected to be asynchronous and the database will not continue its operation until +the proxy function completes to ensure data integrity. It may be desirable to for the `remove` +function to delay attachment cleanup. The proxy blob will be removed from document (effectively +orphaning the proxied attachment until it is cleaned). + +## Example + +To create an attachment proxy, supply an object to the proxy wrapper with the three handler +functions. + +```js +import { + PouchDBAttachmentProxy, + SENTRY_MIMETYPE, + blobToString, + stringToBlob +} from 'pouchdb-attachmentproxy'; + +const dataUrlToBlob = data => { + const [header, b64str] = data.split(';base64,'); + const byteCharacters = atob(b64str); + const byteNumbers = new Array(byteCharacters.length); + for (var i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + return new Blob([byteArray], { type: header.substr(5) }); +}; + +const LocalStorageExampleAdapter = PouchDBAttachmentProxy({ + getFn: async function getAttachment(blob) { + const id = await blobToString(blob); + return dataUrlToBlob(localStorage[id]); + }, + remove: async function removeAttachment(blob) { + const id = await blobToString(blob); + delete localStorage[id]; + }, + save: async function saveAttachment(blob) { + let id; + while (!id || localStorage[id] !== undefined) { + id = '' + Math.ceil(Math.random() * 100000); + } + // placeholder to prevent a race condition + localStorage[id] = true; + + return new Promise(resolve => { + const reader = new FileReader(); + + reader.onloadend = function() { + localStorage[id] = reader.result; + // The returned blob must have the SENTRY_MIMETYPE mime_type in + // order to be proxied to `getFn` and `remove`. + resolve(stringToBlob(id, SENTRY_MIMETYPE)); + }; + reader.readAsDataURL(blob); + }); + } +}); + +const ProxiedPouch = PouchDB.plugin(LocalStorageExampleAdapter); +``` diff --git a/packages/pouchdb-attachmentproxy/package.json b/packages/pouchdb-attachmentproxy/package.json new file mode 100644 index 0000000..f07562d --- /dev/null +++ b/packages/pouchdb-attachmentproxy/package.json @@ -0,0 +1,27 @@ +{ + "name": "pouchdb-attachmentproxy", + "version": "1.0.0", + "description": "A pouchdb plugin to intercept and redirect document attachments.", + "main": "src/index.js", + "files": [ + "src" + ], + "scripts": { + "test": "npm run build:test && mv dist/index.html ./.spec_runner.html && node ../../bin/runTests.js ./", + "pre-commit": "npm run test", + "build:test": "webpack --config webpack.test-config.js" + }, + "author": "Timothy Farrell (https://github.com/explorigin)", + "license": "Apache-2.0", + "dependencies": { + "pouchdb-core": "~7.0.0" + }, + "devDependencies": { + "html-webpack-plugin": "~3.2.0", + "pouchdb": "~7.0.0", + "pouchdb-adapter-memory": "^7.0.0", + "script-ext-html-webpack-plugin": "^2.0.1", + "webpack": "~4.10.2", + "webpack-cli": "~2.1.5" + } +} diff --git a/packages/pouchdb-attachmentproxy/spec/index.html b/packages/pouchdb-attachmentproxy/spec/index.html new file mode 100644 index 0000000..fd152f4 --- /dev/null +++ b/packages/pouchdb-attachmentproxy/spec/index.html @@ -0,0 +1,15 @@ + + + + + Jasmine Spec Runner v3.1.0 + + + + + + + + + + \ No newline at end of file diff --git a/packages/pouchdb-attachmentproxy/spec/proxy.spec.js b/packages/pouchdb-attachmentproxy/spec/proxy.spec.js new file mode 100644 index 0000000..3999414 --- /dev/null +++ b/packages/pouchdb-attachmentproxy/spec/proxy.spec.js @@ -0,0 +1,230 @@ +import PouchDB from 'pouchdb'; +import memory from 'pouchdb-adapter-memory'; +import { + PouchDBAttachmentProxy, + SENTRY_MIMETYPE, + blobToString, + stringToBlob +} from '../src/index.js'; + +const notDesignDocs = d => !d.id.startsWith('_design'); + +const dataUrlToBlob = data => { + const [header, b64str, ...nothing] = data.split(';base64,'); + const byteCharacters = atob(b64str); + const byteNumbers = new Array(byteCharacters.length); + for (var i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + return new Blob([byteArray], { type: header.substr(5) }); +}; + +describe('PouchDB Attachment Proxy', () => { + let db; + let wrap = true; + + const LocalStorageExampleAdapter = PouchDBAttachmentProxy({ + getFn: async function getAttachment(blob) { + if (!wrap) { + return blob; + } + const id = await blobToString(blob); + return dataUrlToBlob(localStorage[id]); + }, + remove: async function removeAttachment(blob) { + if (!wrap) { + return blob; + } + const id = await blobToString(blob); + delete localStorage[id]; + }, + save: async function saveAttachment(blob) { + if (!wrap) { + return blob; + } + let id; + while (!id || localStorage[id] !== undefined) { + id = '' + Math.ceil(Math.random() * 100000); + } + localStorage[id] = true; // placeholder to prevent a race condition while the FileReader reads the blob. + + return new Promise(resolve => { + const reader = new FileReader(); + + reader.onloadend = function() { + localStorage[id] = reader.result; + resolve(stringToBlob(id)); + }; + reader.readAsDataURL(blob); + }); + } + }); + const PDB = PouchDB.plugin(memory).plugin(LocalStorageExampleAdapter); + + beforeEach(() => { + db = new PDB('pouchdb-test', { adapter: 'memory' }); + wrap = true; + }); + + afterEach(async () => { + const res = await db.allDocs(); + await Promise.all(res.rows.filter(notDesignDocs).map(d => db.remove(d.id, d.value.rev))).catch( + () => {} + ); + await db.compact(); + }); + + async function checkBlob(blob, data) { + const blobData = await blobToString(blob); + expect(blobData).toEqual(data); + } + + async function checkDocument(id, attName, mime_type, data) { + wrap = false; + const placeholderBlob = await db.getAttachment(id, attName); + wrap = true; + + const placeholder = await blobToString(placeholderBlob); + + expect(placeholderBlob.type).toEqual(SENTRY_MIMETYPE); + + const resultDataStr = localStorage[placeholder]; + const [header, b64str, ...nothing] = resultDataStr.split(';base64,'); + expect(header.substr(5)).toEqual(mime_type); + expect(b64str).toEqual(btoa(data)); + return resultDataStr; + } + + describe('getFn()', () => { + it('returns attachments when PouchDB.getAttachment() is called', async () => { + const id = 'abc123'; + localStorage['abc123'] = `data:text/plain;base64,${btoa('Test3 document')}`; + + wrap = false; + await db.bulkDocs([ + { + _id: 'test3', + _attachments: { + test3: { + content_type: SENTRY_MIMETYPE, + data: new Blob(['abc123'], { type: SENTRY_MIMETYPE }) + } + } + } + ]); + wrap = true; + + const blob = await db.getAttachment('test3', 'test3'); + checkBlob(blob, 'Test3 document'); + expect(blob.type).toEqual('text/plain'); + }); + }); + + describe('save()', () => { + it('writes attachments in documents saved with PouchDB.put()', async () => { + await db.put({ + _id: 'test1', + _attachments: { + test: { + content_type: 'text/plain', + data: new Blob(['Test document'], { type: 'text/plain' }) + } + } + }); + await checkDocument('test1', 'test', 'text/plain', 'Test document'); + }); + + it('writes attachments in documents saved with PouchDB.bulkDocs()', async () => { + await db.bulkDocs([ + { + _id: 'test2', + _attachments: { + test2: { + content_type: 'text/plain', + data: new Blob(['Test2 document'], { type: 'text/plain' }) + } + } + } + ]); + await checkDocument('test2', 'test2', 'text/plain', 'Test2 document'); + }); + + it('writes attachments saved with PouchDB.putAttachment() ', async () => { + const res = await db.put({ + _id: 'test5', + hi: 'there' + }); + + const res2 = await db.putAttachment( + res.id, + 'test5', + res.rev, + new Blob(['Test5 document'], { type: 'text/plain' }), + 'text/plain' + ); + await checkDocument('test5', 'test5', 'text/plain', 'Test5 document'); + }); + }); + + describe('remove()', () => { + it('is called after PouchDB.removeAttachment()', async () => { + await db.bulkDocs([ + { + _id: 'test4', + _attachments: { + test4: { + content_type: 'text/plain', + data: new Blob(['Test4 document'], { type: 'text/plain' }) + } + } + } + ]); + + const doc = await db.get('test4', { attachments: true }); + expect(doc && doc._attachments && doc._attachments.test4).toBeTruthy(); + + wrap = false; + const placeholderBlob = await db.getAttachment('test4', 'test4'); + wrap = true; + + const placeholder = await blobToString(placeholderBlob); + expect(placeholderBlob.type).toEqual(SENTRY_MIMETYPE); + expect(localStorage[placeholder]).toBeTruthy(); + + await db.removeAttachment(doc._id, 'test4', doc._rev); + + const doc2 = await db.get('test4', { attachments: true }); + expect(doc2._attachments).toBeFalsy(); + + expect(localStorage[placeholder]).toBeUndefined(); + }); + + it('is called for attachments of deleted documents', async () => { + await db.bulkDocs([ + { + _id: 'test6', + _attachments: { + test6: { + content_type: 'text/plain', + data: new Blob(['Test6 document'], { type: 'text/plain' }) + } + } + } + ]); + + await checkDocument('test6', 'test6', 'text/plain', 'Test6 document'); + + wrap = false; + const placeholderBlob = await db.getAttachment('test6', 'test6'); + wrap = true; + const placeholder = await blobToString(placeholderBlob); + + const doc = await db.get('test6'); + + await db.remove(doc._id, doc._rev); + + expect(localStorage[placeholder]).toBeUndefined(); + }); + }); +}); diff --git a/packages/pouchdb-attachmentproxy/src/index.js b/packages/pouchdb-attachmentproxy/src/index.js new file mode 100644 index 0000000..c6be64f --- /dev/null +++ b/packages/pouchdb-attachmentproxy/src/index.js @@ -0,0 +1,2 @@ +export { PouchDBAttachmentProxy } from './proxy.js'; +export { SENTRY_MIMETYPE, stringToBlob, blobToString } from './utils.js'; diff --git a/packages/pouchdb-attachmentproxy/src/proxy.js b/packages/pouchdb-attachmentproxy/src/proxy.js new file mode 100644 index 0000000..26a0589 --- /dev/null +++ b/packages/pouchdb-attachmentproxy/src/proxy.js @@ -0,0 +1,97 @@ +import core from 'pouchdb-core'; +import { SENTRY_MIMETYPE } from './utils.js'; + +const pouchBulkDocs = core.prototype.bulkDocs; +const pouchGetAttachment = core.prototype.getAttachment; +const pouchRemoveAttachment = core.prototype.removeAttachment; + +export function PouchDBAttachmentProxy({ save, getFn, remove }) { + const override = { + $$attachmentsProxied: true + }; + + if (getFn) { + override.getAttachment = async function getAttachment(...args) { + const att = await pouchGetAttachment.apply(this, args); + if (att.type !== SENTRY_MIMETYPE) { + return att; + } + return await getFn(att); + }; + } + + if (remove && getFn) { + override.removeAttachment = async function removeAttachment(docId, attName, rev) { + const doc = await pouchGetAttachment.call(this, docId, attName); + await cleanupFiles(doc); + return await pouchRemoveAttachment.call(this, docId, attName, rev); + }; + } + + if (save || remove) { + override.bulkDocs = function bulkDocs(...args) { + const self = this; + let docs; + if (Array.isArray(args[0])) { + docs = args[0]; + } else { + docs = args[0].docs; + } + + // All documents must have a .name field. + const doomedDocs = []; + const attachments = []; + docs.map(f => { + if (f._deleted) { + doomedDocs.push(f._attachments ? f : this.get(f._id, { attachments: true, binary: true })); + return; + } + if (f._attachments) { + Object.entries(f._attachments).forEach(([attName, att]) => { + attachments.push([f, attName, att.data]); + delete f._attachments[attName]; + }); + } + }); + + return handleBulkAttachments(attachments, doomedDocs).then(() => + pouchBulkDocs.call(self, ...args) + ); + }; + } + + async function handleBulkAttachments(saveableAttachments, doomedDocsAndPromises) { + const self = this; + const doomedDocs = await Promise.all(doomedDocsAndPromises); + const doomedAttachments = doomedDocs + .map(doc => Object.values(doc._attachments || {})) // extract attachments + .reduce((acc, val) => acc.concat(val), []); // flatten + await Promise.all(doomedAttachments.map(a => cleanupFiles.call(self, a.data))); + + await Promise.all( + saveableAttachments.map(async ([doc, attName, blob]) => { + try { + const proxyBlob = await save.call(self, blob); + doc._attachments[attName] = { + content_type: proxyBlob.type, + data: proxyBlob + }; + } catch (e) { + console.error(`Failed to save attachment ${doc._id}[${attName}]`, e); + } + }) + ); + } + + async function cleanupFiles(att) { + if (att.type === SENTRY_MIMETYPE) { + try { + await remove.call(this, att); + } catch (e) { + console.error('Failed to remove attachment ', att, e); + } + } + } + + return override; +} diff --git a/packages/pouchdb-attachmentproxy/src/utils.js b/packages/pouchdb-attachmentproxy/src/utils.js new file mode 100644 index 0000000..5a3f57b --- /dev/null +++ b/packages/pouchdb-attachmentproxy/src/utils.js @@ -0,0 +1,13 @@ +export const SENTRY_MIMETYPE = 'application/pouchdb-attachmentmetadata'; + +export function blobToString(blob) { + return new Promise((resolve, reject) => { + const f = new FileReader(); + f.onload = _ => resolve(f.result); + f.readAsText(blob); + }); +} + +export function stringToBlob(str, mime_type = SENTRY_MIMETYPE) { + return new Blob([str], { type: mime_type }); +} diff --git a/packages/pouchdb-attachmentproxy/webpack.test-config.js b/packages/pouchdb-attachmentproxy/webpack.test-config.js new file mode 100644 index 0000000..cb9b134 --- /dev/null +++ b/packages/pouchdb-attachmentproxy/webpack.test-config.js @@ -0,0 +1,31 @@ +const path = require('path'); +const webpack = require('webpack'); +const HtmlWebpackPlugin = require('html-webpack-plugin'); +const ScriptExtHtmlWebpackPlugin = require('script-ext-html-webpack-plugin'); + +module.exports = { + context: path.resolve(__dirname, './spec'), + mode: 'development', + entry: { + proxy: './proxy.spec.js' + }, + output: { + path: path.resolve(__dirname, './dist'), + filename: '[name].bundle.js', + publicPath: '/packages/pouchdb-attachmentproxy/dist/' + }, + devServer: { + contentBase: path.join(__dirname, 'dist') + }, + module: {}, + plugins: [ + new webpack.DefinePlugin({ __DEV__: true }), + new HtmlWebpackPlugin({ + template: 'index.html', + inject: 'body' + }), + new ScriptExtHtmlWebpackPlugin({ + module: /.*/ + }) + ] +};