Split attachmentProxy into its own package

packages/gallery/package.json

@@ -2,7 +2,9 @@
 	"name": "Gallery",
 	"version": "0.0.1",
 	"description": "Personal photo gallery",
-	"keywords": ["javascript"],
+	"keywords": [
+		"javascript"
+	],
 	"author": "Timothy Farrell <tim@thecookiejar.me> (https://github.com/explorigin)",
 	"license": "Apache-2.0",
 	"scripts": {

packages/gallery/src/services/b2.js

@@ -1,5 +1,5 @@
 import core from 'pouchdb-core';
-import { PouchDBAttachmentProxy } from '../utils/attachmentProxy.js';
+import { PouchDBAttachmentProxy, blobToString, stringToBlob } from 'pouchdb-attachmentproxy';
 import { deepAssign, blobToArrayBuffer } from '../utils/conversion.js';
 import { prop, computed, stream } from 'frptools';
 import { sha1 } from '../utils/crypto.js';
@@ -72,19 +72,19 @@ export const B2Adapter = function(b2apikey, b2secret, b2bucket) {
 	}
 
 	return PouchDBAttachmentProxy({
-		getFn: async function getAttachment(id) {
+		getFn: async function getAttachment(blob) {
-			const res = await fetch(await downloadUrl(id), {
+			const res = await fetch(await downloadUrl(blobToString(blob)), {
 				headers: await headers()
 			});
 
 			return res.blob();
 		},
-		remove: async function removeAttachment(id) {
+		remove: async function removeAttachment(blob) {
 			const s = await session();
 			const res = await fetch('/api/v1/get_file_info', {
 				headers: await headers(),
 				method: 'POST',
-				body: JSON.stringify({ fileId: id })
+				body: JSON.stringify({ fileId: blobToString(blob) })
 			});
 			const { fileName, fileId } = await res.json();
 			return await fetch('/api/v1/remove_file', {
@@ -108,7 +108,7 @@ export const B2Adapter = function(b2apikey, b2secret, b2bucket) {
 				body: blob
 			});
 			const { fileId } = await res.json();
-			return fileId;
+			return stringToBlob(fileId);
 		}
 	});
 };

packages/gallery/src/utils/attachmentProxy.js

@@ -1,119 +0,0 @@
-import core from 'pouchdb-core';
-import { backgroundTask } from 'backgroundtask';
-
-import { deepAssign, blobToObj } from '../utils/conversion.js';
-import { error, log } from '../utils/console.js';
-
-const pouchBulkDocs = core.prototype.bulkDocs;
-const pouchGetAttachment = core.prototype.getAttachment;
-const pouchRemoveAttachment = core.prototype.removeAttachment;
-const STORAGE_MIMETYPE = 'application/b2storagemap';
-
-export function PouchDBAttachmentProxy({ save, getFn, remove }) {
-	const override = {
-		$attachmentsProxied: true
-	};
-
-	if (getFn) {
-		override.getAttachment = async function getAttachment(...args) {
-			const att = await pouchGetAttachment.apply(this, args);
-			if (att.type !== STORAGE_MIMETYPE) {
-				return att;
-			}
-			return await getFn(await blobToObj(att));
-		};
-	}
-
-	if (remove) {
-		override.removeAttachment = async function removeAttachment(...args) {
-			cleanupFiles(await pouchGetAttachment.apply(this, args));
-			return await pouchRemoveAttachment.apply(this, args);
-		};
-	}
-
-	if (save || remove) {
-		override.bulkDocs = function bulkDocs(...args) {
-			let docs;
-			if (Array.isArray(args[0])) {
-				docs = args[0];
-			} else {
-				docs = args[0].docs;
-			}
-
-			// All documents must have a .name field.
-			const deletedFiles = [];
-			const attachments = [];
-			docs.filter(d => d.type === 'file').forEach(f => {
-				if (f._deleted) {
-					deletedFiles.push(pouchGetAttachment.call(this, f._id, 'data'));
-					return;
-				}
-				if (f._attachments && f._attachments.data.data instanceof Blob) {
-					log(`Saving File ${f._id} attachment`);
-					attachments.push([f, 'data', f._attachments.data.data]);
-					delete f._attachments.data;
-				}
-			});
-
-			Promise.all(deletedFiles).then(atts => atts.forEach(cleanupFiles));
-
-			return Promise.all(
-				attachments.map(([doc, attName, blob]) =>
-					save(blob)
-						.then(resData => {
-							deepAssign(doc, {
-								_attachments: {
-									[attName]: {
-										content_type: STORAGE_MIMETYPE,
-										data: btoa(JSON.stringify(resData))
-									}
-								}
-							});
-						})
-						.catch(e => error(`Failed to save attachment ${doc._id}[${attName}]`, resData))
-				)
-			).then(() => {
-				return pouchBulkDocs.call(this, ...args);
-			});
-		};
-	}
-
-	const cleanupFiles = backgroundTask(function(att) {
-		if (att.type === STORAGE_MIMETYPE) {
-			blobToObj(att)
-				.then(remove)
-				.catch(e => error(`Failed to remove attachment ${args}`, e));
-		}
-	}, false);
-
-	return override;
-}
-
-// export const LocalStorageExampleAdapter = function() {
-// 	return PouchDBAttachmentProxy({
-// 		get: async function getAttachment(docId, attName) {
-// 			const data = localStorage[`${docId}-${attName}`].split(';base64,');
-// 			var byteCharacters = atob(data[1]);
-// 			var byteNumbers = new Array(byteCharacters.length);
-// 			for (var i = 0; i < byteCharacters.length; i++) {
-// 			    byteNumbers[i] = byteCharacters.charCodeAt(i);
-// 			}
-// 			var byteArray = new Uint8Array(byteNumbers);
-// 			return Promise.resolve(new Blob([byteArray], {type: data[0].substr(5)}));
-// 		},
-// 		remove: async function removeAttachment(docId, attName, rev) {
-// 			delete localStorage[`${docId}-${attName}`];
-// 			return Promise.resolve({"ok": true});
-// 		},
-// 		save: async function saveAttachment(docId, attName, obj) {
-// 			return new Promise((resolve) => {
-// 				var reader = new FileReader();
-// 				reader.onloadend = function() {
-// 					localStorage[`${docId}-${attName}`]  = reader.result;
-// 					resolve({"ok": true});
-// 				}
-// 				reader.readAsDataURL(obj.data);
-// 			});
-// 		}
-// 	});
-// };

packages/gallery/src/utils/conversion.js

@@ -26,18 +26,6 @@ export function blobToArrayBuffer(blob) {
 	});
 }
 
-export function blobToString(blob) {
-	return new Promise((resolve, reject) => {
-		const f = new FileReader();
-		f.onload = _ => resolve(f.result);
-		f.readAsText(blob);
-	});
-}
-
-export function blobToObj(blob) {
-	return blobToString(blob).then(JSON.parse);
-}
-
 export const arrayHashWrapper = hash => arr => (Array.isArray(arr) ? arr.map(hash).join('?') : arr);
 
 export function pouchDocHash(d) {

packages/pouchdb-attachmentproxy/README.md

@@ -0,0 +1,79 @@
+# PouchDB Attachment Proxy
+
+PouchDB Attachment Proxy is a PouchDB plugin that intercepts and allows you to redirect document
+attachments in a PouchDB database. The intention is to use the PouchDB API to manage all attachment
+metadata without having the raw data bogging down your PouchDB database.
+
+## Overview
+
+Attachment Proxy works by intercepting attachments bound for the pouchdb backend and passing them to
+a custom `save` function that will return a sentry blob to be saved in the pouchdb document.
+
+When an attachment is requested, the sentry blob will be provided to a custom `getFn` function to be
+converted into desired document.
+
+Similar to `getFn`, the sentry blob is provided to `remove` when an attachment or its containing
+document is deleted.
+
+Each function is expected to be asynchronous and the database will not continue its operation until
+the proxy function completes to ensure data integrity. It may be desirable to for the `remove`
+function to delay attachment cleanup. The proxy blob will be removed from document (effectively
+orphaning the proxied attachment until it is cleaned).
+
+## Example
+
+To create an attachment proxy, supply an object to the proxy wrapper with the three handler
+functions.
+
+```js
+import {
+	PouchDBAttachmentProxy,
+	SENTRY_MIMETYPE,
+	blobToString,
+	stringToBlob
+} from 'pouchdb-attachmentproxy';
+
+const dataUrlToBlob = data => {
+	const [header, b64str] = data.split(';base64,');
+	const byteCharacters = atob(b64str);
+	const byteNumbers = new Array(byteCharacters.length);
+	for (var i = 0; i < byteCharacters.length; i++) {
+		byteNumbers[i] = byteCharacters.charCodeAt(i);
+	}
+	const byteArray = new Uint8Array(byteNumbers);
+	return new Blob([byteArray], { type: header.substr(5) });
+};
+
+const LocalStorageExampleAdapter = PouchDBAttachmentProxy({
+	getFn: async function getAttachment(blob) {
+		const id = await blobToString(blob);
+		return dataUrlToBlob(localStorage[id]);
+	},
+	remove: async function removeAttachment(blob) {
+		const id = await blobToString(blob);
+		delete localStorage[id];
+	},
+	save: async function saveAttachment(blob) {
+		let id;
+		while (!id || localStorage[id] !== undefined) {
+			id = '' + Math.ceil(Math.random() * 100000);
+		}
+		// placeholder to prevent a race condition
+		localStorage[id] = true;
+
+		return new Promise(resolve => {
+			const reader = new FileReader();
+
+			reader.onloadend = function() {
+				localStorage[id] = reader.result;
+				// The returned blob must have the SENTRY_MIMETYPE mime_type in
+				// order to be proxied to `getFn` and `remove`.
+				resolve(stringToBlob(id, SENTRY_MIMETYPE));
+			};
+			reader.readAsDataURL(blob);
+		});
+	}
+});
+
+const ProxiedPouch = PouchDB.plugin(LocalStorageExampleAdapter);
+```

packages/pouchdb-attachmentproxy/package.json

@@ -0,0 +1,27 @@
+{
+	"name": "pouchdb-attachmentproxy",
+	"version": "1.0.0",
+	"description": "A pouchdb plugin to intercept and redirect document attachments.",
+	"main": "src/index.js",
+	"files": [
+		"src"
+	],
+	"scripts": {
+		"test": "npm run build:test && mv dist/index.html ./.spec_runner.html && node ../../bin/runTests.js ./",
+		"pre-commit": "npm run test",
+		"build:test": "webpack --config webpack.test-config.js"
+	},
+	"author": "Timothy Farrell <tim@thecookiejar.me> (https://github.com/explorigin)",
+	"license": "Apache-2.0",
+	"dependencies": {
+		"pouchdb-core": "~7.0.0"
+	},
+	"devDependencies": {
+		"html-webpack-plugin": "~3.2.0",
+		"pouchdb": "~7.0.0",
+		"pouchdb-adapter-memory": "^7.0.0",
+		"script-ext-html-webpack-plugin": "^2.0.1",
+		"webpack": "~4.10.2",
+		"webpack-cli": "~2.1.5"
+	}
+}

packages/pouchdb-attachmentproxy/spec/index.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Jasmine Spec Runner v3.1.0</title>
+
+  <link rel="shortcut icon" type="image/png" href="/vendor/jasmine/jasmine_favicon.png">
+  <link rel="stylesheet" href="/vendor/jasmine/jasmine.css">
+
+  <script src="/vendor/jasmine/jasmine.js"></script>
+  <script src="/vendor/jasmine/jasmine-html.js"></script>
+  <script defer src="/vendor/jasmine/boot.js"></script>
+</head><body></body>
+</html>
+	

packages/pouchdb-attachmentproxy/spec/proxy.spec.js

@@ -0,0 +1,230 @@
+import PouchDB from 'pouchdb';
+import memory from 'pouchdb-adapter-memory';
+import {
+	PouchDBAttachmentProxy,
+	SENTRY_MIMETYPE,
+	blobToString,
+	stringToBlob
+} from '../src/index.js';
+
+const notDesignDocs = d => !d.id.startsWith('_design');
+
+const dataUrlToBlob = data => {
+	const [header, b64str, ...nothing] = data.split(';base64,');
+	const byteCharacters = atob(b64str);
+	const byteNumbers = new Array(byteCharacters.length);
+	for (var i = 0; i < byteCharacters.length; i++) {
+		byteNumbers[i] = byteCharacters.charCodeAt(i);
+	}
+	const byteArray = new Uint8Array(byteNumbers);
+	return new Blob([byteArray], { type: header.substr(5) });
+};
+
+describe('PouchDB Attachment Proxy', () => {
+	let db;
+	let wrap = true;
+
+	const LocalStorageExampleAdapter = PouchDBAttachmentProxy({
+		getFn: async function getAttachment(blob) {
+			if (!wrap) {
+				return blob;
+			}
+			const id = await blobToString(blob);
+			return dataUrlToBlob(localStorage[id]);
+		},
+		remove: async function removeAttachment(blob) {
+			if (!wrap) {
+				return blob;
+			}
+			const id = await blobToString(blob);
+			delete localStorage[id];
+		},
+		save: async function saveAttachment(blob) {
+			if (!wrap) {
+				return blob;
+			}
+			let id;
+			while (!id || localStorage[id] !== undefined) {
+				id = '' + Math.ceil(Math.random() * 100000);
+			}
+			localStorage[id] = true; // placeholder to prevent a race condition while the FileReader reads the blob.
+
+			return new Promise(resolve => {
+				const reader = new FileReader();
+
+				reader.onloadend = function() {
+					localStorage[id] = reader.result;
+					resolve(stringToBlob(id));
+				};
+				reader.readAsDataURL(blob);
+			});
+		}
+	});
+	const PDB = PouchDB.plugin(memory).plugin(LocalStorageExampleAdapter);
+
+	beforeEach(() => {
+		db = new PDB('pouchdb-test', { adapter: 'memory' });
+		wrap = true;
+	});
+
+	afterEach(async () => {
+		const res = await db.allDocs();
+		await Promise.all(res.rows.filter(notDesignDocs).map(d => db.remove(d.id, d.value.rev))).catch(
+			() => {}
+		);
+		await db.compact();
+	});
+
+	async function checkBlob(blob, data) {
+		const blobData = await blobToString(blob);
+		expect(blobData).toEqual(data);
+	}
+
+	async function checkDocument(id, attName, mime_type, data) {
+		wrap = false;
+		const placeholderBlob = await db.getAttachment(id, attName);
+		wrap = true;
+
+		const placeholder = await blobToString(placeholderBlob);
+
+		expect(placeholderBlob.type).toEqual(SENTRY_MIMETYPE);
+
+		const resultDataStr = localStorage[placeholder];
+		const [header, b64str, ...nothing] = resultDataStr.split(';base64,');
+		expect(header.substr(5)).toEqual(mime_type);
+		expect(b64str).toEqual(btoa(data));
+		return resultDataStr;
+	}
+
+	describe('getFn()', () => {
+		it('returns attachments when PouchDB.getAttachment() is called', async () => {
+			const id = 'abc123';
+			localStorage['abc123'] = `data:text/plain;base64,${btoa('Test3 document')}`;
+
+			wrap = false;
+			await db.bulkDocs([
+				{
+					_id: 'test3',
+					_attachments: {
+						test3: {
+							content_type: SENTRY_MIMETYPE,
+							data: new Blob(['abc123'], { type: SENTRY_MIMETYPE })
+						}
+					}
+				}
+			]);
+			wrap = true;
+
+			const blob = await db.getAttachment('test3', 'test3');
+			checkBlob(blob, 'Test3 document');
+			expect(blob.type).toEqual('text/plain');
+		});
+	});
+
+	describe('save()', () => {
+		it('writes attachments in documents saved with PouchDB.put()', async () => {
+			await db.put({
+				_id: 'test1',
+				_attachments: {
+					test: {
+						content_type: 'text/plain',
+						data: new Blob(['Test document'], { type: 'text/plain' })
+					}
+				}
+			});
+			await checkDocument('test1', 'test', 'text/plain', 'Test document');
+		});
+
+		it('writes attachments in documents saved with PouchDB.bulkDocs()', async () => {
+			await db.bulkDocs([
+				{
+					_id: 'test2',
+					_attachments: {
+						test2: {
+							content_type: 'text/plain',
+							data: new Blob(['Test2 document'], { type: 'text/plain' })
+						}
+					}
+				}
+			]);
+			await checkDocument('test2', 'test2', 'text/plain', 'Test2 document');
+		});
+
+		it('writes attachments saved with PouchDB.putAttachment() ', async () => {
+			const res = await db.put({
+				_id: 'test5',
+				hi: 'there'
+			});
+
+			const res2 = await db.putAttachment(
+				res.id,
+				'test5',
+				res.rev,
+				new Blob(['Test5 document'], { type: 'text/plain' }),
+				'text/plain'
+			);
+			await checkDocument('test5', 'test5', 'text/plain', 'Test5 document');
+		});
+	});
+
+	describe('remove()', () => {
+		it('is called after PouchDB.removeAttachment()', async () => {
+			await db.bulkDocs([
+				{
+					_id: 'test4',
+					_attachments: {
+						test4: {
+							content_type: 'text/plain',
+							data: new Blob(['Test4 document'], { type: 'text/plain' })
+						}
+					}
+				}
+			]);
+
+			const doc = await db.get('test4', { attachments: true });
+			expect(doc && doc._attachments && doc._attachments.test4).toBeTruthy();
+
+			wrap = false;
+			const placeholderBlob = await db.getAttachment('test4', 'test4');
+			wrap = true;
+
+			const placeholder = await blobToString(placeholderBlob);
+			expect(placeholderBlob.type).toEqual(SENTRY_MIMETYPE);
+			expect(localStorage[placeholder]).toBeTruthy();
+
+			await db.removeAttachment(doc._id, 'test4', doc._rev);
+
+			const doc2 = await db.get('test4', { attachments: true });
+			expect(doc2._attachments).toBeFalsy();
+
+			expect(localStorage[placeholder]).toBeUndefined();
+		});
+
+		it('is called for attachments of deleted documents', async () => {
+			await db.bulkDocs([
+				{
+					_id: 'test6',
+					_attachments: {
+						test6: {
+							content_type: 'text/plain',
+							data: new Blob(['Test6 document'], { type: 'text/plain' })
+						}
+					}
+				}
+			]);
+
+			await checkDocument('test6', 'test6', 'text/plain', 'Test6 document');
+
+			wrap = false;
+			const placeholderBlob = await db.getAttachment('test6', 'test6');
+			wrap = true;
+			const placeholder = await blobToString(placeholderBlob);
+
+			const doc = await db.get('test6');
+
+			await db.remove(doc._id, doc._rev);
+
+			expect(localStorage[placeholder]).toBeUndefined();
+		});
+	});
+});

packages/pouchdb-attachmentproxy/src/index.js

@@ -0,0 +1,2 @@
+export { PouchDBAttachmentProxy } from './proxy.js';
+export { SENTRY_MIMETYPE, stringToBlob, blobToString } from './utils.js';

packages/pouchdb-attachmentproxy/src/proxy.js

@@ -0,0 +1,97 @@
+import core from 'pouchdb-core';
+import { SENTRY_MIMETYPE } from './utils.js';
+
+const pouchBulkDocs = core.prototype.bulkDocs;
+const pouchGetAttachment = core.prototype.getAttachment;
+const pouchRemoveAttachment = core.prototype.removeAttachment;
+
+export function PouchDBAttachmentProxy({ save, getFn, remove }) {
+	const override = {
+		$$attachmentsProxied: true
+	};
+
+	if (getFn) {
+		override.getAttachment = async function getAttachment(...args) {
+			const att = await pouchGetAttachment.apply(this, args);
+			if (att.type !== SENTRY_MIMETYPE) {
+				return att;
+			}
+			return await getFn(att);
+		};
+	}
+
+	if (remove && getFn) {
+		override.removeAttachment = async function removeAttachment(docId, attName, rev) {
+			const doc = await pouchGetAttachment.call(this, docId, attName);
+			await cleanupFiles(doc);
+			return await pouchRemoveAttachment.call(this, docId, attName, rev);
+		};
+	}
+
+	if (save || remove) {
+		override.bulkDocs = function bulkDocs(...args) {
+			const self = this;
+			let docs;
+			if (Array.isArray(args[0])) {
+				docs = args[0];
+			} else {
+				docs = args[0].docs;
+			}
+
+			// All documents must have a .name field.
+			const doomedDocs = [];
+			const attachments = [];
+			docs.map(f => {
+				if (f._deleted) {
+					doomedDocs.push(f._attachments ? f : this.get(f._id, { attachments: true, binary: true }));
+					return;
+				}
+				if (f._attachments) {
+					Object.entries(f._attachments).forEach(([attName, att]) => {
+						attachments.push([f, attName, att.data]);
+						delete f._attachments[attName];
+					});
+				}
+			});
+
+			return handleBulkAttachments(attachments, doomedDocs).then(() =>
+				pouchBulkDocs.call(self, ...args)
+			);
+		};
+	}
+
+	async function handleBulkAttachments(saveableAttachments, doomedDocsAndPromises) {
+		const self = this;
+		const doomedDocs = await Promise.all(doomedDocsAndPromises);
+		const doomedAttachments = doomedDocs
+			.map(doc => Object.values(doc._attachments || {})) // extract attachments
+			.reduce((acc, val) => acc.concat(val), []); // flatten
+		await Promise.all(doomedAttachments.map(a => cleanupFiles.call(self, a.data)));
+
+		await Promise.all(
+			saveableAttachments.map(async ([doc, attName, blob]) => {
+				try {
+					const proxyBlob = await save.call(self, blob);
+					doc._attachments[attName] = {
+						content_type: proxyBlob.type,
+						data: proxyBlob
+					};
+				} catch (e) {
+					console.error(`Failed to save attachment ${doc._id}[${attName}]`, e);
+				}
+			})
+		);
+	}
+
+	async function cleanupFiles(att) {
+		if (att.type === SENTRY_MIMETYPE) {
+			try {
+				await remove.call(this, att);
+			} catch (e) {
+				console.error('Failed to remove attachment ', att, e);
+			}
+		}
+	}
+
+	return override;
+}

packages/pouchdb-attachmentproxy/src/utils.js

@@ -0,0 +1,13 @@
+export const SENTRY_MIMETYPE = 'application/pouchdb-attachmentmetadata';
+
+export function blobToString(blob) {
+	return new Promise((resolve, reject) => {
+		const f = new FileReader();
+		f.onload = _ => resolve(f.result);
+		f.readAsText(blob);
+	});
+}
+
+export function stringToBlob(str, mime_type = SENTRY_MIMETYPE) {
+	return new Blob([str], { type: mime_type });
+}

packages/pouchdb-attachmentproxy/webpack.test-config.js

@@ -0,0 +1,31 @@
+const path = require('path');
+const webpack = require('webpack');
+const HtmlWebpackPlugin = require('html-webpack-plugin');
+const ScriptExtHtmlWebpackPlugin = require('script-ext-html-webpack-plugin');
+
+module.exports = {
+	context: path.resolve(__dirname, './spec'),
+	mode: 'development',
+	entry: {
+		proxy: './proxy.spec.js'
+	},
+	output: {
+		path: path.resolve(__dirname, './dist'),
+		filename: '[name].bundle.js',
+		publicPath: '/packages/pouchdb-attachmentproxy/dist/'
+	},
+	devServer: {
+		contentBase: path.join(__dirname, 'dist')
+	},
+	module: {},
+	plugins: [
+		new webpack.DefinePlugin({ __DEV__: true }),
+		new HtmlWebpackPlugin({
+			template: 'index.html',
+			inject: 'body'
+		}),
+		new ScriptExtHtmlWebpackPlugin({
+			module: /.*/
+		})
+	]
+};

packages/pouchtype/package.json

@@ -3,7 +3,11 @@
 	"version": "1.0.2",
 	"description": "Document Management Layer for PouchDB",
 	"main": "src/index.js",
-	"files": ["dist", "lib", "src"],
+	"files": [
+		"dist",
+		"lib",
+		"src"
+	],
 	"scripts": {
 		"test": "node ../../bin/runTests.js ./",
 		"pre-commit": "npm run test"