So much in here

- Image.upload calls File.upload
- doc deletes are PUT instead of deleted
- generate thumbnails is called off of processImportables
- got rid of stupid context loaders
- added Type.getOrCreate()
- fix massively broken comparators
- get rid of global watcher (unnecessary with new delete method)

Still broken
- indexes (albums)
- files not deleted along with images
- some wonky jpgs
This commit is contained in:
Timothy Farrell 2017-11-18 00:42:55 -06:00
parent cb4af2c407
commit e5107e72f9
12 changed files with 231 additions and 344 deletions

View File

@ -2,17 +2,15 @@
import { createView } from 'domvm/dist/micro/domvm.micro.js';
import * as styles from './app.css';
import generateThumbnails from './contextLoaders/generateThumbnails.js';
import { GalleryView } from './interface/gallery.js';
import { router } from './services/router.js';
import { getDatabase } from './services/db.js';
import { EventEmitter } from 'events';
EventEmitter.defaultMaxListeners = 1000; // https://github.com/pouchdb/pouchdb/issues/6123
// Attach our root view to the DOM
createView(GalleryView, { db: getDatabase() }).mount(document.querySelector('#app'));
createView(GalleryView, {}).mount(document.querySelector('#app'));
// Start the router
router.start('home');

View File

@ -1,7 +1,6 @@
import pica from 'pica/dist/pica';
import { generateAttachmentUrl, getDatabase } from '../services/db.js';
import { find, update, addAttachment } from '../data/image.js';
import { FileType } from '../data/file.js';
export function maxLinearSize(width, height, max) {
const ratio = width / height;
@ -45,28 +44,21 @@ async function resizeImage(imageBlob, mimetype, width, height) {
});
}
export async function generateThumbnailForImage(id) {
const results = await find([id], { attachments: true, binary: true });
const doc = results.rows[0].doc;
if (doc.attachmentUrls.thumbnail && doc._attachments.thumbnail) {
export async function generateThumbnailForImage(doc) {
if (doc.sizes.thumbnail) {
return;
}
const attachment = doc._attachments.image;
const mimetype = attachment.content_type;
const attachment = await FileType.getFromURL(doc.sizes.full);
const mimetype = attachment.content_type || attachment.type;
const { width, height } = maxLinearSize(doc.width, doc.height, 320);
const resizedBlob = await resizeImage(attachment.data, mimetype, width, height);
const url = generateAttachmentUrl(getDatabase().name, id, 'thumbnail');
const resizedBlob = await resizeImage(attachment, mimetype, width, height);
await addAttachment(doc, 'thumbnail', resizedBlob);
await update(doc._id, {
attachmentUrls: {
thumbnail: url
const thumbfile = await FileType.upload(resizedBlob);
await doc.update({
sizes: {
thumbnail: FileType.getURL(thumbfile)
}
});
return resizedBlob;
}
export const invoke = generateThumbnailForImage;

View File

@ -1,11 +0,0 @@
import * as image from '../data/image.js';
// Watch for new images, generate thumbnails if they need them.
image.watcher(async function generateThumbnails(id, deleted, doc) {
if (deleted || (doc.attachmentUrls.thumbnail && doc._attachments.thumbnail)) {
return;
}
const module = await import('../context/generateThumbnails');
module.invoke(id);
});

View File

@ -20,29 +20,32 @@ export const FileType = PouchDB.registerType({
// }
// },
methods: {
upload: async function(fileListOrEvent) {
const files = Array.from(
fileListOrEvent instanceof Event ? fileListOrEvent.currentTarget.files : fileListOrEvent
);
return files.map(async f => {
const digest = await sha256(await blobToArrayBuffer(f));
const file = FileType.new({
name: f.name,
mimetype: f.type,
size: f.size,
modifiedDate: new Date(f.lastModified),
addDate: new Date(),
digest,
tags: {},
_attachments: {
data: {
content_type: f.type,
data: f
}
getURL: doc => `/${FileType.prefix}/${doc._id}/data`,
getFromURL: async path => {
if (path.endsWith('/')) {
path = path.substr(0, path.length - 1);
}
const [_, db, id, attname] = path.split('/');
const doc = await FileType.find(id);
return await doc.getAttachment(attname);
},
upload: async function(blob) {
const digest = await sha256(await blobToArrayBuffer(blob));
const lastModified = blob.lastModified ? new Date(blob.lastModified) : new Date();
return await FileType.getOrCreate({
name: blob.name,
mimetype: blob.type,
size: blob.size,
lastModified: lastModified.toISOString(),
addDate: new Date().toISOString(),
digest,
tags: {},
_attachments: {
data: {
content_type: blob.type,
data: blob
}
});
await file.save();
return file;
}
});
}
}

View File

@ -1,150 +1,104 @@
import { getDatabase, generateAttachmentUrl } from '../services/db.js';
import { log, error } from '../services/console.js';
import { sha256 } from '../utils/crypto.js';
import { blobToArrayBuffer, deepAssign } from '../utils/conversion.js';
import { Event, backgroundTask } from '../utils/event.js';
import { Watcher } from '../utils/watcher.js';
import { PouchDB, TYPES as t } from '../services/db.js';
import { blobToArrayBuffer } from '../utils/conversion.js';
import { backgroundTask } from '../utils/event.js';
import { FileType } from './file.js';
const db = getDatabase();
const PROCESS_PREFIX = 'importing';
const PREFIX = 'image';
export const SELECTOR = {
_id: {
$gt: `${PREFIX}_`,
$lt: `${PREFIX}_\ufff0`
}
};
const IMPORT_SELECTOR = {
_id: {
$gt: `${PROCESS_PREFIX}_`,
$lt: `${PROCESS_PREFIX}_\ufff0`
}
};
export const ImageType = PouchDB.registerType({
name: 'Image',
getUniqueID: doc => doc.digest.substr(0, 16),
getSequence: doc => new Date(doc.originalDate).getTime(),
// schema: {
// originalDate: t.REQUIRED_DATE,
// digest: t.REQUIRED_STRING,
// width: t.INTEGER,
// height: t.INTEGER,
// sizes: {
// type: 'object',
// properties: {
// full: t.REQUIRED_STRING,
// thumbnail: t.STRING,
// }
// },
// orientation: t.INTEGER,
// make: t.STRING,
// model: t.STRING,
// flash: t.BOOLEAN,
// iso: t.INTEGER,
// gps: {
// type: 'object',
// properties: {
// latitude: t.NUMBER,
// longitude: t.NUMBER,
// altitude: t.NUMBER,
// heading: t.NUMBER,
// }
// },
// tags: {
// type: "object",
// additionalProperties: t.BOOLEAN
// }
// },
methods: {
upload: async function(blob) {
const f = await FileType.upload(blob, false);
return await ImageType.getOrCreate({
digest: f.digest,
originalDate: f.lastModified,
importing: true,
width: 0,
height: 0,
sizes: {
full: FileType.getURL(f)
}
});
},
processImportables: backgroundTask(async function _processImportables(importables) {
if (!importables.length) {
return;
}
// Events
export const imported = new Event('Image.imported');
export const removed = new Event('Image.removed');
const image = importables[0];
const { _id, _rev } = image;
const imageData = await FileType.getFromURL(image.sizes.full);
// Watchers
export const watcher = Watcher(db, SELECTOR, true);
export const importWatcher = Watcher(db, IMPORT_SELECTOR);
const ExifParser = await import('exif-parser');
// Methods
const getId = id => (id.startsWith(PREFIX) ? id : `${PREFIX}_${id}`);
const buffer = await blobToArrayBuffer(imageData);
export async function find(keys, options = {}) {
let opts = { include_docs: true };
if (Array.isArray(keys)) {
Object.assign(opts, options);
opts.keys = keys.map(getId);
} else {
Object.assign(opts, keys);
opts.startkey = `${PREFIX}_0`;
opts.endkey = `${PREFIX}_\ufff0`;
}
return await db.allDocs(opts);
}
const exifData = ExifParser.create(buffer).parse();
const { tags, imageSize } = exifData;
const { width, height } = imageSize;
const originalDate = new Date(
tags.DateTimeOriginal
? new Date(tags.DateTimeOriginal * 1000).toISOString()
: image.originalDate
).toISOString();
export async function getAttachment(id, attName) {
return await db.getAttachment(id, attName);
}
export async function remove(ids) {
const docs = await find(Array.isArray(ids) ? ids : [ids]);
const foundDocs = docs.rows.filter(r => !r.error);
const result = await db.bulkDocs(foundDocs.map(r => Object.assign(r.doc, { _deleted: true })));
foundDocs.filter((_, i) => result[i].ok).map(r => removed.fire(r.doc));
return result.reduce((a, r) => a && r.ok, true);
}
export async function update(id, properties) {
const results = await find([id]);
const doc = results.rows[0].doc;
deepAssign(doc, properties);
await db.put(doc);
return doc;
}
export async function addAttachment(doc, key, blob) {
return db.putAttachment(doc._id, key, doc._rev, blob, blob.type);
}
// Internal Functions
const processImportables = backgroundTask(async function _processImportables(importables) {
if (!importables.length) {
return;
}
const file = importables[0];
const { _id, _rev } = file;
const imageData = await file.getAttachment('data');
const ExifParser = await import('exif-parser');
const buffer = await blobToArrayBuffer(imageData);
// Check if this image already exists
// TODO - Create an image.digest index
const digestQuery = await db.find({
selector: { digest: file.digest },
fields: ['_id'],
limit: 1
});
if (digestQuery.docs.length) {
imported.fire(digestQuery.docs[0]._id, _id, false);
} else {
const exifData = ExifParser.create(buffer).parse();
const { tags, imageSize } = exifData;
const originalDate = new Date(
tags.DateTimeOriginal ? new Date(tags.DateTimeOriginal * 1000).toISOString() : file.modifiedDate
);
const id = `${PREFIX}_${originalDate.getTime().toString(36)}_${file.digest.substr(0, 6)}`;
const newDoc = Object.assign(
{},
{
_id: id,
originalDate: originalDate,
const img = await ImageType.getOrCreate({
originalDate,
width,
height,
orientation: tags.Orientation,
digest: file.digest,
digest: image.digest,
make: tags.Make,
model: tags.Model,
flash: !!tags.Flash,
ISO: tags.ISO,
fileId: file._id,
url: generateAttachmentUrl('file', file._id, 'data'),
iso: tags.ISO,
sizes: image.sizes,
gps: {
latitude: tags.GPSLatitude,
longitude: tags.GPSLongitude,
altitude: tags.GPSAltitude,
heading: tags.GPSImgDirection
}
},
imageSize // width & height
);
delete newDoc._rev; // assigned from doc but not desired.
});
try {
await db.put(newDoc);
file.update({ tags: { galleryImage: false } });
imported.fire(id, _id, true);
} catch (e) {
error(`Error processing Image ${id}`, e);
}
image.delete();
const module = await import('../context/generateThumbnails');
await module.generateThumbnailForImage(img);
}, false)
}
}, false);
FileType.find(
{
$and: [{ mimetype: { $in: ['image/jpeg'] } }, { $not: { ['tags.galleryImage']: false } }]
},
true
).then(fw => {
fw.subscribe((...props) => {
processImportables(...props);
});
});
ImageType.find({ importing: true }, true).then(fw => fw.subscribe(ImageType.processImportables));

View File

@ -1,37 +1,39 @@
import { defineElement as el } from 'domvm';
import { prop, computed, bundle } from 'frptools';
import * as imageType from '../data/image.js';
import { ImageType } from '../data/image.js';
import { FileType } from '../data/file.js';
import { pouchDocComparator } from '../utils/comparators.js';
export function AttachmentImageView(vm, params) {
export function AttachmentImageView(vm, doc) {
const model = bundle({
doc: prop(params.src),
attachmentKey: prop(params.attachmentKey || 'image')
_id: prop(doc._id),
_rev: prop(doc._rev),
sizes: prop(doc.sizes)
});
const blobURL = prop('');
const imageID = computed(doc => doc._id, [model.doc]);
const imageURL = computed((doc, key, bURL) => bURL || doc.attachmentUrls[key], [
model.doc,
model.attachmentKey,
const imageURL = computed((sizes, bURL) => bURL || sizes.thumbnail || sizes.full, [
model.sizes,
blobURL
]);
const imageSignature = computed((id, key) => id + ' ' + key, [imageID, model.attachmentKey]);
const _key = computed((id, rev) => id + rev, [model._id, model._rev]);
async function loadImageFromBlob() {
const id = imageID();
const key = model.attachmentKey();
const options = ['thumbnail', 'full'].filter(o => model.sizes().hasOwnProperty(o));
try {
const data = await imageType.getAttachment(id, key);
if (blobURL()) {
URL.revokeObjectURL(blobURL());
for (let attempt of options) {
try {
const data = await FileType.getFromURL(model.sizes()[attempt]);
if (blobURL()) {
URL.revokeObjectURL(blobURL());
}
blobURL(URL.createObjectURL(data));
return;
} catch (err) {
continue;
}
blobURL(URL.createObjectURL(data));
} catch (err) {
// Probably hasn't created the thumbnail yet.
console.log("Probably hasn't created the thumbnail yet.", err);
}
}
@ -42,18 +44,17 @@ export function AttachmentImageView(vm, params) {
const redrawOff = imageURL.subscribe(() => vm.redraw());
return function render(vm, params) {
const imgSig = imageSignature();
model(params);
if (imgSig !== imageSignature()) {
return function render(vm, doc) {
if (!pouchDocComparator(doc, { _id: model._id(), _rev: model._rev() })) {
URL.revokeObjectURL(blobURL());
blobURL('');
}
model(doc);
return el('img', {
src: imageURL(),
onerror: loadImageFromBlob,
_key: imageSignature(),
_key: _key(),
_hooks: {
didRemove: cleanup
}

View File

@ -1,39 +1,51 @@
import * as image from '../data/image.js';
import * as index from '../data/indexType.js';
import { FileType } from '../data/file.js';
import * as imageTag from '../context/manageImageTags.js';
import { defineView as vw } from 'domvm';
import { ImageType } from '../data/image.js';
// import * as index from '../data/indexType.js';
// import * as imageTag from '../context/manageImageTags.js';
import { ThumbnailView } from './thumbnail.js';
import { AlbumView } from './album.js';
import { router, routeChanged } from '../services/router.js';
import { styled, el } from '../services/style.js';
import { LiveArray } from '../utils/livearray.js';
import { Watcher } from '../utils/watcher.js';
export function GalleryView(vm, model) {
const { db } = model;
const NAV_OPTIONS = {
images: {
selector: image.SELECTOR,
data: ImageType.find(
{
importing: { $exists: false }
},
true
),
title: 'Images'
},
albums: {
selector: index.SELECTOR,
title: 'Albums'
}
// albums: {
// selector: index.SELECTOR,
// title: 'Albums'
// }
};
let data = null;
let laCleanup = null;
let title = '';
function uploadImages(evt) {
Array.from(evt.currentTarget.files).forEach(ImageType.upload);
}
routeChanged.subscribe(function onRouteChange(router, route) {
if (data) {
data.cleanup();
if (laCleanup) {
laCleanup();
}
const o = NAV_OPTIONS[route.name];
data = LiveArray(db, o.selector);
title = o.title;
data.subscribe(() => vm.redraw());
return o.data.then(la => {
data = la;
laCleanup = data.subscribe(() => {
vm.redraw();
});
data.ready.subscribe(() => vm.redraw);
});
});
return function(vm, model, key, opts) {
@ -44,7 +56,7 @@ export function GalleryView(vm, model) {
type: 'file',
multiple: true,
accept: 'image/jpeg',
onchange: FileType.upload
onchange: uploadImages
})
]),
...(!data || !data.ready()
@ -60,9 +72,9 @@ export function GalleryView(vm, model) {
{
doc: i,
showTags: true,
addTag: imageTag.add,
remove: image.remove,
removeTag: imageTag.remove
// addTag: imageTag.add,
remove: i.delete.bind(i)
// removeTag: imageTag.remove
},
i._id + i._rev
);
@ -72,9 +84,9 @@ export function GalleryView(vm, model) {
AlbumView,
{
doc: a,
db,
addTag: imageTag.add,
remove: imageTag.remove
db
// addTag: imageTag.add,
// remove: imageTag.remove
},
a._id + a._rev
);

View File

@ -1,7 +1,7 @@
import { defineView as vw, defineElement as el } from 'domvm';
import { prop, computed } from 'frptools';
import { isObject } from '../utils/comparators.js';
import * as image from '../data/image.js';
import { AttachmentImageView } from './attachmentImage.js';
export function ThumbnailView(vm, model) {
@ -15,7 +15,10 @@ export function ThumbnailView(vm, model) {
const { doc, showTags, remove, removeTag } = model;
const { _id: id, _rev: rev, tags } = doc;
const _showTags = showTags !== undefined ? showTags : true;
const filteredTags = _showTags ? Object.entries(doc.tags).filter(([_, visible]) => visible) : [];
const filteredTags =
_showTags && isObject(doc.tags)
? Object.entries(doc.tags).filter(([_, visible]) => visible)
: [];
return el('div', { _key: id }, [
el(
@ -24,10 +27,7 @@ export function ThumbnailView(vm, model) {
onclick: { img: [remove, id, rev] }
},
[
vw(AttachmentImageView, {
src: doc,
attachmentKey: 'thumbnail'
}),
vw(AttachmentImageView, doc, doc._id + doc._rev),
filteredTags.length
? el(
'figcaption',

View File

@ -105,44 +105,46 @@ export function PouchORM(PouchDB) {
return docOrResultSet;
}
async function find(idOrQuery, live = false, raw = false) {
async function find(idOrQuery, live = false) {
let results = [];
if (typeof idOrQuery === 'undefined') {
results = await db.find({
selector: {
_id: { $gt: `${prefix}_0`, $lt: `${prefix}_\ufff0` }
}
});
} else if (typeof idOrQuery === 'string') {
results = await db.find({
selector: { _id: idOrQuery }
});
} else if (isObject(idOrQuery)) {
if (typeof idOrQuery === 'string') {
results = await db.get(idOrQuery);
} else {
const selector = Object.assign(
{ _deleted: { exists: false } },
isObject(idOrQuery) ? idOrQuery : { _id: { $gt: `${prefix}_0`, $lt: `${prefix}_\ufff0` } }
);
if (live) {
return LiveArray(db, idOrQuery, instantiate);
}
results = await db.find({
selector: idOrQuery
});
results = await db.find({ selector: idOrQuery });
}
return raw ? results : instantiate(results);
return instantiate(results);
}
async function _delete() {
try {
const { ok } = await db.remove(this);
this.update({ _id: undefined, _rev: undefined }, false);
return ok;
} catch (e) {
return false;
}
return await this.update({ _deleted: true });
}
function _new(props, save = false) {
async function _new(props, save = true) {
const doc = instantiate(populateId(props));
if (save) {
doc.save();
await doc.save();
}
return doc;
}
async function getOrCreate(props) {
let doc = await _new(props, false);
try {
await doc.save();
} catch (e) {
if (e.status !== 409) {
throw e;
}
doc = await find(doc._id);
}
return doc;
}
@ -150,7 +152,11 @@ export function PouchORM(PouchDB) {
return Object.assign(
{
new: _new,
find
getOrCreate,
find,
prefix,
db
// delete: // FIXME
},
opts.methods || {}
);

View File

@ -1,14 +1,18 @@
import { extractID } from './conversion.js';
import { equals } from './set.js';
import { pick } from './conversion.js';
const extractID = pick('_id');
const extractREV = pick('_rev');
export function pouchDocComparator(a, b) {
return extractID(a) === extractID(b) && extractREV(a) === extractREV(b);
}
export function pouchDocArrayComparator(a, b) {
if (!Array.isArray(a)) {
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) {
return false;
}
const aIDs = a.map(extractID);
const bIDs = b.map(extractID);
return equals(new Set(...aIDs), new Set(...bIDs));
return a.every((aRec, index) => pouchDocComparator(aRec, b[index]));
}
export function isObject(obj) {

View File

@ -36,6 +36,4 @@ export function deepAssign(to, ...rest) {
return to;
}
export function extractID(doc) {
return doc._id;
}
export const pick = id => doc => doc[id];

View File

@ -1,66 +1,7 @@
import { prop, computed } from 'frptools';
import { matchesSelector } from 'pouchdb-selector-core';
import { getDatabase } from '../services/db.js';
import { Watcher } from './watcher.js';
import { pouchDocArrayComparator } from './comparators.js';
import { difference } from './set.js';
// The point of the watcher mechanism is that PouchDB.changes doesn't register
// when a document changes in such a way that removes it from the selector
// specifications. For Example: a selector looks for images with a specific
// tag. If a change removes that tag, the changes API will not register a change
// event. globalWatcher watches the document IDs for exactly this type of
// change and triggers the LiveArray to refresh.
const watcherMap = new Map();
const watchingIDs = new Map();
const dbIDs = new Map();
function checkDocs(id, deleted, doc) {
// Is the changed doc one that we're watching?
if (watchingIDs.has(id)) {
const refresherMap = watchingIDs.get(id);
// if the doc doesn't match a watching selector, then refresh its LA
[...refresherMap.keys()]
.filter(s => !matchesSelector(doc, s))
.forEach(s => refresherMap.get(s)());
return;
}
}
function addID(db, id, selector, refresher) {
if (!watcherMap.has(db)) {
watcherMap.set(db, Watcher(db, {}, true)(checkDocs));
}
if (!dbIDs.has(db)) {
dbIDs.set(db, new Set());
}
dbIDs.get(db).add(id);
if (!watchingIDs.has(id)) {
watchingIDs.set(id, new Map());
}
watchingIDs.get(id).set(selector, refresher);
}
function removeID(db, id, selector) {
if (watchingIDs.has(id)) {
const idSet = watchingIDs.get(id);
idSet.delete(selector);
if (idSet.size === 0) {
watchingIDs.delete(selector);
}
const dbIDMap = dbIDs.get(db);
dbIDMap.delete(id);
if (dbIDMap.size === 0) {
// Unsubscribe from this watcher
watcherMap.get(db)();
dbIDs.delete(db);
}
}
}
// LiveArray is a subscribable property function that always returns the db results that match the provided selector and calls subscribers when the results change.
export function LiveArray(db, selector, mapper) {
@ -72,10 +13,6 @@ export function LiveArray(db, selector, mapper) {
const data = prop({ docs: [] });
const docs = computed(r => r.docs.map(_mapper), [data], pouchDocArrayComparator);
const idSet = () => docs().reduce((acc, d) => acc.add(d._id), new Set());
const addThisID = id => addID(db, id, selector, refresh);
const removeThisID = id => removeID(db, id, selector);
const cleanup = () => {
docs.unsubscribeAll();
ready.unsubscribeAll();
@ -83,18 +20,11 @@ export function LiveArray(db, selector, mapper) {
changeSub();
changeSub = null;
}
[...idSet()].forEach(removeThisID);
data({ docs: [] });
};
const refresh = async function refresh() {
const oldIdSet = idSet();
const refresh = async function refresh(...args) {
data(await db.find({ selector }));
const currentIDSet = idSet();
// Removes IDs not in the new set
[...difference(oldIdSet, currentIDSet)].forEach(removeThisID);
// Add IDs in the new set
[...difference(currentIDSet, oldIdSet)].forEach(addThisID);
};
docs.ready = ready;