hermes-explorigin
fbf939d236
All checks were successful
CI / test-and-build (push) Successful in 47s
- osm-dump.mjs: capture way name/ref as edgeNames; honor oneway & roundabout as directed edges; graphHighways widened to include named local roads - routing.ts: buildTurnInstructions emits 'Head north on Pine St' / 'Turn right onto 21st St'; merges short legs to avoid phantom turns; nameResolverFor() maps node path to street names - Rebuilt tulsa-dump-graph.json: 225,956 nodes / 227,468 edges (was 55k); ~27MB (was 4.7MB) - tests, README updated
299 lines
12 KiB
JavaScript
299 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* osm-dump.mjs — ingest OSM data from .osm.pbf data dumps (an alternative to
|
|
* the live Overpass API in osm-data.mjs).
|
|
*
|
|
* For each configured source it:
|
|
* 1. Downloads (and caches) a regional .osm.pbf extract (Geofabrik, BBBike,
|
|
* etc. — any URL works).
|
|
* 2. Stream-parses it with osm-pbf-parser.
|
|
* 3. Clips to a bounding box:
|
|
* - Collects POI features (amenity / shop / tourism / etc.) as Points.
|
|
* - Collects road ways and builds a routable ROAD GRAPH.
|
|
* 4. Writes static/data/<id>-poi.geojson + <id>-graph.json and registers
|
|
* them in the manifest.
|
|
*
|
|
* Usage:
|
|
* node scripts/osm-dump.mjs # build all configured dumps
|
|
* node scripts/osm-dump.mjs --source=tulsa # build one source
|
|
* node scripts/osm-dump.mjs --list # list configured sources
|
|
*
|
|
* NOTE: This script intentionally never hits the Overpass API. The full
|
|
* regional dump is cached under scripts/.cache/ so repeated runs skip the
|
|
* network download.
|
|
*/
|
|
|
|
import { mkdir, writeFile, stat } from 'node:fs/promises';
|
|
import { createReadStream, existsSync } from 'node:fs';
|
|
import { dirname, resolve, basename } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import parseOSM from 'osm-pbf-parser';
|
|
import { pipeline } from 'node:stream/promises';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const ROOT = resolve(__dirname, '..');
|
|
const OUT_DIR = resolve(ROOT, 'static', 'data');
|
|
const CACHE_DIR = resolve(__dirname, '.cache');
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Configuration. Each source: { id, name, url (a .osm.pbf), bbox, poiTags }.
|
|
// Roads (highway=*) are always extracted for the graph.
|
|
// ---------------------------------------------------------------------------
|
|
export const DUMPS = [
|
|
{
|
|
id: 'tulsa-dump',
|
|
name: 'Tulsa OSM (dump)',
|
|
url: 'https://download.geofabrik.de/north-america/us/oklahoma-latest.osm.pbf',
|
|
bbox: [35.9, -96.1, 36.3, -95.7], // [south, west, north, east]
|
|
// Road classes included in the routing graph. Major classes keep the graph
|
|
// small enough to fetch in the browser; service/paths/appended local roads
|
|
// are excluded by default.
|
|
// Road classes included in the routing graph. Includes local streets so
|
|
// street names appear in turn-by-turn instructions. One-way/roundabout
|
|
// handling below makes the graph directed where the data says so.
|
|
graphHighways: [
|
|
'motorway', 'motorway_link',
|
|
'trunk', 'trunk_link',
|
|
'primary', 'primary_link',
|
|
'secondary', 'secondary_link',
|
|
'tertiary', 'tertiary_link',
|
|
'unclassified', 'residential', 'living_street'
|
|
],
|
|
poiTags: {
|
|
amenity: ['school', 'fire_station', 'hospital', 'restaurant', 'cafe', 'fuel', 'parking'],
|
|
shop: ['supermarket', 'convenience'],
|
|
tourism: ['museum', 'attraction', 'hotel']
|
|
}
|
|
}
|
|
];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Download helpers (with on-disk cache).
|
|
// ---------------------------------------------------------------------------
|
|
async function download(url, dest) {
|
|
console.log(` downloading ${url}`);
|
|
const res = await fetch(url);
|
|
if (!res.ok) throw new Error(`download failed: HTTP ${res.status} for ${url}`);
|
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
await writeFile(dest, buf);
|
|
console.log(` cached ${buf.length} bytes -> ${dest}`);
|
|
}
|
|
|
|
async function ensureDump(cfg) {
|
|
const cachedPath = resolve(CACHE_DIR, basename(new URL(cfg.url).pathname));
|
|
await mkdir(dirname(cachedPath), { recursive: true });
|
|
if (existsSync(cachedPath)) {
|
|
const st = await stat(cachedPath);
|
|
if (st.size > 0) {
|
|
console.log(` using cached dump: ${cachedPath}`);
|
|
return cachedPath;
|
|
}
|
|
}
|
|
await download(cfg.url, cachedPath);
|
|
return cachedPath;
|
|
}
|
|
|
|
// Haversine distance in meters.
|
|
function haversine(a, b) {
|
|
const R = 6371000;
|
|
const [lata, lona] = a;
|
|
const [latb, lonb] = b;
|
|
const dLat = ((latb - lata) * Math.PI) / 180;
|
|
const dLon = ((lonb - lona) * Math.PI) / 180;
|
|
const s =
|
|
Math.sin(dLat / 2) ** 2 +
|
|
Math.cos((lata * Math.PI) / 180) * Math.cos((latb * Math.PI) / 180) * Math.sin(dLon / 2) ** 2;
|
|
return 2 * R * Math.asin(Math.sqrt(s));
|
|
}
|
|
|
|
function inBbox(lat, lon, [s, w, n, e], margin = 0.02) {
|
|
return lat >= s - margin && lat <= n + margin && lon >= w - margin && lon <= e + margin;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stream-parse the dump and build POI features + a road graph for the bbox.
|
|
// ---------------------------------------------------------------------------
|
|
async function buildFromDump(cfg) {
|
|
const parser = parseOSM();
|
|
const pois = [];
|
|
|
|
const nodeCoords = new Map(); // osm nodeId -> [lat, lon]
|
|
const nodeIdx = new Map(); // osm nodeId -> compact index
|
|
const coords = []; // coords[i] = [lat, lon]
|
|
const adj = []; // adj[i] = [[neighborIdx, meters], ...]
|
|
const roadNames = new Map(); // "a|b" -> street name (for turn-by-turn)
|
|
const poiTagSet = new Set(
|
|
Object.entries(cfg.poiTags).flatMap(([k, vals]) => vals.map((v) => `${k}=${v}`))
|
|
);
|
|
|
|
const parserInput = createReadStream(await ensureDump(cfg));
|
|
|
|
await new Promise((resolvePromise, reject) => {
|
|
parser
|
|
.on('data', (items) => {
|
|
for (const it of items) {
|
|
if (it.type === 'node') {
|
|
if (inBbox(it.lat, it.lon, cfg.bbox)) {
|
|
nodeCoords.set(it.id, [it.lat, it.lon]);
|
|
const key = Object.keys(it.tags || {}).find((k) =>
|
|
poiTagSet.has(`${k}=${it.tags[k]}`));
|
|
if (key) {
|
|
pois.push({
|
|
type: 'Feature',
|
|
id: `dump/${cfg.id}/node/${it.id}`,
|
|
properties: { ...it.tags, name: it.tags?.name, _type: 'node', _id: it.id },
|
|
geometry: { type: 'Point', coordinates: [it.lon, it.lat] }
|
|
});
|
|
}
|
|
}
|
|
} else if (it.type === 'way') {
|
|
const hw = it.tags?.highway;
|
|
if (hw && cfg.graphHighways && !cfg.graphHighways.includes(hw)) continue;
|
|
// POI footprints (non-road ways with a targeted tag).
|
|
if (!hw) {
|
|
const key = Object.keys(it.tags || {}).find((k) =>
|
|
poiTagSet.has(`${k}=${it.tags[k]}`));
|
|
if (key) {
|
|
const pts = it.refs.map((rid) => nodeCoords.get(rid)).filter(Boolean);
|
|
if (pts.length) {
|
|
const lat = pts.reduce((a, p) => a + p[0], 0) / pts.length;
|
|
const lon = pts.reduce((a, p) => a + p[1], 0) / pts.length;
|
|
if (inBbox(lat, lon, cfg.bbox)) {
|
|
pois.push({
|
|
type: 'Feature',
|
|
id: `dump/${cfg.id}/way/${it.id}`,
|
|
properties: { ...it.tags, name: it.tags?.name, _type: 'way', _id: it.id },
|
|
geometry: { type: 'Point', coordinates: [lon, lat] }
|
|
});
|
|
}
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
// Road: snap refs into the compact graph.
|
|
const pts = [];
|
|
for (const rid of it.refs) {
|
|
const c = nodeCoords.get(rid);
|
|
if (!c) continue;
|
|
if (!nodeIdx.has(rid)) {
|
|
nodeIdx.set(rid, coords.length);
|
|
coords.push(c);
|
|
adj.push([]);
|
|
}
|
|
pts.push(nodeIdx.get(rid));
|
|
}
|
|
|
|
// Street name for turn-by-turn (name prefered, ref as fallback).
|
|
const streetName = it.tags?.name || it.tags?.ref || '';
|
|
|
|
// One-way? OSM oneway: yes/true/1 = forward (ref order), -1 = reverse.
|
|
// Roundabouts & circular junctions are one-way by convention.
|
|
const ow = String(it.tags?.oneway ?? '').toLowerCase();
|
|
const roundabout = it.tags?.junction === 'roundabout' || it.tags?.junction === 'circular';
|
|
const onewayFwd = roundabout || ['yes', 'true', '1'].includes(ow);
|
|
const onewayRev = ow === '-1';
|
|
|
|
for (let i = 0; i < pts.length - 1; i++) {
|
|
const a = pts[i];
|
|
const b = pts[i + 1];
|
|
if (a === b) continue;
|
|
const d = haversine(coords[a], coords[b]);
|
|
if (streetName) {
|
|
roadNames.set(a < b ? `${a}|${b}` : `${b}|${a}`, streetName);
|
|
}
|
|
// Two-way: add both directions. One-way: only the legal direction.
|
|
if (onewayRev) {
|
|
adj[b].push([a, d]); // travel opposite to ref order
|
|
} else if (onewayFwd) {
|
|
adj[a].push([b, d]); // travel in ref order
|
|
} else {
|
|
adj[a].push([b, d]);
|
|
adj[b].push([a, d]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
.on('end', resolvePromise)
|
|
.on('error', reject);
|
|
pipeline(parserInput, parser).catch(reject);
|
|
});
|
|
|
|
const edgeCount = Math.round(adj.reduce((a, x) => a + x.length, 0) / 2);
|
|
console.log(` parsed ${coords.length} graph nodes, ${edgeCount} road edges, ${pois.length} POIs`);
|
|
|
|
// --- write POI GeoJSON ---
|
|
const poiFc = {
|
|
type: 'FeatureCollection',
|
|
crs: { type: 'name', properties: { name: 'urn:ogc:def:crs:OGC:1.3:CRS84' } },
|
|
features: pois
|
|
};
|
|
const poiPath = resolve(OUT_DIR, `${cfg.id}-poi.geojson`);
|
|
await writeFile(poiPath, JSON.stringify(poiFc));
|
|
console.log(` wrote ${pois.length} POIs -> ${poiPath}`);
|
|
|
|
// --- write routing graph ---
|
|
const graph = {
|
|
meta: { source: cfg.url, bbox: cfg.bbox, nodeCount: coords.length, edgeCount },
|
|
coords,
|
|
adj,
|
|
edgeNames: Object.fromEntries(roadNames)
|
|
};
|
|
const graphPath = resolve(OUT_DIR, `${cfg.id}-graph.json`);
|
|
await writeFile(graphPath, JSON.stringify(graph));
|
|
console.log(` wrote routing graph -> ${graphPath}`);
|
|
|
|
return {
|
|
name: cfg.id,
|
|
label: cfg.name,
|
|
category: 'dump',
|
|
layerType: 'poi',
|
|
color: '#8b5cf6',
|
|
featureCount: pois.length,
|
|
path: `/data/${cfg.id}-poi.geojson`,
|
|
poiPath: `/data/${cfg.id}-poi.geojson`,
|
|
graphPath: `/data/${cfg.id}-graph.json`,
|
|
graphNodeCount: coords.length
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Manifest handling + main.
|
|
// ---------------------------------------------------------------------------
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
if (args.includes('--list')) {
|
|
console.log('Configured dump sources:\n');
|
|
for (const d of DUMPS) console.log(` ${d.id}: ${d.name} (${d.url})`);
|
|
return;
|
|
}
|
|
const onlyArg = args.find((a) => a.startsWith('--source='));
|
|
const only = onlyArg ? onlyArg.split('=')[1] : null;
|
|
|
|
await mkdir(OUT_DIR, { recursive: true });
|
|
|
|
let manifest = [];
|
|
const { existsSync: exists } = await import('node:fs');
|
|
const manifestPath = resolve(OUT_DIR, '_index.json');
|
|
if (exists(manifestPath)) {
|
|
const { readFile } = await import('node:fs/promises');
|
|
manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
|
}
|
|
|
|
for (const cfg of DUMPS) {
|
|
if (only && cfg.id !== only) continue;
|
|
console.log(`\n==> Building dump source "${cfg.id}"`);
|
|
const entry = await buildFromDump(cfg);
|
|
manifest = manifest.filter((m) => !(m.name === cfg.id && m.category === 'dump'));
|
|
manifest.push(entry);
|
|
}
|
|
|
|
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
|
console.log(`\nWrote manifest -> ${manifestPath}`);
|
|
console.log('Done.');
|
|
}
|
|
|
|
main().catch((e) => {
|
|
console.error('Fatal:', e.message);
|
|
process.exit(1);
|
|
}); |