navigator/scripts/osm-dump.mjs

#!/usr/bin/env node
/**
 * osm-dump.mjs — ingest OSM data from .osm.pbf data dumps (an alternative to
 * the live Overpass API in osm-data.mjs).
 *
 * For each configured source it:
 *   1. Downloads (and caches) a regional .osm.pbf extract (Geofabrik, BBBike,
 *      etc. — any URL works).
 *   2. Stream-parses it with osm-pbf-parser.
 *   3. Clips to a bounding box:
 *        - Collects POI features (amenity / shop / tourism / etc.) as Points.
 *        - Collects road ways and builds a routable ROAD GRAPH.
 *   4. Writes static/data/<id>-poi.geojson + <id>-graph.json and registers
 *      them in the manifest.
 *
 * Usage:
 *   node scripts/osm-dump.mjs                 # build all configured dumps
 *   node scripts/osm-dump.mjs --source=tulsa  # build one source
 *   node scripts/osm-dump.mjs --list          # list configured sources
 *
 * NOTE: This script intentionally never hits the Overpass API. The full
 * regional dump is cached under scripts/.cache/ so repeated runs skip the
 * network download.
 */

import { mkdir, writeFile, stat } from 'node:fs/promises';
import { createReadStream, existsSync } from 'node:fs';
import { dirname, resolve, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
import parseOSM from 'osm-pbf-parser';
import { pipeline } from 'node:stream/promises';

const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(__dirname, '..');
const OUT_DIR = resolve(ROOT, 'static', 'data');
const CACHE_DIR = resolve(__dirname, '.cache');

// ---------------------------------------------------------------------------
// Configuration. Each source: { id, name, url (a .osm.pbf), bbox, poiTags }.
// Roads (highway=*) are always extracted for the graph.
// ---------------------------------------------------------------------------
export const DUMPS = [
  {
    id: 'tulsa-dump',
    name: 'Tulsa OSM (dump)',
    url: 'https://download.geofabrik.de/north-america/us/oklahoma-latest.osm.pbf',
    bbox: [35.9, -96.1, 36.3, -95.7], // [south, west, north, east]
    // Road classes included in the routing graph. Major classes keep the graph
    // small enough to fetch in the browser; service/paths/appended local roads
    // are excluded by default.
    graphHighways: ['motorway', 'trunk', 'primary', 'secondary', 'tertiary'],
    poiTags: {
      amenity: ['school', 'fire_station', 'hospital', 'restaurant', 'cafe', 'fuel', 'parking'],
      shop: ['supermarket', 'convenience'],
      tourism: ['museum', 'attraction', 'hotel']
    }
  }
];

// ---------------------------------------------------------------------------
// Download helpers (with on-disk cache).
// ---------------------------------------------------------------------------
async function download(url, dest) {
  console.log(`  downloading ${url}`);
  const res = await fetch(url);
  if (!res.ok) throw new Error(`download failed: HTTP ${res.status} for ${url}`);
  const buf = Buffer.from(await res.arrayBuffer());
  await writeFile(dest, buf);
  console.log(`  cached ${buf.length} bytes -> ${dest}`);
}

async function ensureDump(cfg) {
  const cachedPath = resolve(CACHE_DIR, basename(new URL(cfg.url).pathname));
  await mkdir(dirname(cachedPath), { recursive: true });
  if (existsSync(cachedPath)) {
    const st = await stat(cachedPath);
    if (st.size > 0) {
      console.log(`  using cached dump: ${cachedPath}`);
      return cachedPath;
    }
  }
  await download(cfg.url, cachedPath);
  return cachedPath;
}

// Haversine distance in meters.
function haversine(a, b) {
  const R = 6371000;
  const [lata, lona] = a;
  const [latb, lonb] = b;
  const dLat = ((latb - lata) * Math.PI) / 180;
  const dLon = ((lonb - lona) * Math.PI) / 180;
  const s =
    Math.sin(dLat / 2) ** 2 +
    Math.cos((lata * Math.PI) / 180) * Math.cos((latb * Math.PI) / 180) * Math.sin(dLon / 2) ** 2;
  return 2 * R * Math.asin(Math.sqrt(s));
}

function inBbox(lat, lon, [s, w, n, e], margin = 0.02) {
  return lat >= s - margin && lat <= n + margin && lon >= w - margin && lon <= e + margin;
}

// ---------------------------------------------------------------------------
// Stream-parse the dump and build POI features + a road graph for the bbox.
// ---------------------------------------------------------------------------
async function buildFromDump(cfg) {
  const parser = parseOSM();
  const pois = [];

  const nodeCoords = new Map();   // osm nodeId -> [lat, lon]
  const nodeIdx = new Map();      // osm nodeId -> compact index
  const coords = [];              // coords[i] = [lat, lon]
  const adj = [];                 // adj[i] = [[neighborIdx, meters], ...]
  const poiTagSet = new Set(
    Object.entries(cfg.poiTags).flatMap(([k, vals]) => vals.map((v) => `${k}=${v}`))
  );

  const parserInput = createReadStream(await ensureDump(cfg));

  await new Promise((resolvePromise, reject) => {
    parser
      .on('data', (items) => {
        for (const it of items) {
          if (it.type === 'node') {
            if (inBbox(it.lat, it.lon, cfg.bbox)) {
              nodeCoords.set(it.id, [it.lat, it.lon]);
              const key = Object.keys(it.tags || {}).find((k) =>
                poiTagSet.has(`${k}=${it.tags[k]}`));
              if (key) {
                pois.push({
                  type: 'Feature',
                  id: `dump/${cfg.id}/node/${it.id}`,
                  properties: { ...it.tags, name: it.tags?.name, _type: 'node', _id: it.id },
                  geometry: { type: 'Point', coordinates: [it.lon, it.lat] }
                });
              }
            }
          } else if (it.type === 'way') {
            const hw = it.tags?.highway;
            if (hw && cfg.graphHighways && !cfg.graphHighways.includes(hw)) continue;
            // POI footprints (non-road ways with a targeted tag).
            if (!hw) {
              const key = Object.keys(it.tags || {}).find((k) =>
                poiTagSet.has(`${k}=${it.tags[k]}`));
              if (key) {
                const pts = it.refs.map((rid) => nodeCoords.get(rid)).filter(Boolean);
                if (pts.length) {
                  const lat = pts.reduce((a, p) => a + p[0], 0) / pts.length;
                  const lon = pts.reduce((a, p) => a + p[1], 0) / pts.length;
                  if (inBbox(lat, lon, cfg.bbox)) {
                    pois.push({
                      type: 'Feature',
                      id: `dump/${cfg.id}/way/${it.id}`,
                      properties: { ...it.tags, name: it.tags?.name, _type: 'way', _id: it.id },
                      geometry: { type: 'Point', coordinates: [lon, lat] }
                    });
                  }
                }
              }
              continue;
            }
            // Road: snap refs into the compact graph.
            const pts = [];
            for (const rid of it.refs) {
              const c = nodeCoords.get(rid);
              if (!c) continue;
              if (!nodeIdx.has(rid)) {
                nodeIdx.set(rid, coords.length);
                coords.push(c);
                adj.push([]);
              }
              pts.push(nodeIdx.get(rid));
            }
            for (let i = 0; i < pts.length - 1; i++) {
              const a = pts[i];
              const b = pts[i + 1];
              if (a === b) continue;
              const d = haversine(coords[a], coords[b]);
              adj[a].push([b, d]);
              adj[b].push([a, d]);
            }
          }
        }
      })
      .on('end', resolvePromise)
      .on('error', reject);
    pipeline(parserInput, parser).catch(reject);
  });

  const edgeCount = Math.round(adj.reduce((a, x) => a + x.length, 0) / 2);
  console.log(`  parsed ${coords.length} graph nodes, ${edgeCount} road edges, ${pois.length} POIs`);

  // --- write POI GeoJSON ---
  const poiFc = {
    type: 'FeatureCollection',
    crs: { type: 'name', properties: { name: 'urn:ogc:def:crs:OGC:1.3:CRS84' } },
    features: pois
  };
  const poiPath = resolve(OUT_DIR, `${cfg.id}-poi.geojson`);
  await writeFile(poiPath, JSON.stringify(poiFc));
  console.log(`  wrote ${pois.length} POIs -> ${poiPath}`);

  // --- write routing graph ---
  const graph = {
    meta: { source: cfg.url, bbox: cfg.bbox, nodeCount: coords.length, edgeCount },
    coords,
    adj
  };
  const graphPath = resolve(OUT_DIR, `${cfg.id}-graph.json`);
  await writeFile(graphPath, JSON.stringify(graph));
  console.log(`  wrote routing graph -> ${graphPath}`);

  return {
    name: cfg.id,
    label: cfg.name,
    category: 'dump',
    layerType: 'poi',
    color: '#8b5cf6',
    featureCount: pois.length,
    path: `/data/${cfg.id}-poi.geojson`,
    poiPath: `/data/${cfg.id}-poi.geojson`,
    graphPath: `/data/${cfg.id}-graph.json`,
    graphNodeCount: coords.length
  };
}

// ---------------------------------------------------------------------------
// Manifest handling + main.
// ---------------------------------------------------------------------------
async function main() {
  const args = process.argv.slice(2);
  if (args.includes('--list')) {
    console.log('Configured dump sources:\n');
    for (const d of DUMPS) console.log(`  ${d.id}: ${d.name} (${d.url})`);
    return;
  }
  const onlyArg = args.find((a) => a.startsWith('--source='));
  const only = onlyArg ? onlyArg.split('=')[1] : null;

  await mkdir(OUT_DIR, { recursive: true });

  let manifest = [];
  const { existsSync: exists } = await import('node:fs');
  const manifestPath = resolve(OUT_DIR, '_index.json');
  if (exists(manifestPath)) {
    const { readFile } = await import('node:fs/promises');
    manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
  }

  for (const cfg of DUMPS) {
    if (only && cfg.id !== only) continue;
    console.log(`\n==> Building dump source "${cfg.id}"`);
    const entry = await buildFromDump(cfg);
    manifest = manifest.filter((m) => !(m.name === cfg.id && m.category === 'dump'));
    manifest.push(entry);
  }

  await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
  console.log(`\nWrote manifest -> ${manifestPath}`);
  console.log('Done.');
}

main().catch((e) => {
  console.error('Fatal:', e.message);
  process.exit(1);
});