From cb8e61a03e955399a2c8bcb2bea3392be352db59 Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Wed, 17 Jun 2026 22:36:04 -0700 Subject: [PATCH] #290: live viewport/cross-filtered counts for the Material facet tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Material tree shipped with STATIC global baseline counts; this makes them live when the user zooms in. Map/table filtering was already live — this is the legend. - describeCrossFilters: Material participates via materialSelection() (minimal nodes) ONLY when zoomed (!isGlobalView); at/near global it stays baseline. - buildCrossFilterWhere: Material cross-filters OTHER dims via a membership pid-subquery (concept_uri IN selected) — a selected parent matches its subtree. - updateCrossFilteredCounts: cube disabled in tree mode (no tree nodes in the cube); Material own-counts at global → baseline (instant); zoomed → membership query COUNT(DISTINCT pid) per concept_uri, scoped by bbox (lite JOIN) + other dims (facets_v3 pid-subquery) + search. GLOBAL GATE (perf): the membership COUNT(DISTINCT) is a near-full scan at global/ large views and starved the single DuckDB-WASM connection (samples-table query timed out). At true-global the baseline IS the correct global count, so we use it (instant); live counts engage when zoomed, where the bbox prunes the scan. Verified (202608): earthmaterial 4,091,133 global → 25,988 at Cyprus; legend(node) == table(node filter) = 26,310 (coherence #245); parent>=child in-viewport; source⇄ material cross-filter both directions. 8 facet-tree specs + smoke green; render clean. Codex: no blocking findings (verified cross-filter via direct DuckDB queries). Accepted residual (documented): at global view WITH another filter/search, Material counts show baseline (instant, slightly stale) — a precomputed facet_tree_cross_filter cube is the follow-up if live global cross-filtered material counts are wanted. Co-Authored-By: Claude Opus 4.8 (1M context) --- explorer.qmd | 70 ++++++++++++++++++--- tests/playwright/facet-tree.spec.js | 95 ++++++++++++++++++++++++++++- 2 files changed, 155 insertions(+), 10 deletions(-) diff --git a/explorer.qmd b/explorer.qmd index b598c95..27f440d 100644 --- a/explorer.qmd +++ b/explorer.qmd @@ -3012,13 +3012,16 @@ zoomWatcher = { const sourceChecks = document.querySelectorAll('#sourceFilter input[type="checkbox"]'); const sourceTotal = sourceChecks.length; const sources = getActiveSources(); - // #281/#282 increment 1: in tree mode the Material facet keeps its - // STATIC tree baseline counts and is excluded from the live cross-filter - // count engine (its counts come from facet_tree_summaries, not facets_v3, - // and a selected parent node wouldn't match facets_v3's flat values). The - // table/map still filter correctly via facetFilterSQL's membership path. - // Viewport/cross-filtered material tree counts are the next increment. - const mat = materialTreeActive() ? [] : getCheckedValues('materialFilterBody'); + // #290: in tree mode Material participates in the live count engine via + // its MINIMAL node selection (materialSelection) — but ONLY when zoomed in + // (!isGlobalView). The membership COUNT(DISTINCT pid) query is a near-full + // scan at global/near-global views and would starve the single DuckDB-WASM + // connection (incl. the samples-table query); at global the static tree + // baseline IS the correct global count, so we use it (instant). Its own + // per-node counts + cross-filter contribution use the membership table (NOT + // facets_v3's flat value) — see buildCrossFilterWhere + updateCrossFilteredCounts. + const mat = (materialTreeActive() && !isGlobalView()) ? materialSelection() + : (materialTreeActive() ? [] : getCheckedValues('materialFilterBody')); const ctx = getCheckedValues('contextFilterBody'); const ot = getCheckedValues('objectTypeFilterBody'); const dims = [ @@ -3048,6 +3051,13 @@ zoomWatcher = { .filter(d => d.key !== excludeFacet) .map(d => { const list = d.values.map(v => `'${escSql(v)}'`).join(','); + // #290: in tree mode the Material selection is a set of concept + // NODES — constrain via the membership table (which encodes every + // ancestor) so a selected parent matches its whole subtree, rather + // than facets_v3's single flat `material` value. + if (d.key === 'material' && materialTreeActive()) { + return `${colPrefix}pid IN (SELECT pid FROM read_parquet('${membership_url}') WHERE facet_type='material' AND concept_uri IN (${list}))`; + } return `${colPrefix}${d.col} IN (${list})`; }); @@ -3112,7 +3122,12 @@ zoomWatcher = { const singleActiveDim = !sourceImpossible && activeDims.length === 1 && activeDims[0].values.length === 1 ? activeDims[0] : null; - if (singleActiveDim && totalActiveValues === 1 && bboxSQL === null && !searchIsActive()) { + // #290: the cube (facet_cross_filter) is pre-aggregated over flat facet + // values and has no tree nodes, so it can't answer Material tree counts + // (nor a Material-node cross-filter). In tree mode, always take the slow + // path; the common global-no-filter case is still fast via the baseline + // early-return above (material baseline = the global tree counts). + if (singleActiveDim && totalActiveValues === 1 && bboxSQL === null && !searchIsActive() && !materialTreeActive()) { try { const filterCols = ['filter_source', 'filter_material', 'filter_context', 'filter_object_type']; const filterColForKey = { @@ -3150,7 +3165,44 @@ zoomWatcher = { await Promise.all(dims.map(async (d) => { try { let rows; - if (bboxSQL) { + if (d.key === 'material' && materialTreeActive() && !bboxSQL) { + // #290: global / near-global view → the static tree baseline IS + // the correct global count (instant). Avoids a near-full-scan + // membership query that would starve the WASM connection. + applyFacetCounts('material', null); + return; + } + if (d.key === 'material' && materialTreeActive()) { + // #290: live Material tree counts from membership — COUNT(DISTINCT + // pid) per concept node, scoped to viewport (bbox via lite JOIN) + + // the OTHER active dims (a facets_v3 pid-subquery) + search. NOT + // filtered by Material's own selection (show all nodes' counts), and + // distinct-pid so ancestor rows don't inflate. Parent ≥ child holds. + const others = activeDims.filter(x => x.key !== 'material'); + let otherCond = ''; + if (sourceImpossible) { + otherCond = ' AND 1=0'; + } else if (others.length) { + const oc = others.map(x => `${x.col} IN (${x.values.map(v => `'${escSql(v)}'`).join(',')})`).join(' AND '); + otherCond = ` AND m.pid IN (SELECT DISTINCT pid FROM read_parquet('${facets_url}') WHERE ${oc})`; + } + if (bboxSQL) { + rows = await db.query(` + SELECT m.concept_uri AS value, COUNT(DISTINCT m.pid) AS count + FROM read_parquet('${membership_url}') m + JOIN read_parquet('${lite_url}') l ON l.pid = m.pid + WHERE m.facet_type='material'${otherCond}${bboxSQL}${searchFilterSQL('m.pid')} + GROUP BY m.concept_uri + `); + } else { + rows = await db.query(` + SELECT m.concept_uri AS value, COUNT(DISTINCT m.pid) AS count + FROM read_parquet('${membership_url}') m + WHERE m.facet_type='material'${otherCond}${searchFilterSQL('m.pid')} + GROUP BY m.concept_uri + `); + } + } else if (bboxSQL) { // B1 bbox-scoped slow path: JOIN facets_url to lite_url // on pid so we can filter by lite.latitude / lite.longitude. // facets_url has no coordinates of its own. Per-PR follow-up: diff --git a/tests/playwright/facet-tree.spec.js b/tests/playwright/facet-tree.spec.js index d5663d3..65d5799 100644 --- a/tests/playwright/facet-tree.spec.js +++ b/tests/playwright/facet-tree.spec.js @@ -16,7 +16,9 @@ const { test, expect } = require('@playwright/test'); const LOCAL = !!process.env.FACET_TREE_LOCAL; const DATA = LOCAL ? '&data_base=/data' : ''; -const WORLD = '#v=1&lat=20&lng=0&alt=10000000'; +// Clearly-global altitude (> isGlobalView's 1e7 threshold) so Material counts take +// the fast baseline path (live membership counts are reserved for zoomed views). +const WORLD = '#v=1&lat=20&lng=0&alt=15000000'; test.describe('Material facet tree (#281/#282 preview)', () => { test.skip(!LOCAL, 'needs hierarchy data — run with FACET_TREE_LOCAL=1 against the docs/data mirror until R2 publish'); @@ -141,6 +143,97 @@ test.describe('Material facet tree (#281/#282 preview)', () => { expect(restored).toEqual({ parentChecked: true, kidChecked: true, kidDisabled: true }); }); + // #290: live viewport / cross-filtered Material tree counts (from membership). + const legendCount = (page, sub) => page.evaluate((s) => { + const sp = document.querySelector(`#materialFilterBody .facet-count[data-value*="${s}"]`); + const m = (sp?.textContent || '').match(/([\d,]+)/); + return m ? parseInt(m[1].replace(/,/g, ''), 10) : null; + }, sub); + const tableTotal = (page) => page.evaluate(() => { + const m = (document.getElementById('tablePageInfo')?.textContent || '').match(/of ([\d,]+)\)/); + return m ? parseInt(m[1].replace(/,/g, ''), 10) : null; + }); + + test('live counts: tree node counts shrink to the viewport (not static baseline)', async ({ page }) => { + test.setTimeout(180000); + // Global view → baseline (global tree counts). + await page.goto(`/explorer.html?facets=tree${DATA}#v=1&lat=0&lng=0&alt=15000000`); + await page.waitForFunction(() => document.querySelectorAll('#materialFilterBody .facet-treenode').length > 0, null, { timeout: 90000 }); + await page.waitForTimeout(2500); + const globalEarth = await legendCount(page, '/earthmaterial'); + expect(globalEarth).toBeGreaterThan(1000000); + // Zoom to a small region → the same node's count must drop (viewport-scoped). + await page.evaluate(async () => { + const v = await window._ojs.ojsConnector.mainModule.value('viewer'); + v.scene.requestRenderMode = false; + v.camera.flyTo({ destination: window.Cesium.Cartesian3.fromDegrees(33, 35, 400000), duration: 0 }); + }); + await page.waitForTimeout(4000); + const zoomedEarth = await legendCount(page, '/earthmaterial'); + expect(zoomedEarth).toBeLessThan(globalEarth); + expect(zoomedEarth).toBeGreaterThanOrEqual(0); + }); + + test('live counts coherence: legend(node) == table when that node is the filter (#245), parent >= child', async ({ page }) => { + test.setTimeout(180000); + await page.goto(`/explorer.html?facets=tree${DATA}#v=1&lat=35&lng=33&alt=500000`); + await page.waitForFunction(() => document.querySelectorAll('#materialFilterBody .facet-treenode').length > 0, null, { timeout: 90000 }); + await page.waitForTimeout(3500); + const legEarth = await legendCount(page, '/earthmaterial'); + const legRock = await legendCount(page, '/rock'); + expect(legEarth).toBeGreaterThanOrEqual(legRock); // parent >= child, in-viewport + expect(legEarth).toBeGreaterThan(0); + // Selecting earthmaterial filters the table to exactly its viewport legend count. + await page.evaluate(() => { + const cb = document.querySelector('#materialFilterBody input[value*="/earthmaterial"]'); + cb.checked = true; cb.dispatchEvent(new Event('change', { bubbles: true })); + }); + await expect.poll(() => tableTotal(page), { timeout: 60000, intervals: [500, 1000, 2000] }).toBe(legEarth); + }); + + test('live counts cross-filter both ways (zoomed): a source narrows Material; Material narrows sources', async ({ page }) => { + test.setTimeout(180000); + const sumCounts = (page, container) => page.evaluate((c) => { + let s = 0; + document.querySelectorAll(`#${c} .facet-count`).forEach(el => { + const m = (el.textContent || '').match(/([\d,]+)/); + if (m) s += parseInt(m[1].replace(/,/g, ''), 10); + }); + return s; + }, container); + await page.goto(`/explorer.html?facets=tree${DATA}#v=1&lat=35&lng=33&alt=500000`); + await page.waitForFunction(() => document.querySelectorAll('#materialFilterBody .facet-treenode').length > 0, null, { timeout: 90000 }); + await page.waitForTimeout(3500); + const matEarth0 = await legendCount(page, '/earthmaterial'); + expect(matEarth0).toBeGreaterThan(0); + + // (a) source → material: unchecking a source must not INCREASE a material count. + await page.evaluate(() => { + const cb = document.querySelector('#sourceFilter input[type="checkbox"]:checked'); + if (cb) { cb.checked = false; cb.dispatchEvent(new Event('change', { bubbles: true })); } + }); + await page.waitForTimeout(3000); + const matEarth1 = await legendCount(page, '/earthmaterial'); + expect(matEarth1).toBeLessThanOrEqual(matEarth0); + + // restore source, then (b) material → source: selecting a Material node must not + // INCREASE the source-count total (it scopes sources to that subtree). + await page.evaluate(() => { + const cb = document.querySelector('#sourceFilter input[type="checkbox"]:not(:checked)'); + if (cb) { cb.checked = true; cb.dispatchEvent(new Event('change', { bubbles: true })); } + }); + await page.waitForTimeout(3000); + const srcSum0 = await sumCounts(page, 'sourceFilter'); + await page.evaluate(() => { + const cb = document.querySelector('#materialFilterBody input[value*="/earthmaterial"]'); + cb.checked = true; cb.dispatchEvent(new Event('change', { bubbles: true })); + }); + await page.waitForTimeout(3000); + const srcSum1 = await sumCounts(page, 'sourceFilter'); + expect(srcSum1).toBeLessThanOrEqual(srcSum0); + expect(srcSum1).toBeGreaterThan(0); + }); + test('graceful fallback: if the tree data 404s, Material renders flat and still filters', async ({ page }) => { // Deploy-safety (Codex r2/r3): with ?facets=tree but the hierarchy files // missing, renderMaterialTreeFacet() catches and renders the flat list, and