Performance refactor

app.js

@@ -42,7 +42,7 @@ async function init() {
     prototype.nunjucks.addGlobal(name, global)
   }
 
-  prototype.start(config.port)
+  prototype.start(port)
 }
 
 init()

app/data/session-data-defaults.js

@@ -27,9 +27,6 @@ if (!fs.existsSync(generatedDataPath)) {
   fs.mkdirSync(generatedDataPath)
 }
 
-let participants = []
-let clinics = []
-let events = []
 let generationInfo = {
   generatedAt: 'Never',
   seedDataProfile: DEFAULT_SEED_DATA_PROFILE,
@@ -73,17 +70,17 @@ if (needsRegeneration(generationInfo)) {
   if (!generationInfo.seedDataProfile) {
     generationInfo.seedDataProfile = DEFAULT_SEED_DATA_PROFILE
   }
-}
 
-// Load generated data
-try {
-  participants = require('./generated/participants.json').participants
-  clinics = require('./generated/clinics.json').clinics
-  events = require('./generated/events.json').events
-} catch (err) {
-  console.warn('Error loading generated data:', err)
+  // The shared data store may already have loaded the stale files (require
+  // order at boot), so reload it now the generator has written fresh ones
+  require('../lib/data-store').reload()
 }
 
+// The generated collections (participants, clinics, events) are NOT loaded
+// here: they live in the shared data store (app/lib/data-store.js) and are
+// attached to each request by middleware in app/routes.js, so they never get
+// copied into (or serialised with) sessions. Same for generationInfo.
+
 const defaultSettings = {
   darkMode: 'false',
   compactMode: 'false',
@@ -132,10 +129,6 @@ const defaults = {
   breastScreeningUnits,
   allBreastScreeningUnits,
   screeningRooms,
-  participants,
-  clinics,
-  events,
-  generationInfo,
   config,
   settings: defaultSettings,
   defaultSettings,

app/lib/data-store.js

@@ -0,0 +1,147 @@
+// app/lib/data-store.js
+//
+// Shared, read-only store for the generated seed data.
+//
+// Loaded once at boot and shared across every session. Sessions no longer
+// carry their own copy of the big collections - they only store changed
+// records in `data._changes` (see the attach middleware in app/routes.js),
+// which are overlaid on these shared arrays at request time.
+//
+// Because these arrays and records are shared across sessions, nothing may
+// mutate them at request time. To change a record, code must go through the
+// update helpers (updateEvent etc), which write a whole replacement record
+// into the session's `data._changes`. In development the store is deep-frozen
+// so any leftover in-place mutation throws at the exact line, rather than
+// silently leaking one session's changes into every other session. See
+// docs/data-conventions.md.
+
+const fs = require('fs')
+const path = require('path')
+const config = require('../config')
+
+const generatedDataPath = config.paths.generatedData
+
+// Freeze shared data in development so accidental mutation fails loudly.
+// Escape hatch: set freezeSharedData: false in config if a frozen object is
+// ever blocking work - the store still works, mutations just go undetected.
+const shouldFreeze =
+  process.env.NODE_ENV !== 'production' && config.freezeSharedData !== false
+
+/**
+ * Recursively freeze an object graph. Handles cycles and only descends into
+ * plain objects/arrays (seed data is plain JSON so that covers everything).
+ *
+ * @param {object} value - Object or array to freeze
+ * @param {WeakSet} seen - Already-frozen objects (cycle guard)
+ * @returns {object} The same value, frozen
+ */
+const deepFreeze = (value, seen = new WeakSet()) => {
+  if (value === null || typeof value !== 'object' || seen.has(value)) {
+    return value
+  }
+  seen.add(value)
+  for (const key of Object.keys(value)) {
+    deepFreeze(value[key], seen)
+  }
+  return Object.freeze(value)
+}
+
+/**
+ * Read a generated JSON file, returning a fallback on any failure
+ *
+ * @param {string} filename - File within the generated data folder
+ * @param {object} fallback - Value to use if the file is missing/invalid
+ * @returns {object} Parsed JSON or fallback
+ */
+const readGeneratedFile = (filename, fallback) => {
+  try {
+    return JSON.parse(
+      fs.readFileSync(path.join(generatedDataPath, filename), 'utf8')
+    )
+  } catch (err) {
+    console.warn(`Data store: error reading ${filename}:`, err.message)
+    return fallback
+  }
+}
+
+// The store's mutable state - swapped wholesale by reload()
+const state = {
+  participants: [],
+  clinics: [],
+  events: [],
+  participantsById: new Map(),
+  clinicsById: new Map(),
+  eventsById: new Map(),
+  eventIdsByClinic: new Map(),
+  eventIdsByParticipant: new Map(),
+  generationInfo: {},
+  // Identifies which generation of seed data the store holds. Sessions stamp
+  // their _changes with this; the attach middleware discards changes made
+  // against a different generation (i.e. before a regenerate/profile swap).
+  generationId: null
+}
+
+/**
+ * (Re)load all collections from the generated JSON on disk and rebuild
+ * indexes. Called at boot and again after any data regeneration.
+ */
+const reload = () => {
+  const started = Date.now()
+
+  const participants =
+    readGeneratedFile('participants.json', {}).participants || []
+  const clinics = readGeneratedFile('clinics.json', {}).clinics || []
+  const events = readGeneratedFile('events.json', {}).events || []
+  const generationInfo = readGeneratedFile('generation-info.json', {
+    generatedAt: 'Never',
+    stats: { participants: 0, clinics: 0, events: 0 }
+  })
+
+  if (shouldFreeze) {
+    deepFreeze(participants)
+    deepFreeze(clinics)
+    deepFreeze(events)
+    deepFreeze(generationInfo)
+  }
+
+  state.participants = participants
+  state.clinics = clinics
+  state.events = events
+  state.generationInfo = generationInfo
+  state.generationId = generationInfo.generatedAt
+
+  state.participantsById = new Map(participants.map((p) => [p.id, p]))
+  state.clinicsById = new Map(clinics.map((c) => [c.id, c]))
+  state.eventsById = new Map(events.map((e) => [e.id, e]))
+
+  state.eventIdsByClinic = new Map()
+  state.eventIdsByParticipant = new Map()
+  for (const event of events) {
+    if (!state.eventIdsByClinic.has(event.clinicId)) {
+      state.eventIdsByClinic.set(event.clinicId, [])
+    }
+    state.eventIdsByClinic.get(event.clinicId).push(event.id)
+
+    if (!state.eventIdsByParticipant.has(event.participantId)) {
+      state.eventIdsByParticipant.set(event.participantId, [])
+    }
+    state.eventIdsByParticipant.get(event.participantId).push(event.id)
+  }
+
+  console.log(
+    `Data store: loaded ${participants.length} participants, ` +
+      `${clinics.length} clinics, ${events.length} events ` +
+      `in ${Date.now() - started}ms${shouldFreeze ? ' (frozen)' : ''}`
+  )
+}
+
+// Initial load at boot. If the generated files are missing/stale the app's
+// regeneration path will run the generator and call reload() again.
+reload()
+
+module.exports = {
+  state,
+  reload,
+  deepFreeze,
+  shouldFreeze
+}

app/lib/utils/event-data.js

@@ -1,6 +1,23 @@
 // app/lib/utils/event-data.js
 const { getParticipant } = require('./participants.js')
 
+/**
+ * Record a changed event so it persists for this session
+ *
+ * The events array attached to `data` is rebuilt from the shared data store
+ * on every request; only records in data._changes survive across requests
+ * (see the attach middleware in app/routes.js). Callers should also replace
+ * the record in data.events so reads later in the same request see it.
+ *
+ * @param {object} data - Session data
+ * @param {object} event - Whole replacement event record
+ */
+const recordEventChange = (data, event) => {
+  if (data._changes?.events) {
+    data._changes.events[event.id] = event
+  }
+}
+
 /**
  * Get an event by ID
  *
@@ -50,8 +67,10 @@ const updateEvent = (data, eventId, updatedEvent) => {
   const eventIndex = data.events.findIndex((e) => e.id === eventId)
   if (eventIndex === -1) return null
 
-  // Update in the array
+  // Update in the attached array (same-request reads) and record the change
+  // (persistence across requests)
   data.events[eventIndex] = updatedEvent
+  recordEventChange(data, updatedEvent)
   return updatedEvent
 }
 
@@ -88,6 +107,7 @@ const updateEventStatus = (data, eventId, newStatus) => {
 
   // Update main data
   data.events[eventIndex] = updatedEvent
+  recordEventChange(data, updatedEvent)
 
   // Also update temp event data if it exists and matches this event
   // Only update the status-related fields to preserve other temp changes
@@ -125,6 +145,7 @@ const updateEventData = (data, eventId, updates) => {
 
   // Update main data
   data.events[eventIndex] = updatedEvent
+  recordEventChange(data, updatedEvent)
 
   // Also update temp event data if it exists and matches this event
   // Merge updates into existing temp event to preserve any unsaved changes

app/lib/utils/participants.js

@@ -250,8 +250,13 @@ const updateParticipant = (data, participantId, updatedParticipant) => {
   )
   if (participantIndex === -1) return null
 
-  // Update in the array
+  // Update in the attached array (same-request reads) and record the change
+  // in data._changes (persistence - the attached array is rebuilt from the
+  // shared data store on every request; see middleware in app/routes.js)
   data.participants[participantIndex] = updatedParticipant
+  if (data._changes?.participants) {
+    data._changes.participants[participantId] = updatedParticipant
+  }
   return updatedParticipant
 }
 

app/lib/utils/reading.js

@@ -4,6 +4,7 @@ const dayjs = require('dayjs')
 const { eligibleForReading, getStatusTagColour } = require('./status')
 const { isWithinDayRange } = require('./dates')
 const { awaitingPriors, userRequestedPriors } = require('./prior-mammograms')
+const { updateEventData } = require('./event-data')
 
 // /**
 //  * Get first unread event in a clinic
@@ -83,44 +84,45 @@ const getReadsAsArray = function (event) {
 }
 
 /**
- * Update the writeReading function to also handle removing from skipped events
+ * Save a user's reading for an event, and remove the event from the reading
+ * session's skipped list if present
+ *
+ * Builds a new imageReading object and saves it through updateEventData
+ * rather than mutating the event - event records are shared read-only data.
  *
  * @param {object} event - The event to update
  * @param {string} userId - User ID
  * @param {object} reading - Reading data to save
- * @param {object | null} [data] - Session data (needed for session context)
- * @param {string | null} [sessionId] - Session ID (if in session context)
+ * @param {object} data - Session data
+ * @param {string | null} [sessionId] - Reading session ID (if in session context)
  */
-const writeReading = (
-  event,
-  userId,
-  reading,
-  data = null,
-  sessionId = null
-) => {
-  // Ensure imageReading structure exists
-  if (!event.imageReading) {
-    event.imageReading = { reads: {} }
-  } else if (!event.imageReading.reads) {
-    event.imageReading.reads = {}
+const writeReading = (event, userId, reading, data, sessionId = null) => {
+  // Work on a clone so the shared record is never touched in place
+  const imageReading = structuredClone(event.imageReading || {})
+  if (!imageReading.reads) {
+    imageReading.reads = {}
   }
 
   // Calculate readNumber based on existing reads
-  const existingReadCount = Object.keys(event.imageReading.reads).length
+  const existingReadCount = Object.keys(imageReading.reads).length
   // If this user already has a read, keep their readNumber; otherwise assign next number
-  const existingRead = event.imageReading.reads[userId]
+  const existingRead = imageReading.reads[userId]
   const readNumber = existingRead?.readNumber || existingReadCount + 1
 
   // Add the reading with timestamp and readNumber
-  event.imageReading.reads[userId] = {
+  imageReading.reads[userId] = {
     ...reading,
     readerId: userId, // Ensure the reader ID is saved
     readNumber,
     timestamp: new Date().toISOString()
   }
 
+  // Saves to the event and mirrors into data.event if it matches
+  updateEventData(data, event.id, { imageReading })
+
   // If we have session context, remove this event from skipped events
-  if (data && sessionId && data.readingSessions?.[sessionId]) {
+  // (readingSessions is per-session working data, so in-place edits are fine)
+  if (sessionId && data.readingSessions?.[sessionId]) {
     const session = data.readingSessions[sessionId]
 
     // Remove event from skipped list if present