Backend caching + AODP refetch cooldown

- In-memory TTL cache (server/src/cache.ts) with a get/set/clear surface
  that a Redis backend can replace later for multi-replica setups.
- Cache the expensive opportunities ranking (5 min), item detail/search
  (1 h), and per-item market prices (10 min). All keyed by request; each
  read endpoint takes a refresh/force bypass.
- Invalidate precisely on writes: price upserts, overrides, profile
  activate/update, economy config, and game-data reparse (full clear).
- AODP anti-spam: a price download now skips items priced within a
  configurable cooldown (default 15 min, Settings page) unless "force"
  is checked; the job reports how many fresh items it skipped.
- No client-side caching (per request); the browser still calls the API,
  but repeats are served from the server cache without recompute/AODP.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

server/src/cache.ts

@@ -0,0 +1,53 @@
+/**
+ * Process-local TTL cache for expensive/repeated reads (opportunities ranking,
+ * item lookups, market summaries). Keeps tab-switches and repeated requests cheap
+ * without recomputing or re-querying.
+ *
+ * Single-instance in-memory is enough for now. The surface (get/set/clear) is
+ * deliberately tiny so a Redis-backed implementation can replace it later if the
+ * app is ever scaled to multiple replicas that must share a cache — set
+ * REDIS_URL and swap the internals; callers don't change.
+ */
+interface Entry {
+  at: number;
+  value: unknown;
+}
+
+const store = new Map<string, Entry>();
+
+export function cacheGet<T>(key: string, ttlMs: number): T | undefined {
+  const e = store.get(key);
+  if (!e) return undefined;
+  if (Date.now() - e.at > ttlMs) {
+    store.delete(key);
+    return undefined;
+  }
+  return e.value as T;
+}
+
+export function cacheSet(key: string, value: unknown): void {
+  store.set(key, { at: Date.now(), value });
+}
+
+/** Clear entries whose key starts with prefix (or everything if omitted). */
+export function cacheClear(prefix?: string): void {
+  if (!prefix) {
+    store.clear();
+    return;
+  }
+  for (const k of [...store.keys()]) if (k.startsWith(prefix)) store.delete(k);
+}
+
+/**
+ * Get-or-compute. Returns the cached value when fresh; otherwise runs producer,
+ * stores, and returns it. `force` bypasses the read (still repopulates).
+ */
+export async function cached<T>(key: string, ttlMs: number, producer: () => Promise<T>, force = false): Promise<T> {
+  if (!force) {
+    const hit = cacheGet<T>(key, ttlMs);
+    if (hit !== undefined) return hit;
+  }
+  const value = await producer();
+  cacheSet(key, value);
+  return value;
+}

server/src/character/service.ts

@@ -1,4 +1,5 @@
 import { get, query, transaction } from '../db/index.js';
+import { cacheClear } from '../cache.js';
 
 /**
  * Character profile: the user's destiny board progress, entered manually
@@ -82,6 +83,7 @@ export async function updateProfile(
     new Date().toISOString(),
     id,
   ]);
+  cacheClear('opp:'); // premium/spec levels feed every ranking
   return rowToProfile(await get('SELECT * FROM profiles WHERE id = ?', [id]));
 }
 
@@ -90,6 +92,7 @@ export async function activateProfile(id: number): Promise<void> {
     await tx.query('UPDATE profiles SET is_active = 0');
     await tx.query('UPDATE profiles SET is_active = 1 WHERE id = ?', [id]);
   });
+  cacheClear('opp:'); // a different active profile changes every ranking
 }
 
 export async function deleteProfile(id: number): Promise<void> {

server/src/gamedata/parse.ts

@@ -1,5 +1,6 @@
 import fs from 'node:fs';
 import { setMeta, transaction, type Tx } from '../db/index.js';
+import { cacheClear } from '../cache.js';
 import { gamedataFilePath } from './download.js';
 
 /**
@@ -245,5 +246,6 @@ export async function parseGameData(): Promise<ParseStats> {
   };
   await setMeta('gamedata_parsed_at', new Date().toISOString());
   await setMeta('gamedata_stats', JSON.stringify(stats));
+  cacheClear(); // items/recipes changed → drop every cached read
   return stats;
 }

server/src/market/service.ts

@@ -1,4 +1,5 @@
 import { get, getMeta, query, setMeta, transaction } from '../db/index.js';
+import { cacheClear } from '../cache.js';
 import {
   AodpUnreachableError,
   CITIES,
@@ -7,6 +8,12 @@ import {
   SERVER_HOSTS,
 } from './client.js';
 
+/** Prices changed → opportunities and market reads are stale. */
+function invalidatePriceCaches(): void {
+  cacheClear('opp:');
+  cacheClear('market:');
+}
+
 export async function getAlbionServer(): Promise<string> {
   const s = await getMeta('albion_server');
   return s && s in SERVER_HOSTS ? s : 'europe';
@@ -26,6 +33,7 @@ export interface RefreshJob {
   doneChunks: number;
   itemCount: number;
   rowsUpserted: number;
+  skippedFresh?: number; // items skipped because their prices were still within the cooldown
   error?: string;
   startedAt?: string;
   finishedAt?: string;
@@ -42,6 +50,24 @@ export interface RefreshOptions {
   maxTier?: number;
   cities?: string[];
   items?: string[]; // explicit list overrides tier-based selection
+  force?: boolean; // ignore the refetch cooldown
+}
+
+/** Minutes an item's prices are considered fresh — within this, a refresh skips it to spare the AODP API. */
+async function priceCooldownMinutes(): Promise<number> {
+  const v = Number(await getMeta('price_cooldown_min'));
+  return Number.isFinite(v) && v > 0 ? v : 15;
+}
+
+/** Items whose most recent price fetch (any city) is within the cooldown window. */
+async function recentlyFetchedItems(cooldownMin: number): Promise<Set<string>> {
+  // fetched_at is stored as ISO-8601 UTC, which orders chronologically as text.
+  const thresholdIso = new Date(Date.now() - cooldownMin * 60_000).toISOString();
+  const rows = await query<{ item: string }>(
+    'SELECT item FROM market_prices GROUP BY item HAVING MAX(fetched_at) > ?',
+    [thresholdIso],
+  );
+  return new Set(rows.map((r) => r.item));
 }
 
 /** Items worth pricing: everything that appears in a recipe (as output or input). */
@@ -91,6 +117,7 @@ export async function upsertPriceRows(rows: unknown[][], mode: 'overwrite' | 'co
       await tx.query(`INSERT INTO market_prices (${PRICE_COLUMNS.join(', ')}) VALUES ${groups.join(', ')}${tail}`, params);
     }
   });
+  invalidatePriceCaches();
   return rows.length;
 }
 
@@ -108,7 +135,7 @@ export function startRefresh(opts: RefreshOptions): { started: boolean; reason?:
 
   void (async () => {
     try {
-      const items =
+      let items =
         opts.items && opts.items.length > 0
           ? opts.items
           : await selectRecipeRelevantItems(opts.minTier ?? 2, opts.maxTier ?? 8);
@@ -118,6 +145,25 @@ export function startRefresh(opts: RefreshOptions): { started: boolean; reason?:
         job.finishedAt = new Date().toISOString();
         return;
       }
+
+      // Skip items priced recently, so repeated downloads don't spam AODP.
+      const requested = items.length;
+      let skipped = 0;
+      if (!opts.force) {
+        const cooldown = await priceCooldownMinutes();
+        const fresh = await recentlyFetchedItems(cooldown);
+        const filtered = items.filter((i) => !fresh.has(i));
+        skipped = items.length - filtered.length;
+        items = filtered;
+      }
+      job.skippedFresh = skipped;
+      if (items.length === 0) {
+        job.status = 'done';
+        job.itemCount = requested;
+        job.rowsUpserted = 0;
+        job.finishedAt = new Date().toISOString();
+        return; // everything was fresh — nothing to fetch
+      }
       job.itemCount = items.length;
       job.totalChunks = Math.ceil(items.length / 100);
 
@@ -254,10 +300,12 @@ export async function setOverride(o: {
        updated_at = excluded.updated_at`,
     [o.item, o.city, o.quality ?? 1, o.sellPrice ?? null, o.buyPrice ?? null, o.note ?? null, new Date().toISOString()],
   );
+  invalidatePriceCaches(); // manual override changes prices used everywhere
 }
 
 export async function deleteOverride(item: string, city: string, quality = 1): Promise<void> {
   await query('DELETE FROM price_overrides WHERE item = ? AND city = ? AND quality = ?', [item, city, quality]);
+  invalidatePriceCaches();
 }
 
 export async function marketSummary() {

server/src/routes/economy.ts

@@ -1,20 +1,36 @@
 import type { FastifyInstance } from 'fastify';
+import { cached, cacheClear } from '../cache.js';
 import { getEconomyConfig, setEconomyConfig, type EconomyConfig } from '../economy/config.js';
 import { rankOpportunities, type RankRequest } from '../economy/service.js';
 
+const OPP_TTL_MS = 5 * 60 * 1000; // opportunities are recomputed at most every 5 min per param-set
+
 export async function economyRoutes(app: FastifyInstance): Promise<void> {
   app.get('/api/economy/config', async () => getEconomyConfig());
 
   app.put('/api/economy/config', async (req) => {
     await setEconomyConfig((req.body ?? {}) as Partial<EconomyConfig>);
+    cacheClear('opp:'); // config affects every ranking
     return { ok: true, config: await getEconomyConfig() };
   });
 
   app.post('/api/economy/opportunities', async (req, reply) => {
-    const body = (req.body ?? {}) as RankRequest;
+    const body = (req.body ?? {}) as RankRequest & { refresh?: boolean };
     if (!body.city) return reply.code(400).send({ error: 'city is required' });
+
+    const { refresh, ...params } = body;
+    const key = `opp:${JSON.stringify(params)}`;
     const started = Date.now();
-    const result = await rankOpportunities(body);
+    let cacheHit = true;
-    return { ...result, tookMs: Date.now() - started };
+    const result = await cached(
+      key,
+      OPP_TTL_MS,
+      async () => {
+        cacheHit = false;
+        return rankOpportunities(params);
+      },
+      refresh === true,
+    );
+    return { ...result, tookMs: Date.now() - started, cached: cacheHit };
   });
 }

server/src/routes/items.ts

@@ -1,47 +1,58 @@
 import type { FastifyInstance } from 'fastify';
 import { get, query } from '../db/index.js';
+import { cached } from '../cache.js';
+
+// Item/recipe data only changes when game data is re-parsed (which calls
+// cacheClear()), so these can cache for a long time.
+const ITEM_TTL_MS = 60 * 60 * 1000;
 
 export async function itemRoutes(app: FastifyInstance): Promise<void> {
   app.get('/api/items/search', async (req) => {
     const { q = '' } = req.query as { q?: string };
     const queryStr = q.trim();
     if (!queryStr) return { results: [] };
-    const results = await query(
+    const results = await cached(`item:search:${queryStr.toLowerCase()}`, ITEM_TTL_MS, () =>
-      `SELECT unique_name, name_en, tier, enchantment, shop_category, shop_subcategory
+      query(
-       FROM items
+        `SELECT unique_name, name_en, tier, enchantment, shop_category, shop_subcategory
-       WHERE name_en ILIKE ? OR unique_name ILIKE ?
+         FROM items
-       ORDER BY tier, enchantment, unique_name
+         WHERE name_en ILIKE ? OR unique_name ILIKE ?
-       LIMIT 50`,
+         ORDER BY tier, enchantment, unique_name
-      [`%${queryStr}%`, `%${queryStr}%`],
+         LIMIT 50`,
+        [`%${queryStr}%`, `%${queryStr}%`],
+      ),
     );
     return { results };
   });
 
   app.get('/api/items/:uniqueName', async (req, reply) => {
     const { uniqueName } = req.params as { uniqueName: string };
-    const item = await get('SELECT * FROM items WHERE unique_name = ?', [uniqueName]);
+    const data = await cached(`item:detail:${uniqueName}`, ITEM_TTL_MS, async () => {
-    if (!item) return reply.code(404).send({ error: 'item not found' });
+      const item = await get('SELECT * FROM items WHERE unique_name = ?', [uniqueName]);
+      if (!item) return null;
 
-    const recipes = (await query('SELECT * FROM recipes WHERE output_item = ? ORDER BY source, variant', [uniqueName])) as any[];
+      const recipes = (await query('SELECT * FROM recipes WHERE output_item = ? ORDER BY source, variant', [uniqueName])) as any[];
-    for (const r of recipes) {
+      for (const r of recipes) {
-      r.inputs = await query(
+        r.inputs = await query(
-        `SELECT ri.input_item, ri.count, ri.max_return_amount, i.name_en
+          `SELECT ri.input_item, ri.count, ri.max_return_amount, i.name_en
-         FROM recipe_inputs ri LEFT JOIN items i ON i.unique_name = ri.input_item
+           FROM recipe_inputs ri LEFT JOIN items i ON i.unique_name = ri.input_item
-         WHERE ri.recipe_id = ?`,
+           WHERE ri.recipe_id = ?`,
-        [r.id],
+          [r.id],
+        );
+      }
+
+      const usedIn = await query(
+        `SELECT DISTINCT r.output_item, i.name_en
+         FROM recipe_inputs ri
+         JOIN recipes r ON r.id = ri.recipe_id
+         LEFT JOIN items i ON i.unique_name = r.output_item
+         WHERE ri.input_item = ?
+         LIMIT 100`,
+        [uniqueName],
       );
-    }
+      return { item, recipes, usedIn };
+    });
 
-    const usedIn = await query(
+    if (!data) return reply.code(404).send({ error: 'item not found' });
-      `SELECT DISTINCT r.output_item, i.name_en
+    return data;
-       FROM recipe_inputs ri
-       JOIN recipes r ON r.id = ri.recipe_id
-       LEFT JOIN items i ON i.unique_name = r.output_item
-       WHERE ri.input_item = ?
-       LIMIT 100`,
-      [uniqueName],
-    );
-
-    return { item, recipes, usedIn };
   });
 }

server/src/routes/market.ts

@@ -1,5 +1,6 @@
 import type { FastifyInstance } from 'fastify';
 import { getMeta, query, setMeta } from '../db/index.js';
+import { cached } from '../cache.js';
 import { CITIES, SERVER_HOSTS } from '../market/client.js';
 import {
   deleteOverride,
@@ -25,6 +26,7 @@ export async function marketRoutes(app: FastifyInstance): Promise<void> {
       maxTier?: number;
       cities?: string[];
       items?: string[];
+      force?: boolean;
     };
     const result = startRefresh(body);
     if (!result.started) return reply.code(409).send({ error: result.reason });
@@ -33,7 +35,10 @@ export async function marketRoutes(app: FastifyInstance): Promise<void> {
 
   app.get('/api/market/prices/:item', async (req) => {
     const { item } = req.params as { item: string };
-    return { prices: await getPricesForItem(item) };
+    const refresh = (req.query as { refresh?: string }).refresh === 'true';
+    // cache is busted whenever prices/overrides change, so a long TTL is safe
+    const prices = await cached(`market:prices:${item}`, 10 * 60 * 1000, () => getPricesForItem(item), refresh);
+    return { prices };
   });
 
   app.put('/api/market/override', async (req, reply) => {
@@ -69,13 +74,15 @@ export async function marketRoutes(app: FastifyInstance): Promise<void> {
     availableServers: Object.keys(SERVER_HOSTS),
     proxyUrl: (await getMeta('proxy_url')) ?? '',
     gamePath: (await getMeta('game_path')) ?? '',
+    priceCooldownMin: Number((await getMeta('price_cooldown_min')) ?? 15),
   }));
 
   app.put('/api/settings', async (req, reply) => {
-    const { albionServer, proxyUrl, gamePath } = (req.body ?? {}) as {
+    const { albionServer, proxyUrl, gamePath, priceCooldownMin } = (req.body ?? {}) as {
       albionServer?: string;
       proxyUrl?: string;
       gamePath?: string;
+      priceCooldownMin?: number;
     };
     if (albionServer) {
       try {
@@ -92,6 +99,9 @@ export async function marketRoutes(app: FastifyInstance): Promise<void> {
       if (gamePath === '') await query("DELETE FROM meta WHERE key = 'game_path'");
       else await setMeta('game_path', gamePath);
     }
+    if (priceCooldownMin !== undefined && Number.isFinite(priceCooldownMin) && priceCooldownMin > 0) {
+      await setMeta('price_cooldown_min', String(Math.round(priceCooldownMin)));
+    }
     return { ok: true };
   });
 }

web/src/apiCache.ts

@@ -1,55 +0,0 @@
-/**
- * Tiny in-memory client cache so switching tabs doesn't refetch data that's
- * still fresh. Keyed by an arbitrary string; entries expire after a TTL. Pages
- * call cachedJson() on mount (instant when warm) and pass force:true from a
- * manual Refresh button to bypass and repopulate.
- *
- * Lives at module scope, so it survives component unmount/remount (tab switches)
- * but resets on a full page reload.
- */
-interface Entry {
-  at: number;
-  data: unknown;
-}
-
-const store = new Map<string, Entry>();
-const DEFAULT_TTL = 5 * 60 * 1000; // 5 minutes
-
-export interface CachedOptions {
-  ttl?: number;
-  force?: boolean;
-  init?: RequestInit;
-}
-
-export async function cachedJson<T = any>(key: string, url: string, opts: CachedOptions = {}): Promise<T> {
-  const ttl = opts.ttl ?? DEFAULT_TTL;
-  const hit = store.get(key);
-  if (!opts.force && hit && Date.now() - hit.at < ttl) {
-    return hit.data as T;
-  }
-  const res = await fetch(url, opts.init);
-  if (!res.ok) throw new Error(`${url} → HTTP ${res.status}`);
-  const data = (await res.json()) as T;
-  store.set(key, { at: Date.now(), data });
-  return data;
-}
-
-/** Peek a cached value without fetching (for instant first paint). */
-export function peek<T = any>(key: string): T | undefined {
-  return store.get(key)?.data as T | undefined;
-}
-
-/** How old the cached entry is, in ms (undefined if absent). */
-export function ageMs(key: string): number | undefined {
-  const hit = store.get(key);
-  return hit ? Date.now() - hit.at : undefined;
-}
-
-/** Drop cache entries whose key starts with prefix (or everything if omitted). */
-export function invalidate(prefix?: string): void {
-  if (!prefix) {
-    store.clear();
-    return;
-  }
-  for (const k of [...store.keys()]) if (k.startsWith(prefix)) store.delete(k);
-}

web/src/pages/MarketPage.tsx

@@ -6,6 +6,7 @@ interface Job {
   doneChunks: number;
   itemCount: number;
   rowsUpserted: number;
+  skippedFresh?: number;
   error?: string;
 }
 
@@ -44,6 +45,7 @@ export default function MarketPage() {
   const [summary, setSummary] = useState<Summary | null>(null);
   const [minTier, setMinTier] = useState(2);
   const [maxTier, setMaxTier] = useState(8);
+  const [force, setForce] = useState(false);
   const [q, setQ] = useState('');
   const [results, setResults] = useState<{ unique_name: string; name_en: string | null }[]>([]);
   const [selected, setSelected] = useState<{ unique_name: string; name_en: string | null } | null>(null);
@@ -103,7 +105,7 @@ export default function MarketPage() {
     await fetch('/api/market/refresh', {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ minTier, maxTier }),
+      body: JSON.stringify({ minTier, maxTier, force }),
     });
     loadSummary();
   };
@@ -157,6 +159,9 @@ export default function MarketPage() {
               {[1, 2, 3, 4, 5, 6, 7, 8].map((t) => <option key={t} value={t}>{t}</option>)}
             </select>
           </label>
+          <label title="Ignore the refetch cooldown and re-download everything">
+            <input type="checkbox" checked={force} onChange={(e) => setForce(e.target.checked)} /> force
+          </label>
           <button className="primary" disabled={job?.status === 'running'} onClick={startRefresh}>
             {job?.status === 'running' ? 'Refreshing…' : 'Download prices'}
           </button>
@@ -165,7 +170,12 @@ export default function MarketPage() {
               {job.doneChunks}/{job.totalChunks} batches ({job.itemCount} items)
             </span>
           )}
-          {job?.status === 'done' && <span className="muted">Last run stored {job.rowsUpserted} price rows.</span>}
+          {job?.status === 'done' && (
+            <span className="muted">
+              Last run stored {job.rowsUpserted} price rows
+              {job.skippedFresh ? `, skipped ${job.skippedFresh} still-fresh items` : ''}.
+            </span>
+          )}
         </div>
         {job?.status === 'error' && (
           <p style={{ color: '#e08869' }}>

web/src/pages/SettingsPage.tsx

@@ -5,12 +5,14 @@ interface Settings {
   availableServers: string[];
   proxyUrl: string;
   gamePath: string;
+  priceCooldownMin: number;
 }
 
 export default function SettingsPage() {
   const [settings, setSettings] = useState<Settings | null>(null);
   const [proxyUrl, setProxyUrl] = useState('');
   const [gamePath, setGamePath] = useState('');
+  const [cooldown, setCooldown] = useState(15);
   const [saved, setSaved] = useState(false);
 
   useEffect(() => {
@@ -20,10 +22,11 @@ export default function SettingsPage() {
         setSettings(s);
         setProxyUrl(s.proxyUrl);
         setGamePath(s.gamePath);
+        setCooldown(s.priceCooldownMin);
       });
   }, []);
 
-  const save = async (patch: Partial<{ albionServer: string; proxyUrl: string; gamePath: string }>) => {
+  const save = async (patch: Partial<{ albionServer: string; proxyUrl: string; gamePath: string; priceCooldownMin: number }>) => {
     await fetch('/api/settings', {
       method: 'PUT',
       headers: { 'Content-Type': 'application/json' },
@@ -78,6 +81,19 @@ export default function SettingsPage() {
           <button className="primary" onClick={() => save({ gamePath })}>Save</button>
         </div>
       </div>
+      <div className="card">
+        <h3>Price refetch cooldown</h3>
+        <p className="muted">
+          A price download skips items whose prices were fetched within this many minutes, to avoid spamming the
+          market API. Use the "force" checkbox on the Market Data page to override it when you truly need fresh data.
+        </p>
+        <div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
+          <input type="number" min={1} value={cooldown} style={{ width: 90 }}
+            onChange={(e) => setCooldown(Number(e.target.value))} />
+          <span className="muted">minutes</span>
+          <button className="primary" onClick={() => save({ priceCooldownMin: cooldown })}>Save</button>
+        </div>
+      </div>
     </div>
   );
 }