From c4683ff5b0e482813514e47465e2ee3e41ad6c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=97=D0=B0=D0=B8=D0=B4=20=D0=9E=D0=BC=D0=B0=D1=80=20?= =?UTF-8?q?=D0=9C=D0=B5=D0=B4=D1=85=D0=B0=D1=82?= Date: Sun, 23 Aug 2026 05:09:38 +0500 Subject: [PATCH] Backend caching + AODP refetch cooldown - In-memory TTL cache (server/src/cache.ts) with a get/set/clear surface that a Redis backend can replace later for multi-replica setups. - Cache the expensive opportunities ranking (5 min), item detail/search (1 h), and per-item market prices (10 min). All keyed by request; each read endpoint takes a refresh/force bypass. - Invalidate precisely on writes: price upserts, overrides, profile activate/update, economy config, and game-data reparse (full clear). - AODP anti-spam: a price download now skips items priced within a configurable cooldown (default 15 min, Settings page) unless "force" is checked; the job reports how many fresh items it skipped. - No client-side caching (per request); the browser still calls the API, but repeats are served from the server cache without recompute/AODP. Co-Authored-By: Claude Fable 5 --- server/src/cache.ts | 53 ++++++++++++++++++++++++++ server/src/character/service.ts | 3 ++ server/src/gamedata/parse.ts | 2 + server/src/market/service.ts | 50 +++++++++++++++++++++++- server/src/routes/economy.ts | 22 +++++++++-- server/src/routes/items.ts | 67 +++++++++++++++++++-------------- server/src/routes/market.ts | 14 ++++++- web/src/apiCache.ts | 55 --------------------------- web/src/pages/MarketPage.tsx | 14 ++++++- web/src/pages/SettingsPage.tsx | 18 ++++++++- 10 files changed, 206 insertions(+), 92 deletions(-) create mode 100644 server/src/cache.ts delete mode 100644 web/src/apiCache.ts diff --git a/server/src/cache.ts b/server/src/cache.ts new file mode 100644 index 0000000..f11065c --- /dev/null +++ b/server/src/cache.ts @@ -0,0 +1,53 @@ +/** + * Process-local TTL cache for expensive/repeated reads (opportunities ranking, + * item lookups, market summaries). Keeps tab-switches and repeated requests cheap + * without recomputing or re-querying. + * + * Single-instance in-memory is enough for now. The surface (get/set/clear) is + * deliberately tiny so a Redis-backed implementation can replace it later if the + * app is ever scaled to multiple replicas that must share a cache — set + * REDIS_URL and swap the internals; callers don't change. + */ +interface Entry { + at: number; + value: unknown; +} + +const store = new Map(); + +export function cacheGet(key: string, ttlMs: number): T | undefined { + const e = store.get(key); + if (!e) return undefined; + if (Date.now() - e.at > ttlMs) { + store.delete(key); + return undefined; + } + return e.value as T; +} + +export function cacheSet(key: string, value: unknown): void { + store.set(key, { at: Date.now(), value }); +} + +/** Clear entries whose key starts with prefix (or everything if omitted). */ +export function cacheClear(prefix?: string): void { + if (!prefix) { + store.clear(); + return; + } + for (const k of [...store.keys()]) if (k.startsWith(prefix)) store.delete(k); +} + +/** + * Get-or-compute. Returns the cached value when fresh; otherwise runs producer, + * stores, and returns it. `force` bypasses the read (still repopulates). + */ +export async function cached(key: string, ttlMs: number, producer: () => Promise, force = false): Promise { + if (!force) { + const hit = cacheGet(key, ttlMs); + if (hit !== undefined) return hit; + } + const value = await producer(); + cacheSet(key, value); + return value; +} diff --git a/server/src/character/service.ts b/server/src/character/service.ts index 38ee0a9..1209bc8 100644 --- a/server/src/character/service.ts +++ b/server/src/character/service.ts @@ -1,4 +1,5 @@ import { get, query, transaction } from '../db/index.js'; +import { cacheClear } from '../cache.js'; /** * Character profile: the user's destiny board progress, entered manually @@ -82,6 +83,7 @@ export async function updateProfile( new Date().toISOString(), id, ]); + cacheClear('opp:'); // premium/spec levels feed every ranking return rowToProfile(await get('SELECT * FROM profiles WHERE id = ?', [id])); } @@ -90,6 +92,7 @@ export async function activateProfile(id: number): Promise { await tx.query('UPDATE profiles SET is_active = 0'); await tx.query('UPDATE profiles SET is_active = 1 WHERE id = ?', [id]); }); + cacheClear('opp:'); // a different active profile changes every ranking } export async function deleteProfile(id: number): Promise { diff --git a/server/src/gamedata/parse.ts b/server/src/gamedata/parse.ts index e29f41d..ccb86d7 100644 --- a/server/src/gamedata/parse.ts +++ b/server/src/gamedata/parse.ts @@ -1,5 +1,6 @@ import fs from 'node:fs'; import { setMeta, transaction, type Tx } from '../db/index.js'; +import { cacheClear } from '../cache.js'; import { gamedataFilePath } from './download.js'; /** @@ -245,5 +246,6 @@ export async function parseGameData(): Promise { }; await setMeta('gamedata_parsed_at', new Date().toISOString()); await setMeta('gamedata_stats', JSON.stringify(stats)); + cacheClear(); // items/recipes changed → drop every cached read return stats; } diff --git a/server/src/market/service.ts b/server/src/market/service.ts index 70ca760..060b9dc 100644 --- a/server/src/market/service.ts +++ b/server/src/market/service.ts @@ -1,4 +1,5 @@ import { get, getMeta, query, setMeta, transaction } from '../db/index.js'; +import { cacheClear } from '../cache.js'; import { AodpUnreachableError, CITIES, @@ -7,6 +8,12 @@ import { SERVER_HOSTS, } from './client.js'; +/** Prices changed → opportunities and market reads are stale. */ +function invalidatePriceCaches(): void { + cacheClear('opp:'); + cacheClear('market:'); +} + export async function getAlbionServer(): Promise { const s = await getMeta('albion_server'); return s && s in SERVER_HOSTS ? s : 'europe'; @@ -26,6 +33,7 @@ export interface RefreshJob { doneChunks: number; itemCount: number; rowsUpserted: number; + skippedFresh?: number; // items skipped because their prices were still within the cooldown error?: string; startedAt?: string; finishedAt?: string; @@ -42,6 +50,24 @@ export interface RefreshOptions { maxTier?: number; cities?: string[]; items?: string[]; // explicit list overrides tier-based selection + force?: boolean; // ignore the refetch cooldown +} + +/** Minutes an item's prices are considered fresh — within this, a refresh skips it to spare the AODP API. */ +async function priceCooldownMinutes(): Promise { + const v = Number(await getMeta('price_cooldown_min')); + return Number.isFinite(v) && v > 0 ? v : 15; +} + +/** Items whose most recent price fetch (any city) is within the cooldown window. */ +async function recentlyFetchedItems(cooldownMin: number): Promise> { + // fetched_at is stored as ISO-8601 UTC, which orders chronologically as text. + const thresholdIso = new Date(Date.now() - cooldownMin * 60_000).toISOString(); + const rows = await query<{ item: string }>( + 'SELECT item FROM market_prices GROUP BY item HAVING MAX(fetched_at) > ?', + [thresholdIso], + ); + return new Set(rows.map((r) => r.item)); } /** Items worth pricing: everything that appears in a recipe (as output or input). */ @@ -91,6 +117,7 @@ export async function upsertPriceRows(rows: unknown[][], mode: 'overwrite' | 'co await tx.query(`INSERT INTO market_prices (${PRICE_COLUMNS.join(', ')}) VALUES ${groups.join(', ')}${tail}`, params); } }); + invalidatePriceCaches(); return rows.length; } @@ -108,7 +135,7 @@ export function startRefresh(opts: RefreshOptions): { started: boolean; reason?: void (async () => { try { - const items = + let items = opts.items && opts.items.length > 0 ? opts.items : await selectRecipeRelevantItems(opts.minTier ?? 2, opts.maxTier ?? 8); @@ -118,6 +145,25 @@ export function startRefresh(opts: RefreshOptions): { started: boolean; reason?: job.finishedAt = new Date().toISOString(); return; } + + // Skip items priced recently, so repeated downloads don't spam AODP. + const requested = items.length; + let skipped = 0; + if (!opts.force) { + const cooldown = await priceCooldownMinutes(); + const fresh = await recentlyFetchedItems(cooldown); + const filtered = items.filter((i) => !fresh.has(i)); + skipped = items.length - filtered.length; + items = filtered; + } + job.skippedFresh = skipped; + if (items.length === 0) { + job.status = 'done'; + job.itemCount = requested; + job.rowsUpserted = 0; + job.finishedAt = new Date().toISOString(); + return; // everything was fresh — nothing to fetch + } job.itemCount = items.length; job.totalChunks = Math.ceil(items.length / 100); @@ -254,10 +300,12 @@ export async function setOverride(o: { updated_at = excluded.updated_at`, [o.item, o.city, o.quality ?? 1, o.sellPrice ?? null, o.buyPrice ?? null, o.note ?? null, new Date().toISOString()], ); + invalidatePriceCaches(); // manual override changes prices used everywhere } export async function deleteOverride(item: string, city: string, quality = 1): Promise { await query('DELETE FROM price_overrides WHERE item = ? AND city = ? AND quality = ?', [item, city, quality]); + invalidatePriceCaches(); } export async function marketSummary() { diff --git a/server/src/routes/economy.ts b/server/src/routes/economy.ts index c0d2509..196fd7c 100644 --- a/server/src/routes/economy.ts +++ b/server/src/routes/economy.ts @@ -1,20 +1,36 @@ import type { FastifyInstance } from 'fastify'; +import { cached, cacheClear } from '../cache.js'; import { getEconomyConfig, setEconomyConfig, type EconomyConfig } from '../economy/config.js'; import { rankOpportunities, type RankRequest } from '../economy/service.js'; +const OPP_TTL_MS = 5 * 60 * 1000; // opportunities are recomputed at most every 5 min per param-set + export async function economyRoutes(app: FastifyInstance): Promise { app.get('/api/economy/config', async () => getEconomyConfig()); app.put('/api/economy/config', async (req) => { await setEconomyConfig((req.body ?? {}) as Partial); + cacheClear('opp:'); // config affects every ranking return { ok: true, config: await getEconomyConfig() }; }); app.post('/api/economy/opportunities', async (req, reply) => { - const body = (req.body ?? {}) as RankRequest; + const body = (req.body ?? {}) as RankRequest & { refresh?: boolean }; if (!body.city) return reply.code(400).send({ error: 'city is required' }); + + const { refresh, ...params } = body; + const key = `opp:${JSON.stringify(params)}`; const started = Date.now(); - const result = await rankOpportunities(body); - return { ...result, tookMs: Date.now() - started }; + let cacheHit = true; + const result = await cached( + key, + OPP_TTL_MS, + async () => { + cacheHit = false; + return rankOpportunities(params); + }, + refresh === true, + ); + return { ...result, tookMs: Date.now() - started, cached: cacheHit }; }); } diff --git a/server/src/routes/items.ts b/server/src/routes/items.ts index 2eb9bda..de8e76e 100644 --- a/server/src/routes/items.ts +++ b/server/src/routes/items.ts @@ -1,47 +1,58 @@ import type { FastifyInstance } from 'fastify'; import { get, query } from '../db/index.js'; +import { cached } from '../cache.js'; + +// Item/recipe data only changes when game data is re-parsed (which calls +// cacheClear()), so these can cache for a long time. +const ITEM_TTL_MS = 60 * 60 * 1000; export async function itemRoutes(app: FastifyInstance): Promise { app.get('/api/items/search', async (req) => { const { q = '' } = req.query as { q?: string }; const queryStr = q.trim(); if (!queryStr) return { results: [] }; - const results = await query( - `SELECT unique_name, name_en, tier, enchantment, shop_category, shop_subcategory - FROM items - WHERE name_en ILIKE ? OR unique_name ILIKE ? - ORDER BY tier, enchantment, unique_name - LIMIT 50`, - [`%${queryStr}%`, `%${queryStr}%`], + const results = await cached(`item:search:${queryStr.toLowerCase()}`, ITEM_TTL_MS, () => + query( + `SELECT unique_name, name_en, tier, enchantment, shop_category, shop_subcategory + FROM items + WHERE name_en ILIKE ? OR unique_name ILIKE ? + ORDER BY tier, enchantment, unique_name + LIMIT 50`, + [`%${queryStr}%`, `%${queryStr}%`], + ), ); return { results }; }); app.get('/api/items/:uniqueName', async (req, reply) => { const { uniqueName } = req.params as { uniqueName: string }; - const item = await get('SELECT * FROM items WHERE unique_name = ?', [uniqueName]); - if (!item) return reply.code(404).send({ error: 'item not found' }); + const data = await cached(`item:detail:${uniqueName}`, ITEM_TTL_MS, async () => { + const item = await get('SELECT * FROM items WHERE unique_name = ?', [uniqueName]); + if (!item) return null; - const recipes = (await query('SELECT * FROM recipes WHERE output_item = ? ORDER BY source, variant', [uniqueName])) as any[]; - for (const r of recipes) { - r.inputs = await query( - `SELECT ri.input_item, ri.count, ri.max_return_amount, i.name_en - FROM recipe_inputs ri LEFT JOIN items i ON i.unique_name = ri.input_item - WHERE ri.recipe_id = ?`, - [r.id], + const recipes = (await query('SELECT * FROM recipes WHERE output_item = ? ORDER BY source, variant', [uniqueName])) as any[]; + for (const r of recipes) { + r.inputs = await query( + `SELECT ri.input_item, ri.count, ri.max_return_amount, i.name_en + FROM recipe_inputs ri LEFT JOIN items i ON i.unique_name = ri.input_item + WHERE ri.recipe_id = ?`, + [r.id], + ); + } + + const usedIn = await query( + `SELECT DISTINCT r.output_item, i.name_en + FROM recipe_inputs ri + JOIN recipes r ON r.id = ri.recipe_id + LEFT JOIN items i ON i.unique_name = r.output_item + WHERE ri.input_item = ? + LIMIT 100`, + [uniqueName], ); - } + return { item, recipes, usedIn }; + }); - const usedIn = await query( - `SELECT DISTINCT r.output_item, i.name_en - FROM recipe_inputs ri - JOIN recipes r ON r.id = ri.recipe_id - LEFT JOIN items i ON i.unique_name = r.output_item - WHERE ri.input_item = ? - LIMIT 100`, - [uniqueName], - ); - - return { item, recipes, usedIn }; + if (!data) return reply.code(404).send({ error: 'item not found' }); + return data; }); } diff --git a/server/src/routes/market.ts b/server/src/routes/market.ts index 091c6ff..43d4a99 100644 --- a/server/src/routes/market.ts +++ b/server/src/routes/market.ts @@ -1,5 +1,6 @@ import type { FastifyInstance } from 'fastify'; import { getMeta, query, setMeta } from '../db/index.js'; +import { cached } from '../cache.js'; import { CITIES, SERVER_HOSTS } from '../market/client.js'; import { deleteOverride, @@ -25,6 +26,7 @@ export async function marketRoutes(app: FastifyInstance): Promise { maxTier?: number; cities?: string[]; items?: string[]; + force?: boolean; }; const result = startRefresh(body); if (!result.started) return reply.code(409).send({ error: result.reason }); @@ -33,7 +35,10 @@ export async function marketRoutes(app: FastifyInstance): Promise { app.get('/api/market/prices/:item', async (req) => { const { item } = req.params as { item: string }; - return { prices: await getPricesForItem(item) }; + const refresh = (req.query as { refresh?: string }).refresh === 'true'; + // cache is busted whenever prices/overrides change, so a long TTL is safe + const prices = await cached(`market:prices:${item}`, 10 * 60 * 1000, () => getPricesForItem(item), refresh); + return { prices }; }); app.put('/api/market/override', async (req, reply) => { @@ -69,13 +74,15 @@ export async function marketRoutes(app: FastifyInstance): Promise { availableServers: Object.keys(SERVER_HOSTS), proxyUrl: (await getMeta('proxy_url')) ?? '', gamePath: (await getMeta('game_path')) ?? '', + priceCooldownMin: Number((await getMeta('price_cooldown_min')) ?? 15), })); app.put('/api/settings', async (req, reply) => { - const { albionServer, proxyUrl, gamePath } = (req.body ?? {}) as { + const { albionServer, proxyUrl, gamePath, priceCooldownMin } = (req.body ?? {}) as { albionServer?: string; proxyUrl?: string; gamePath?: string; + priceCooldownMin?: number; }; if (albionServer) { try { @@ -92,6 +99,9 @@ export async function marketRoutes(app: FastifyInstance): Promise { if (gamePath === '') await query("DELETE FROM meta WHERE key = 'game_path'"); else await setMeta('game_path', gamePath); } + if (priceCooldownMin !== undefined && Number.isFinite(priceCooldownMin) && priceCooldownMin > 0) { + await setMeta('price_cooldown_min', String(Math.round(priceCooldownMin))); + } return { ok: true }; }); } diff --git a/web/src/apiCache.ts b/web/src/apiCache.ts deleted file mode 100644 index 459d13a..0000000 --- a/web/src/apiCache.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Tiny in-memory client cache so switching tabs doesn't refetch data that's - * still fresh. Keyed by an arbitrary string; entries expire after a TTL. Pages - * call cachedJson() on mount (instant when warm) and pass force:true from a - * manual Refresh button to bypass and repopulate. - * - * Lives at module scope, so it survives component unmount/remount (tab switches) - * but resets on a full page reload. - */ -interface Entry { - at: number; - data: unknown; -} - -const store = new Map(); -const DEFAULT_TTL = 5 * 60 * 1000; // 5 minutes - -export interface CachedOptions { - ttl?: number; - force?: boolean; - init?: RequestInit; -} - -export async function cachedJson(key: string, url: string, opts: CachedOptions = {}): Promise { - const ttl = opts.ttl ?? DEFAULT_TTL; - const hit = store.get(key); - if (!opts.force && hit && Date.now() - hit.at < ttl) { - return hit.data as T; - } - const res = await fetch(url, opts.init); - if (!res.ok) throw new Error(`${url} → HTTP ${res.status}`); - const data = (await res.json()) as T; - store.set(key, { at: Date.now(), data }); - return data; -} - -/** Peek a cached value without fetching (for instant first paint). */ -export function peek(key: string): T | undefined { - return store.get(key)?.data as T | undefined; -} - -/** How old the cached entry is, in ms (undefined if absent). */ -export function ageMs(key: string): number | undefined { - const hit = store.get(key); - return hit ? Date.now() - hit.at : undefined; -} - -/** Drop cache entries whose key starts with prefix (or everything if omitted). */ -export function invalidate(prefix?: string): void { - if (!prefix) { - store.clear(); - return; - } - for (const k of [...store.keys()]) if (k.startsWith(prefix)) store.delete(k); -} diff --git a/web/src/pages/MarketPage.tsx b/web/src/pages/MarketPage.tsx index 8212b9f..b1accb4 100644 --- a/web/src/pages/MarketPage.tsx +++ b/web/src/pages/MarketPage.tsx @@ -6,6 +6,7 @@ interface Job { doneChunks: number; itemCount: number; rowsUpserted: number; + skippedFresh?: number; error?: string; } @@ -44,6 +45,7 @@ export default function MarketPage() { const [summary, setSummary] = useState(null); const [minTier, setMinTier] = useState(2); const [maxTier, setMaxTier] = useState(8); + const [force, setForce] = useState(false); const [q, setQ] = useState(''); const [results, setResults] = useState<{ unique_name: string; name_en: string | null }[]>([]); const [selected, setSelected] = useState<{ unique_name: string; name_en: string | null } | null>(null); @@ -103,7 +105,7 @@ export default function MarketPage() { await fetch('/api/market/refresh', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ minTier, maxTier }), + body: JSON.stringify({ minTier, maxTier, force }), }); loadSummary(); }; @@ -157,6 +159,9 @@ export default function MarketPage() { {[1, 2, 3, 4, 5, 6, 7, 8].map((t) => )} + @@ -165,7 +170,12 @@ export default function MarketPage() { {job.doneChunks}/{job.totalChunks} batches ({job.itemCount} items) )} - {job?.status === 'done' && Last run stored {job.rowsUpserted} price rows.} + {job?.status === 'done' && ( + + Last run stored {job.rowsUpserted} price rows + {job.skippedFresh ? `, skipped ${job.skippedFresh} still-fresh items` : ''}. + + )} {job?.status === 'error' && (

diff --git a/web/src/pages/SettingsPage.tsx b/web/src/pages/SettingsPage.tsx index fcb8bec..aa65a6c 100644 --- a/web/src/pages/SettingsPage.tsx +++ b/web/src/pages/SettingsPage.tsx @@ -5,12 +5,14 @@ interface Settings { availableServers: string[]; proxyUrl: string; gamePath: string; + priceCooldownMin: number; } export default function SettingsPage() { const [settings, setSettings] = useState(null); const [proxyUrl, setProxyUrl] = useState(''); const [gamePath, setGamePath] = useState(''); + const [cooldown, setCooldown] = useState(15); const [saved, setSaved] = useState(false); useEffect(() => { @@ -20,10 +22,11 @@ export default function SettingsPage() { setSettings(s); setProxyUrl(s.proxyUrl); setGamePath(s.gamePath); + setCooldown(s.priceCooldownMin); }); }, []); - const save = async (patch: Partial<{ albionServer: string; proxyUrl: string; gamePath: string }>) => { + const save = async (patch: Partial<{ albionServer: string; proxyUrl: string; gamePath: string; priceCooldownMin: number }>) => { await fetch('/api/settings', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, @@ -78,6 +81,19 @@ export default function SettingsPage() { +

+

Price refetch cooldown

+

+ A price download skips items whose prices were fetched within this many minutes, to avoid spamming the + market API. Use the "force" checkbox on the Market Data page to override it when you truly need fresh data. +

+
+ setCooldown(Number(e.target.value))} /> + minutes + +
+
); }