Harden the Redis layer for production.

- Locks no longer fail open: an unreachable backend throws
  LockUnavailableError and withLock skips the run (or opts into running
  unlocked, as template seeding does). The lnbits payment poller keeps
  polling through an outage, made safe by only sending confirmation
  emails when a row actually transitioned.
- Rate limiter INCR+PEXPIRE now runs as one Lua script, self-healing
  counters stranded without a TTL.
- Per-email login lockout moves to a Redis-backed store shared across
  replicas (in-memory fallback preserved), case-insensitive on email.
- Graceful shutdown on SIGTERM/SIGINT: stop periodic jobs, close the
  server, force-close lingering SSE sockets, close Redis.
- Active PING probe backs the health flag; /health reports last ping.
  SSE payment streams also poll the DB as a pub/sub-gap fallback.
- Scale compose gains requirepass, maxmemory 256mb with noeviction, and
  an authenticated healthcheck; .env.example documents passwords, TLS
  (rediss://), and DB-index selection.
- First tests in the repo: vitest + ioredis-mock covering the lock,
  rate limiter, and login lockout stores (27 tests).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/src/lib/stores/lock.ts

@@ -5,22 +5,45 @@
 //
 // Use acquire/release for long-lived ownership (e.g. a background poller) and
 // withLock for a one-shot critical section. Selection is based on REDIS_URL.
+//
+// There is deliberately no auto-renewal/watchdog: every guarded section (sweep
+// jobs, template seeding) is a handful of status-conditional bulk UPDATEs that
+// finish far below the lock TTL, and a rare TTL overrun only risks one
+// idempotent overlapping run. Keep TTLs generous instead of adding renewal.
 
 import { randomUUID } from 'crypto';
 import { getRedis, isRedisEnabled } from '../redis.js';
 
+// Thrown when Redis is configured but the lock backend cannot be reached.
+// This is distinct from contention (acquire resolves null): the caller must
+// decide whether its critical section is safe to run without mutual exclusion.
+export class LockUnavailableError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'LockUnavailableError';
+  }
+}
+
+export interface WithLockOptions {
+  // What withLock should do when the lock backend is unavailable (not mere
+  // contention): 'skip' (default) returns null as if the lock were held
+  // elsewhere; 'run' executes fn without mutual exclusion.
+  onUnavailable?: 'skip' | 'run';
+}
+
 export interface Lock {
   readonly backend: 'memory' | 'redis';
   // Returns a token when the lock was acquired, or null when already held.
+  // Throws LockUnavailableError when the backend is configured but erroring.
   acquire(key: string, ttlMs: number): Promise<string | null>;
   release(key: string, token: string): Promise<void>;
   // Runs fn while holding the lock; returns fn's result, or null if not acquired.
-  withLock<T>(key: string, ttlMs: number, fn: () => Promise<T>): Promise<T | null>;
+  withLock<T>(key: string, ttlMs: number, fn: () => Promise<T>, opts?: WithLockOptions): Promise<T | null>;
 }
 
 // ==================== Memory implementation ====================
 
-class MemoryLock implements Lock {
+export class MemoryLock implements Lock {
   readonly backend = 'memory' as const;
   private held = new Map<string, { token: string; expiresAt: number }>();
 
@@ -58,23 +81,23 @@ class MemoryLock implements Lock {
 const RELEASE_SCRIPT =
   'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
 
-class RedisLock implements Lock {
+export class RedisLock implements Lock {
   readonly backend = 'redis' as const;
 
   async acquire(key: string, ttlMs: number): Promise<string | null> {
     const redis = getRedis();
     if (!redis) {
-      // Redis configured but unavailable: do not block critical sections.
-      return randomUUID();
+      throw new LockUnavailableError('redis client not initialized');
     }
     const token = randomUUID();
     try {
       const result = await redis.set(`lock:${key}`, token, 'PX', ttlMs, 'NX');
       return result === 'OK' ? token : null;
     } catch (err: any) {
-      console.error('[lock] redis acquire error, proceeding without lock:', err?.message || err);
-      // Fail open so a Redis outage does not deadlock startup or jobs.
-      return randomUUID();
+      // Do NOT fabricate a token here: during an outage every replica would
+      // "acquire" every lock and run the guarded sections concurrently. Let the
+      // caller choose between skipping the run and running unlocked.
+      throw new LockUnavailableError(err?.message || String(err));
     }
   }
 
@@ -88,8 +111,19 @@ class RedisLock implements Lock {
     }
   }
 
-  async withLock<T>(key: string, ttlMs: number, fn: () => Promise<T>): Promise<T | null> {
-    const token = await this.acquire(key, ttlMs);
+  async withLock<T>(key: string, ttlMs: number, fn: () => Promise<T>, opts?: WithLockOptions): Promise<T | null> {
+    let token: string | null;
+    try {
+      token = await this.acquire(key, ttlMs);
+    } catch (err) {
+      if (!(err instanceof LockUnavailableError)) throw err;
+      if (opts?.onUnavailable === 'run') {
+        console.warn(`[lock] backend unavailable, running "${key}" WITHOUT mutual exclusion:`, err.message);
+        return fn();
+      }
+      console.warn(`[lock] backend unavailable, skipping "${key}" this run:`, err.message);
+      return null;
+    }
     if (!token) return null;
     try {
       return await fn();