Standardize all timestamps to UTC ISO-8601 format

- Update schema.sql defaults from datetime('now') to strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
- Update schema-admin.sql with same UTC format
- Update AdminService.js inline table creation and query comparisons
- Update system.js time-based query comparisons
- Add documentation to time.js explaining UTC format convention
This commit is contained in:
Michaël
2025-12-21 19:10:07 -03:00
parent 0db5dec5ec
commit c2a7267459
8 changed files with 133 additions and 75 deletions

View File

@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS admin_audit_log (
notes TEXT, notes TEXT,
ip_address TEXT, ip_address TEXT,
user_agent TEXT, user_agent TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')) created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
CREATE INDEX IF NOT EXISTS idx_audit_log_admin ON admin_audit_log(admin_id); CREATE INDEX IF NOT EXISTS idx_audit_log_admin ON admin_audit_log(admin_id);
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS mint_merges (
target_mint_id TEXT NOT NULL, target_mint_id TEXT NOT NULL,
reason TEXT, reason TEXT,
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'reverted')), status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'reverted')),
merged_at TEXT NOT NULL DEFAULT (datetime('now')), merged_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
reverted_at TEXT, reverted_at TEXT,
admin_id TEXT NOT NULL, admin_id TEXT NOT NULL,
-- Store affected data for potential reversal -- Store affected data for potential reversal

View File

@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS mints (
last_success_at TEXT, last_success_at TEXT,
last_failure_at TEXT, last_failure_at TEXT,
consecutive_failures INTEGER DEFAULT 0, consecutive_failures INTEGER DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now')), created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now')) updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
CREATE INDEX IF NOT EXISTS idx_mints_status ON mints(status); CREATE INDEX IF NOT EXISTS idx_mints_status ON mints(status);
@@ -34,8 +34,8 @@ CREATE TABLE IF NOT EXISTS mint_urls (
url_normalized TEXT NOT NULL, url_normalized TEXT NOT NULL,
type TEXT NOT NULL DEFAULT 'clearnet' CHECK (type IN ('clearnet', 'tor', 'mirror')), type TEXT NOT NULL DEFAULT 'clearnet' CHECK (type IN ('clearnet', 'tor', 'mirror')),
active INTEGER NOT NULL DEFAULT 1, active INTEGER NOT NULL DEFAULT 1,
discovered_at TEXT NOT NULL DEFAULT (datetime('now')), discovered_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
last_seen_at TEXT NOT NULL DEFAULT (datetime('now')) last_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
CREATE INDEX IF NOT EXISTS idx_mint_urls_mint_id ON mint_urls(mint_id); CREATE INDEX IF NOT EXISTS idx_mint_urls_mint_id ON mint_urls(mint_id);
@@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS probes (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
mint_id TEXT NOT NULL REFERENCES mints(mint_id) ON DELETE CASCADE, mint_id TEXT NOT NULL REFERENCES mints(mint_id) ON DELETE CASCADE,
url TEXT NOT NULL, url TEXT NOT NULL,
probed_at TEXT NOT NULL DEFAULT (datetime('now')), probed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
success INTEGER NOT NULL, success INTEGER NOT NULL,
status_code INTEGER, status_code INTEGER,
rtt_ms INTEGER, rtt_ms INTEGER,
@@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS uptime_rollups (
total_checks INTEGER NOT NULL DEFAULT 0, total_checks INTEGER NOT NULL DEFAULT 0,
ok_checks INTEGER NOT NULL DEFAULT 0, ok_checks INTEGER NOT NULL DEFAULT 0,
incident_count INTEGER NOT NULL DEFAULT 0, incident_count INTEGER NOT NULL DEFAULT 0,
computed_at TEXT NOT NULL DEFAULT (datetime('now')), computed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
UNIQUE(mint_id, window, period_start) UNIQUE(mint_id, window, period_start)
); );
@@ -117,14 +117,14 @@ CREATE TABLE IF NOT EXISTS metadata_snapshots (
server_time TEXT, server_time TEXT,
raw_json TEXT, -- Full raw response raw_json TEXT, -- Full raw response
content_hash TEXT, content_hash TEXT,
last_fetched_at TEXT NOT NULL DEFAULT (datetime('now')) last_fetched_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
-- Metadata History: All changes (append-only) -- Metadata History: All changes (append-only)
CREATE TABLE IF NOT EXISTS metadata_history ( CREATE TABLE IF NOT EXISTS metadata_history (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
mint_id TEXT NOT NULL REFERENCES mints(mint_id) ON DELETE CASCADE, mint_id TEXT NOT NULL REFERENCES mints(mint_id) ON DELETE CASCADE,
fetched_at TEXT NOT NULL DEFAULT (datetime('now')), fetched_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
change_type TEXT NOT NULL CHECK (change_type IN ('initial', 'update', 'error')), change_type TEXT NOT NULL CHECK (change_type IN ('initial', 'update', 'error')),
diff TEXT, -- JSON diff of changes diff TEXT, -- JSON diff of changes
content_hash TEXT, content_hash TEXT,
@@ -150,7 +150,7 @@ CREATE TABLE IF NOT EXISTS nostr_events (
tags TEXT, -- JSON array tags TEXT, -- JSON array
sig TEXT NOT NULL, sig TEXT NOT NULL,
raw_json TEXT NOT NULL, raw_json TEXT NOT NULL,
ingested_at TEXT NOT NULL DEFAULT (datetime('now')) ingested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
CREATE INDEX IF NOT EXISTS idx_nostr_events_event_id ON nostr_events(event_id); CREATE INDEX IF NOT EXISTS idx_nostr_events_event_id ON nostr_events(event_id);
@@ -168,7 +168,7 @@ CREATE TABLE IF NOT EXISTS reviews (
created_at INTEGER NOT NULL, created_at INTEGER NOT NULL,
rating INTEGER CHECK (rating >= 1 AND rating <= 5), rating INTEGER CHECK (rating >= 1 AND rating <= 5),
content TEXT, content TEXT,
parsed_at TEXT NOT NULL DEFAULT (datetime('now')) parsed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
CREATE INDEX IF NOT EXISTS idx_reviews_mint_id ON reviews(mint_id); CREATE INDEX IF NOT EXISTS idx_reviews_mint_id ON reviews(mint_id);
@@ -185,7 +185,7 @@ CREATE TABLE IF NOT EXISTS trust_scores (
score_total INTEGER NOT NULL CHECK (score_total >= 0 AND score_total <= 100), score_total INTEGER NOT NULL CHECK (score_total >= 0 AND score_total <= 100),
score_level TEXT NOT NULL CHECK (score_level IN ('unknown', 'low', 'medium', 'high', 'excellent')), score_level TEXT NOT NULL CHECK (score_level IN ('unknown', 'low', 'medium', 'high', 'excellent')),
breakdown TEXT NOT NULL, -- JSON breakdown of score components breakdown TEXT NOT NULL, -- JSON breakdown of score components
computed_at TEXT NOT NULL DEFAULT (datetime('now')), computed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
UNIQUE(mint_id, computed_at) UNIQUE(mint_id, computed_at)
); );
@@ -210,7 +210,7 @@ CREATE TABLE IF NOT EXISTS pageviews (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
mint_id TEXT NOT NULL REFERENCES mints(mint_id) ON DELETE CASCADE, mint_id TEXT NOT NULL REFERENCES mints(mint_id) ON DELETE CASCADE,
session_id TEXT NOT NULL, session_id TEXT NOT NULL,
viewed_at TEXT NOT NULL DEFAULT (datetime('now')), viewed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
user_agent TEXT, user_agent TEXT,
referer TEXT referer TEXT
); );
@@ -228,7 +228,7 @@ CREATE TABLE IF NOT EXISTS pageview_rollups (
period_end TEXT NOT NULL, period_end TEXT NOT NULL,
view_count INTEGER NOT NULL DEFAULT 0, view_count INTEGER NOT NULL DEFAULT 0,
unique_sessions INTEGER NOT NULL DEFAULT 0, unique_sessions INTEGER NOT NULL DEFAULT 0,
computed_at TEXT NOT NULL DEFAULT (datetime('now')), computed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
UNIQUE(mint_id, window, period_start) UNIQUE(mint_id, window, period_start)
); );
@@ -244,13 +244,13 @@ CREATE TABLE IF NOT EXISTS jobs (
payload TEXT, -- JSON payload TEXT, -- JSON
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'completed', 'failed')), status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'completed', 'failed')),
priority INTEGER NOT NULL DEFAULT 0, priority INTEGER NOT NULL DEFAULT 0,
run_at TEXT NOT NULL DEFAULT (datetime('now')), run_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
started_at TEXT, started_at TEXT,
completed_at TEXT, completed_at TEXT,
retries INTEGER NOT NULL DEFAULT 0, retries INTEGER NOT NULL DEFAULT 0,
max_retries INTEGER NOT NULL DEFAULT 3, max_retries INTEGER NOT NULL DEFAULT 3,
error_message TEXT, error_message TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')) created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
CREATE INDEX IF NOT EXISTS idx_jobs_status_run_at ON jobs(status, run_at); CREATE INDEX IF NOT EXISTS idx_jobs_status_run_at ON jobs(status, run_at);
@@ -264,7 +264,7 @@ CREATE TABLE IF NOT EXISTS system_stats (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
stat_name TEXT NOT NULL, stat_name TEXT NOT NULL,
stat_value TEXT NOT NULL, stat_value TEXT NOT NULL,
recorded_at TEXT NOT NULL DEFAULT (datetime('now')) recorded_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
); );
CREATE INDEX IF NOT EXISTS idx_system_stats_name ON system_stats(stat_name); CREATE INDEX IF NOT EXISTS idx_system_stats_name ON system_stats(stat_name);

View File

@@ -75,7 +75,7 @@ router.get('/stats', async(req, res) => {
// Recent activity // Recent activity
const probeResult = queryOne(` const probeResult = queryOne(`
SELECT COUNT(*) as count FROM probes SELECT COUNT(*) as count FROM probes
WHERE probed_at >= datetime('now', '-24 hours') WHERE probed_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-24 hours')
`); `);
const probeCount = probeResult ? probeResult.count : 0; const probeCount = probeResult ? probeResult.count : 0;
@@ -86,7 +86,7 @@ router.get('/stats', async(req, res) => {
const incidentResult = queryOne(` const incidentResult = queryOne(`
SELECT COUNT(*) as count FROM incidents SELECT COUNT(*) as count FROM incidents
WHERE started_at >= datetime('now', '-7 days') WHERE started_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-7 days')
`); `);
const incidentCount = incidentResult ? incidentResult.count : 0; const incidentCount = incidentResult ? incidentResult.count : 0;
@@ -147,7 +147,7 @@ router.get('/stats/timeline', (req, res) => {
COUNT(*) as total, COUNT(*) as total,
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful
FROM probes FROM probes
WHERE probed_at >= datetime('now', '-${numDays} days') WHERE probed_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-${numDays} days')
GROUP BY date(probed_at) GROUP BY date(probed_at)
ORDER BY date ASC ORDER BY date ASC
`); `);
@@ -158,7 +158,7 @@ router.get('/stats/timeline', (req, res) => {
date(started_at) as date, date(started_at) as date,
COUNT(*) as count COUNT(*) as count
FROM incidents FROM incidents
WHERE started_at >= datetime('now', '-${numDays} days') WHERE started_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-${numDays} days')
GROUP BY date(started_at) GROUP BY date(started_at)
ORDER BY date ASC ORDER BY date ASC
`); `);

View File

@@ -36,7 +36,7 @@ export function initAdminSchema() {
notes TEXT, notes TEXT,
ip_address TEXT, ip_address TEXT,
user_agent TEXT, user_agent TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')) created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
) )
`); `);
@@ -49,7 +49,7 @@ export function initAdminSchema() {
target_mint_id TEXT NOT NULL, target_mint_id TEXT NOT NULL,
reason TEXT, reason TEXT,
status TEXT NOT NULL DEFAULT 'active', status TEXT NOT NULL DEFAULT 'active',
merged_at TEXT NOT NULL DEFAULT (datetime('now')), merged_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
reverted_at TEXT, reverted_at TEXT,
admin_id TEXT NOT NULL, admin_id TEXT NOT NULL,
affected_urls TEXT, affected_urls TEXT,
@@ -701,12 +701,12 @@ export function getSystemMetrics() {
const probesLastMinute = getCount(queryOne(` const probesLastMinute = getCount(queryOne(`
SELECT COUNT(*) as count FROM probes SELECT COUNT(*) as count FROM probes
WHERE probed_at >= datetime('now', '-1 minute') WHERE probed_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-1 minute')
`)); `));
const failedProbesLastMinute = getCount(queryOne(` const failedProbesLastMinute = getCount(queryOne(`
SELECT COUNT(*) as count FROM probes SELECT COUNT(*) as count FROM probes
WHERE probed_at >= datetime('now', '-1 minute') AND success = 0 WHERE probed_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-1 minute') AND success = 0
`)); `));
const jobBacklog = getCount(queryOne(` const jobBacklog = getCount(queryOne(`

View File

@@ -215,12 +215,14 @@ export async function getPopularMints(window = '7d', limit = 20) {
/** /**
* Get popular mints from local database (fallback) * Get popular mints from local database (fallback)
* When no pageviews exist, falls back to trust score ranking
*/ */
function getLocalPopularMints(window = '7d', limit = 20) { function getLocalPopularMints(window = '7d', limit = 20) {
const since = window === '24h' ? hoursAgo(24) : const since = window === '24h' ? hoursAgo(24) :
window === '30d' ? daysAgo(30) : daysAgo(7); window === '30d' ? daysAgo(30) : daysAgo(7);
return query(` // First try to get mints with pageviews
const mintsWithViews = query(`
SELECT SELECT
m.mint_id, m.mint_id,
m.name, m.name,
@@ -236,11 +238,37 @@ function getLocalPopularMints(window = '7d', limit = 20) {
LEFT JOIN current_trust_scores ts ON m.mint_id = ts.mint_id LEFT JOIN current_trust_scores ts ON m.mint_id = ts.mint_id
WHERE (m.visibility IS NULL OR m.visibility = 'public') WHERE (m.visibility IS NULL OR m.visibility = 'public')
AND m.status != 'merged' AND m.status != 'merged'
AND m.status IN ('online', 'degraded')
GROUP BY m.mint_id GROUP BY m.mint_id
HAVING views > 0 HAVING views > 0
ORDER BY views DESC ORDER BY views DESC
LIMIT ? LIMIT ?
`, [since, limit]); `, [since, limit]);
if (mintsWithViews.length >= limit) {
return mintsWithViews;
}
// Fallback: return top mints by trust score if not enough pageview data
return query(`
SELECT
m.mint_id,
m.name,
m.canonical_url,
m.icon_url,
m.status,
ts.score_total as trust_score,
ts.score_level as trust_level,
0 as views,
0 as unique_sessions
FROM mints m
LEFT JOIN current_trust_scores ts ON m.mint_id = ts.mint_id
WHERE (m.visibility IS NULL OR m.visibility = 'public')
AND m.status != 'merged'
AND m.status IN ('online', 'degraded')
ORDER BY ts.score_total DESC NULLS LAST, m.name ASC
LIMIT ?
`, [limit]);
} }
// ========================================== // ==========================================

View File

@@ -219,12 +219,14 @@ export async function getTrendingMints(limit = 10, window = '7d') {
/** /**
* Get trending mints from local database (fallback) * Get trending mints from local database (fallback)
* When no pageviews exist, falls back to recently active mints by trust score
*/ */
function getLocalTrendingMints(limit = 10, window = '7d') { function getLocalTrendingMints(limit = 10, window = '7d') {
const since = window === '24h' ? hoursAgo(24) : daysAgo(7); const since = window === '24h' ? hoursAgo(24) : daysAgo(7);
const days = window === '24h' ? 1 : 7; const days = window === '24h' ? 1 : 7;
const mints = query(` // First try to get mints with pageviews
const mintsWithViews = query(`
SELECT SELECT
m.mint_id, m.mint_id,
m.canonical_url, m.canonical_url,
@@ -246,12 +248,39 @@ function getLocalTrendingMints(limit = 10, window = '7d') {
LIMIT ? LIMIT ?
`, [since, limit]); `, [since, limit]);
// Calculate view velocity if (mintsWithViews.length >= limit) {
return mints.map(m => ({ return mintsWithViews.map(m => ({
...m, ...m,
view_velocity: Math.round((m.view_count / days) * 10) / 10, view_velocity: Math.round((m.view_count / days) * 10) / 10,
source: 'local', source: 'local',
})); }));
}
// Fallback: return top mints by trust score if not enough pageview data
const fallbackMints = query(`
SELECT
m.mint_id,
m.canonical_url,
m.name,
m.icon_url,
m.status,
ts.score_total as trust_score,
ts.score_level as trust_level,
0 as view_count,
0 as unique_sessions
FROM mints m
LEFT JOIN current_trust_scores ts ON m.mint_id = ts.mint_id
WHERE m.status IN ('online', 'degraded')
AND (m.visibility IS NULL OR m.visibility = 'public')
ORDER BY ts.score_total DESC NULLS LAST, m.name ASC
LIMIT ?
`, [limit]);
return fallbackMints.map(m => ({
...m,
view_velocity: 0,
source: 'local',
}));
} }
/** /**

View File

@@ -292,7 +292,7 @@ export function getRecentReviews(limit = 20) {
m.canonical_url as mint_url m.canonical_url as mint_url
FROM reviews r FROM reviews r
LEFT JOIN mints m ON r.mint_id = m.mint_id LEFT JOIN mints m ON r.mint_id = m.mint_id
WHERE r.rating IS NOT NULL WHERE r.mint_id IS NOT NULL
ORDER BY r.created_at DESC ORDER BY r.created_at DESC
LIMIT ? LIMIT ?
`, [limit]); `, [limit]);
@@ -438,7 +438,7 @@ export function getRecentEcosystemReviews(limit = 20, since = null) {
r.created_at r.created_at
FROM reviews r FROM reviews r
LEFT JOIN mints m ON r.mint_id = m.mint_id LEFT JOIN mints m ON r.mint_id = m.mint_id
WHERE r.rating IS NOT NULL WHERE r.mint_id IS NOT NULL
`; `;
const params = []; const params = [];

View File

@@ -1,7 +1,9 @@
/** /**
* Time Utilities * Time Utilities
* *
* ISO-8601 UTC timestamps for consistency. * All timestamps are stored as ISO-8601 UTC format: YYYY-MM-DDTHH:MM:SS.sssZ
* This matches JavaScript's Date.toISOString() output.
* Database defaults use SQLite's strftime('%Y-%m-%dT%H:%M:%fZ', 'now') to match.
*/ */
/** /**
@@ -121,4 +123,3 @@ export function bucketStart(timestamp, bucketMs) {
const ts = new Date(timestamp).getTime(); const ts = new Date(timestamp).getTime();
return new Date(Math.floor(ts / bucketMs) * bucketMs).toISOString(); return new Date(Math.floor(ts / bucketMs) * bucketMs).toISOString();
} }