Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e0f0b2f1e9 | ||
|
|
390e1dc0ea | ||
|
|
fa8686276d | ||
|
|
498d7d8a7d |
@@ -86,8 +86,23 @@ npm run dev:photos # go run ./cmd/photo-api
|
||||
npm run build:photos # go build -o bin/photo-api
|
||||
npm run migrate:photos # apply photos_* migrations
|
||||
npm run test:photos # go test ./...
|
||||
|
||||
# Move the photo library between storage backends (both must be configured
|
||||
# in photo-api/.env; the source is left untouched, reruns skip what is there)
|
||||
npm run sync:photos:to-s3 # local disk -> S3
|
||||
npm run sync:photos:to-local # S3 -> local disk
|
||||
npm run sync:photos -- to-s3 --dry-run # flags: --dry-run --overwrite
|
||||
# --concurrency=N --gallery=<id>
|
||||
|
||||
# Uploads are deduplicated per gallery by content hash. Photos uploaded before
|
||||
# that existed need hashing once (idempotent, deletes nothing):
|
||||
npm run backfill:photos:checksums
|
||||
```
|
||||
|
||||
After a sync, set `STORAGE_BACKEND=s3` (or `local`) in `photo-api/.env` and
|
||||
restart the service to serve from the new backend. See
|
||||
[`photo-api/README.md`](photo-api/README.md#move-the-library-between-backends).
|
||||
|
||||
You can also run per workspace:
|
||||
|
||||
```bash
|
||||
@@ -117,6 +132,7 @@ Key settings (see `photo-api/.env.example`):
|
||||
- **DB**: `DB_TYPE` and `DATABASE_URL` — point at the **same** database as the backend
|
||||
- **Auth**: none needed for user auth — the service validates Better Auth session cookies against the shared database. `PHOTO_VIEW_SECRET` signs gallery image view tokens (falls back to `JWT_SECRET` during migration).
|
||||
- **Storage**: `STORAGE_PATH` (local disk) or `S3_ENDPOINT` + `S3_BUCKET` (S3/Garage/MinIO); S3 downloads use short-lived presigned URLs
|
||||
- **Storage switch**: `STORAGE_BACKEND=auto|local|s3` (`auto` = S3 when it is configured). Keep both sides configured and flip this one line to move between them; `npm run sync:photos:to-s3` / `:to-local` copies the existing library first
|
||||
- **Uploads/worker**: `MAX_UPLOAD_MB`, `WORKER_CONCURRENCY` (a worker generates thumb/preview JPEG variants with EXIF stripped)
|
||||
|
||||
### Frontend (`frontend/.env`)
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
# ============================================================
|
||||
# Spanglish Community - Photo Gallery API
|
||||
# photos.spanglishcommunity.com
|
||||
# ============================================================
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name photos.spanglishcommunity.com;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/html;
|
||||
}
|
||||
|
||||
location / {
|
||||
return 301 https://photos.spanglishcommunity.com$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
http2 on;
|
||||
|
||||
server_name photos.spanglishcommunity.com;
|
||||
|
||||
# Photos can be larger than typical JSON payloads
|
||||
client_max_body_size 25m;
|
||||
|
||||
# SSL
|
||||
ssl_certificate /etc/letsencrypt/live/photos.spanglishcommunity.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/photos.spanglishcommunity.com/privkey.pem;
|
||||
|
||||
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||
|
||||
# Security
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||
|
||||
# Logs
|
||||
access_log /var/log/nginx/spanglish_photo_access.log;
|
||||
error_log /var/log/nginx/spanglish_photo_error.log;
|
||||
|
||||
# CORS Configuration
|
||||
set $cors_origin "";
|
||||
if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") {
|
||||
set $cors_origin $http_origin;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||
|
||||
# Ensure 413 returns JSON + CORS
|
||||
error_page 413 = @payload_too_large;
|
||||
location @payload_too_large {
|
||||
default_type application/json;
|
||||
return 413 '{"error":"Payload too large (413). Please upload a smaller file."}';
|
||||
}
|
||||
|
||||
# Ensure 429 (rate limited) returns JSON + CORS
|
||||
error_page 429 = @rate_limited;
|
||||
location @rate_limited {
|
||||
default_type application/json;
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
return 429 '{"error":"Too many requests. Please slow down."}';
|
||||
}
|
||||
|
||||
location / {
|
||||
limit_req zone=spanglish_photo_limit burst=40 nodelay;
|
||||
|
||||
# Preflight
|
||||
if ($request_method = 'OPTIONS') {
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||
add_header 'Access-Control-Max-Age' 86400 always;
|
||||
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||
add_header 'Content-Length' 0;
|
||||
return 204;
|
||||
}
|
||||
|
||||
proxy_pass http://spanglish_photo_api;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 300s;
|
||||
|
||||
# Buffer large image uploads to disk rather than memory
|
||||
proxy_request_buffering on;
|
||||
proxy_max_temp_file_size 1024m;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
# ============================================================
|
||||
# Spanglish Community - Backend API
|
||||
# api.spanglishcommunity.com
|
||||
# ============================================================
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name api.spanglishcommunity.com;
|
||||
|
||||
# ACME
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/html;
|
||||
}
|
||||
|
||||
# Force HTTPS
|
||||
location / {
|
||||
return 301 https://api.spanglishcommunity.com$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
http2 on;
|
||||
|
||||
server_name api.spanglishcommunity.com;
|
||||
|
||||
# Upload size limit (avoid nginx 413 on media uploads)
|
||||
# Keep this >= backend MEDIA_MAX_UPLOAD_MB (default 10MB).
|
||||
client_max_body_size 20m;
|
||||
|
||||
# SSL
|
||||
ssl_certificate /etc/letsencrypt/live/spanglishcommunity.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/spanglishcommunity.com/privkey.pem;
|
||||
|
||||
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||
|
||||
# Security (API)
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
|
||||
# Logs
|
||||
access_log /var/log/nginx/spanglish_api_access.log;
|
||||
error_log /var/log/nginx/spanglish_api_error.log;
|
||||
|
||||
# CORS Configuration (set once, used everywhere)
|
||||
set $cors_origin "";
|
||||
if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") {
|
||||
set $cors_origin $http_origin;
|
||||
}
|
||||
|
||||
# Add CORS headers to all responses (including nginx-generated errors)
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||
|
||||
# Ensure 413 returns JSON + CORS (browser otherwise reports "CORS blocked")
|
||||
error_page 413 = @payload_too_large;
|
||||
location @payload_too_large {
|
||||
default_type application/json;
|
||||
return 413 '{"error":"Payload too large (413). Please upload a smaller file."}';
|
||||
}
|
||||
|
||||
# Photo gallery service (photo-api, port 3003). ^~ wins over the "/"
|
||||
# prefix below, so /api/photos/* reaches the Go photo-api instead of the
|
||||
# Node backend (which has no photo routes and would 404). The admin UI
|
||||
# calls this cross-origin via NEXT_PUBLIC_API_URL, so preflight + CORS
|
||||
# must be handled here just like location / below.
|
||||
location ^~ /api/photos/ {
|
||||
limit_req zone=spanglish_api_limit burst=50 nodelay;
|
||||
|
||||
# Handle preflight OPTIONS requests (add_header inside if{} does NOT
|
||||
# inherit server-level headers, so repeat all CORS headers here).
|
||||
if ($request_method = 'OPTIONS') {
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||
add_header 'Access-Control-Max-Age' 86400 always;
|
||||
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||
add_header 'Content-Length' 0;
|
||||
return 204;
|
||||
}
|
||||
|
||||
proxy_pass http://spanglish_photo_api;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Strip CORS headers from the service (nginx handles CORS here)
|
||||
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||
|
||||
# Photo batches can be large; allow bigger bodies + unbuffered upload.
|
||||
client_max_body_size 100m;
|
||||
proxy_request_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 300s;
|
||||
}
|
||||
|
||||
location / {
|
||||
limit_req zone=spanglish_api_limit burst=50 nodelay;
|
||||
|
||||
# Handle preflight OPTIONS requests
|
||||
# NOTE: add_header inside if{} does NOT inherit server-level headers,
|
||||
# so we must repeat all CORS headers here.
|
||||
if ($request_method = 'OPTIONS') {
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||
add_header 'Access-Control-Max-Age' 86400 always;
|
||||
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||
add_header 'Content-Length' 0;
|
||||
return 204;
|
||||
}
|
||||
|
||||
proxy_pass http://spanglish_backend;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Strip CORS headers from backend (nginx handles CORS at server level)
|
||||
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 300s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# ============================================================
|
||||
# Spanglish Community - Frontend
|
||||
# spanglishcommunity.com / www
|
||||
# ============================================================
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name spanglishcommunity.com www.spanglishcommunity.com;
|
||||
|
||||
# ACME
|
||||
location /.well-known/acme-challenge/ {
|
||||
root /var/www/html;
|
||||
}
|
||||
|
||||
# Force HTTPS
|
||||
location / {
|
||||
return 301 https://spanglishcommunity.com$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
http2 on;
|
||||
|
||||
server_name spanglishcommunity.com www.spanglishcommunity.com;
|
||||
|
||||
# Upload size limit (covers same-origin /api uploads via this vhost)
|
||||
client_max_body_size 20m;
|
||||
|
||||
# SSL
|
||||
ssl_certificate /etc/letsencrypt/live/spanglishcommunity.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/spanglishcommunity.com/privkey.pem;
|
||||
|
||||
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||
|
||||
# Security
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||
|
||||
# Logs
|
||||
access_log /var/log/nginx/spanglish_frontend_access.log;
|
||||
error_log /var/log/nginx/spanglish_frontend_error.log;
|
||||
|
||||
# Proxy /api/photos to the photo-api (Go service, port 3003). ^~ wins over
|
||||
# the /api prefix below so same-origin image/gallery requests reach the
|
||||
# photo-api instead of the Node backend (which has no photo routes -> 404).
|
||||
location ^~ /api/photos/ {
|
||||
proxy_pass http://spanglish_photo_api;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Photo uploads/batches can be large; allow bigger bodies.
|
||||
client_max_body_size 100m;
|
||||
proxy_request_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 300s;
|
||||
|
||||
# Let the photo-api set Cache-Control per image visibility (public vs.
|
||||
# token-gated) rather than forcing a cache policy here.
|
||||
}
|
||||
|
||||
# Proxy /api to backend
|
||||
location /api {
|
||||
proxy_pass http://spanglish_backend;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 300s;
|
||||
}
|
||||
|
||||
# Proxy /uploads to backend
|
||||
location /uploads {
|
||||
proxy_pass http://spanglish_backend;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# Cache static files
|
||||
proxy_cache_valid 200 1d;
|
||||
expires 1d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# Frontend App
|
||||
location / {
|
||||
proxy_pass http://spanglish_frontend;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket / HMR
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
proxy_read_timeout 60s;
|
||||
proxy_connect_timeout 60s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
# ============================================================
|
||||
# Spanglish Community - Photo Gallery API
|
||||
# photos.spanglishcommunity.com
|
||||
# ============================================================
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name photos.spanglishcommunity.com;
|
||||
|
||||
location /.well-known/acme-challenge/ {
|
||||
}
|
||||
|
||||
location / {
|
||||
return 301 https://photos.spanglishcommunity.com$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
http2 on;
|
||||
|
||||
server_name photos.spanglishcommunity.com;
|
||||
|
||||
# Photos can be larger than typical JSON payloads
|
||||
client_max_body_size 25m;
|
||||
|
||||
# SSL
|
||||
ssl_certificate /etc/letsencrypt/live/photos.spanglishcommunity.com/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/photos.spanglishcommunity.com/privkey.pem;
|
||||
|
||||
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||
|
||||
# Security
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||
|
||||
# Logs
|
||||
access_log /var/log/nginx/spanglish_photo_access.log;
|
||||
error_log /var/log/nginx/spanglish_photo_error.log;
|
||||
|
||||
# CORS Configuration
|
||||
set $cors_origin "";
|
||||
if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") {
|
||||
set $cors_origin $http_origin;
|
||||
}
|
||||
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||
|
||||
# Ensure 413 returns JSON + CORS
|
||||
error_page 413 = @payload_too_large;
|
||||
location @payload_too_large {
|
||||
default_type application/json;
|
||||
return 413 '{"error":"Payload too large (413). Please upload a smaller file."}';
|
||||
}
|
||||
|
||||
# Ensure 429 (rate limited) returns JSON + CORS
|
||||
error_page 429 = @rate_limited;
|
||||
location @rate_limited {
|
||||
default_type application/json;
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
return 429 '{"error":"Too many requests. Please slow down."}';
|
||||
}
|
||||
|
||||
location / {
|
||||
limit_req zone=spanglish_photo_limit burst=40 nodelay;
|
||||
|
||||
# Preflight
|
||||
if ($request_method = 'OPTIONS') {
|
||||
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||
add_header 'Access-Control-Max-Age' 86400 always;
|
||||
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||
add_header 'Content-Length' 0;
|
||||
return 204;
|
||||
}
|
||||
|
||||
proxy_pass http://spanglish_photo_api;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 300s;
|
||||
|
||||
# Buffer large image uploads to disk rather than memory
|
||||
proxy_request_buffering on;
|
||||
proxy_max_temp_file_size 1024m;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
upstream spanglish_frontend {
|
||||
server 127.0.0.1:3019;
|
||||
}
|
||||
upstream spanglish_backend {
|
||||
server 127.0.0.1:3018;
|
||||
}
|
||||
upstream spanglish_photo_api {
|
||||
server 127.0.0.1:3003;
|
||||
}
|
||||
|
||||
limit_req_zone $binary_remote_addr zone=spanglish_photo_limit:10m rate=20r/s;
|
||||
limit_req_zone $binary_remote_addr zone=spanglish_api_limit:10m rate=30r/s;
|
||||
@@ -0,0 +1,6 @@
|
||||
import GallerySkeleton from '@/components/gallery/GallerySkeleton';
|
||||
|
||||
// Shown while the server component fetches the event gallery.
|
||||
export default function Loading() {
|
||||
return <GallerySkeleton />;
|
||||
}
|
||||
@@ -7,11 +7,17 @@ import { useLanguage } from '@/context/LanguageContext';
|
||||
import { useAuth } from '@/context/AuthContext';
|
||||
import { photosApi, PhotoGallery, Photo } from '@/lib/api';
|
||||
import Button from '@/components/ui/Button';
|
||||
import { ImageGridSkeleton } from '@/components/ui/Skeleton';
|
||||
import GallerySkeleton from '@/components/gallery/GallerySkeleton';
|
||||
import PhotoTile from '@/components/gallery/PhotoTile';
|
||||
import {
|
||||
GalleryContainer,
|
||||
GalleryHeroFrame,
|
||||
MasonryGrid,
|
||||
} from '@/components/gallery/GalleryLayout';
|
||||
import { useDownloads } from '@/components/gallery/useDownloads';
|
||||
import Lightbox from '@/components/Lightbox';
|
||||
import LoginModal from '@/components/LoginModal';
|
||||
import {
|
||||
ArrowDownTrayIcon,
|
||||
CalendarIcon,
|
||||
CameraIcon,
|
||||
LinkIcon,
|
||||
@@ -42,6 +48,17 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien
|
||||
const [denied, setDenied] = useState<DeniedState>(null);
|
||||
const [lightboxIndex, setLightboxIndex] = useState<number | null>(null);
|
||||
const [loginOpen, setLoginOpen] = useState(false);
|
||||
const downloads = useDownloads(es);
|
||||
const downloadLabels = {
|
||||
download: es ? 'Descargar' : 'Download',
|
||||
downloading: es ? 'Descargando…' : 'Downloading…',
|
||||
};
|
||||
const downloadPhoto = (photo: Photo) =>
|
||||
downloads.start({
|
||||
id: photo.id,
|
||||
url: photo.urls.original,
|
||||
filename: photo.originalFilename,
|
||||
});
|
||||
|
||||
// Server-rendered public galleries need no client fetch. Everything else
|
||||
// (link/ticket/private) is fetched here with the share token and/or the
|
||||
@@ -78,14 +95,10 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien
|
||||
};
|
||||
}, [slug, eventSlug, shareToken, initial, authLoading, user?.id]);
|
||||
|
||||
// Mirrors the hero + masonry layout below, so the real page drops straight
|
||||
// into the placeholder's geometry instead of replacing it.
|
||||
if (loading || (authLoading && !initial)) {
|
||||
return (
|
||||
<div className="section-padding">
|
||||
<div className="container-page">
|
||||
<ImageGridSkeleton />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
return <GallerySkeleton count={gallery?.photoCount || undefined} />;
|
||||
}
|
||||
|
||||
// Gate pages for restricted galleries. After a successful login in the
|
||||
@@ -254,47 +267,48 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien
|
||||
|
||||
return (
|
||||
<div>
|
||||
{/* Hero */}
|
||||
<div className="relative bg-brand-navy overflow-hidden">
|
||||
{heroUrl && (
|
||||
<>
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
<img
|
||||
src={heroUrl}
|
||||
alt=""
|
||||
className="absolute inset-0 w-full h-full object-cover opacity-60"
|
||||
/>
|
||||
<div className="absolute inset-0 bg-gradient-to-t from-black/80 via-black/30 to-black/20" />
|
||||
</>
|
||||
{/* Hero — same frame the skeleton renders (GalleryLayout). */}
|
||||
<GalleryHeroFrame
|
||||
backdrop={
|
||||
heroUrl ? (
|
||||
<>
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
<img
|
||||
src={heroUrl}
|
||||
alt=""
|
||||
className="absolute inset-0 w-full h-full object-cover opacity-60"
|
||||
/>
|
||||
<div className="absolute inset-0 bg-gradient-to-t from-black/80 via-black/30 to-black/20" />
|
||||
</>
|
||||
) : null
|
||||
}
|
||||
>
|
||||
<h1 className="font-heading font-bold text-3xl md:text-5xl text-white drop-shadow-sm">
|
||||
{title}
|
||||
</h1>
|
||||
{description && (
|
||||
<p className="mt-2 max-w-2xl text-white/90 text-sm md:text-base">{description}</p>
|
||||
)}
|
||||
<div className="relative container-page px-4 pt-20 pb-8 md:pt-32 md:pb-12">
|
||||
<h1 className="font-heading font-bold text-3xl md:text-5xl text-white drop-shadow-sm">
|
||||
{title}
|
||||
</h1>
|
||||
{description && (
|
||||
<p className="mt-2 max-w-2xl text-white/90 text-sm md:text-base">{description}</p>
|
||||
<div className="mt-4 flex flex-wrap items-center gap-2 text-sm">
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-white/15 backdrop-blur px-3 py-1 text-white">
|
||||
<CameraIcon className="w-4 h-4" />
|
||||
{readyPhotos.length} {es ? 'fotos' : 'photos'}
|
||||
</span>
|
||||
{gallery.event && (
|
||||
<Link
|
||||
href={`/events/${gallery.event.slug}`}
|
||||
className="inline-flex items-center gap-1.5 rounded-full bg-primary-yellow px-3 py-1 text-primary-dark font-medium hover:brightness-105"
|
||||
>
|
||||
<CalendarIcon className="w-4 h-4" />
|
||||
{eventTitle}
|
||||
{eventDate && <span className="hidden sm:inline font-normal">· {eventDate}</span>}
|
||||
</Link>
|
||||
)}
|
||||
<div className="mt-4 flex flex-wrap items-center gap-2 text-sm">
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full bg-white/15 backdrop-blur px-3 py-1 text-white">
|
||||
<CameraIcon className="w-4 h-4" />
|
||||
{readyPhotos.length} {es ? 'fotos' : 'photos'}
|
||||
</span>
|
||||
{gallery.event && (
|
||||
<Link
|
||||
href={`/events/${gallery.event.slug}`}
|
||||
className="inline-flex items-center gap-1.5 rounded-full bg-primary-yellow px-3 py-1 text-primary-dark font-medium hover:brightness-105"
|
||||
>
|
||||
<CalendarIcon className="w-4 h-4" />
|
||||
{eventTitle}
|
||||
{eventDate && <span className="hidden sm:inline font-normal">· {eventDate}</span>}
|
||||
</Link>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</GalleryHeroFrame>
|
||||
|
||||
{/* Masonry grid */}
|
||||
<div className="container-page px-2 sm:px-4 py-4 md:py-8">
|
||||
<GalleryContainer>
|
||||
{readyPhotos.length === 0 ? (
|
||||
<div className="text-center py-16">
|
||||
<CameraIcon className="w-16 h-16 mx-auto text-gray-300 mb-4" />
|
||||
@@ -303,46 +317,19 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="columns-2 sm:columns-3 lg:columns-4 gap-2 md:gap-3 [column-fill:_balance]">
|
||||
<MasonryGrid>
|
||||
{readyPhotos.map((photo, i) => (
|
||||
<div
|
||||
<PhotoTile
|
||||
key={photo.id}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onClick={() => setLightboxIndex(i)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault();
|
||||
setLightboxIndex(i);
|
||||
}
|
||||
}}
|
||||
className="group relative mb-2 md:mb-3 break-inside-avoid overflow-hidden rounded-xl bg-gray-100 cursor-pointer focus:outline-none focus:ring-2 focus:ring-primary-yellow"
|
||||
style={
|
||||
photo.width && photo.height
|
||||
? { aspectRatio: `${photo.width} / ${photo.height}` }
|
||||
: undefined
|
||||
}
|
||||
>
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
<img
|
||||
src={photo.urls.thumb}
|
||||
alt=""
|
||||
loading={i < 8 ? 'eager' : 'lazy'}
|
||||
className="w-full h-auto group-hover:scale-[1.03] transition-transform duration-300"
|
||||
/>
|
||||
<div className="absolute inset-0 bg-black/0 group-hover:bg-black/20 transition-colors" />
|
||||
<a
|
||||
href={photo.urls.original}
|
||||
download={photo.originalFilename || true}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className="absolute bottom-2 right-2 hidden md:flex p-2 rounded-full bg-black/50 text-white opacity-0 group-hover:opacity-100 transition-opacity hover:bg-black/80"
|
||||
aria-label={es ? 'Descargar' : 'Download'}
|
||||
>
|
||||
<ArrowDownTrayIcon className="w-4 h-4" />
|
||||
</a>
|
||||
</div>
|
||||
photo={photo}
|
||||
eager={i < 8}
|
||||
onOpen={() => setLightboxIndex(i)}
|
||||
onDownload={() => downloadPhoto(photo)}
|
||||
downloading={downloads.isPending(photo.id)}
|
||||
labels={downloadLabels}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</MasonryGrid>
|
||||
)}
|
||||
|
||||
{lightboxIndex !== null && (
|
||||
@@ -351,9 +338,16 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien
|
||||
index={lightboxIndex}
|
||||
onClose={() => setLightboxIndex(null)}
|
||||
onNavigate={setLightboxIndex}
|
||||
onDownload={(it) =>
|
||||
downloads.start({ id: it.id, url: it.downloadUrl, filename: it.filename })
|
||||
}
|
||||
downloadingId={
|
||||
lightboxItems.find((it) => downloads.isPending(it.id))?.id ?? null
|
||||
}
|
||||
downloadLabels={downloadLabels}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</GalleryContainer>
|
||||
|
||||
{/* Call to action: send attendees to their dashboard, everyone else to
|
||||
the next event. Auth state comes from the same useAuth() the gate
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import GallerySkeleton from '@/components/gallery/GallerySkeleton';
|
||||
|
||||
// Shown while the server component fetches the gallery, so the first paint is
|
||||
// already the gallery's layout rather than an empty page.
|
||||
export default function Loading() {
|
||||
return <GallerySkeleton />;
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useLanguage } from '@/context/LanguageContext';
|
||||
import { usePrivacy } from '@/context/PrivacyContext';
|
||||
import { paymentsApi, adminApi, eventsApi, PaymentWithDetails, Event, ExportedPayment, FinancialSummary } from '@/lib/api';
|
||||
import { isManualProvider } from '@/lib/api/payments';
|
||||
import { parseDate, formatRucDisplay } from '@/lib/utils';
|
||||
@@ -36,6 +37,7 @@ type Tab = 'pending_approval' | 'all';
|
||||
|
||||
export default function AdminPaymentsPage() {
|
||||
const { t, locale } = useLanguage();
|
||||
const { privacyMode } = usePrivacy();
|
||||
const [payments, setPayments] = useState<PaymentWithDetails[]>([]);
|
||||
const [pendingApprovalPayments, setPendingApprovalPayments] = useState<PaymentWithDetails[]>([]);
|
||||
// Manual-gateway payments still in bare 'pending': the customer may have paid
|
||||
@@ -778,6 +780,7 @@ export default function AdminPaymentsPage() {
|
||||
)}
|
||||
|
||||
{/* Summary Cards */}
|
||||
{!privacyMode && (
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-5 gap-4 mb-6">
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center gap-3">
|
||||
@@ -846,6 +849,7 @@ export default function AdminPaymentsPage() {
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Tabs */}
|
||||
<div className="border-b mb-6 overflow-x-auto scrollbar-hide">
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
CheckCircleIcon,
|
||||
ChevronDownIcon,
|
||||
ChevronUpIcon,
|
||||
DocumentDuplicateIcon,
|
||||
ExclamationCircleIcon,
|
||||
ExclamationTriangleIcon,
|
||||
LinkIcon,
|
||||
@@ -34,7 +35,9 @@ interface UploadItem {
|
||||
name: string;
|
||||
sizeBytes: number;
|
||||
progress: number; // 0..1 while uploading
|
||||
status: 'queued' | 'uploading' | 'processing' | 'error';
|
||||
// 'duplicate' is terminal: the gallery already held these bytes, so nothing
|
||||
// was stored and no new tile appears in the grid.
|
||||
status: 'queued' | 'uploading' | 'processing' | 'error' | 'duplicate';
|
||||
error?: string;
|
||||
photoId?: string;
|
||||
}
|
||||
@@ -117,8 +120,19 @@ export default function AdminGalleryDetailPage() {
|
||||
const { photos: added } = await photosApi.uploadPhotoWithProgress(id, file, (fraction) =>
|
||||
patchUpload(key, { progress: fraction })
|
||||
);
|
||||
setPhotos((prev) => [...prev, ...added]);
|
||||
patchUpload(key, { status: 'processing', progress: 1, photoId: added[0]?.id });
|
||||
// A duplicate echoes back a photo already in the grid, so merge by id
|
||||
// instead of appending (`duplicate` is an upload outcome, not photo
|
||||
// state, so it is dropped here).
|
||||
setPhotos((prev) => {
|
||||
const byId = new Map(prev.map((p) => [p.id, p]));
|
||||
added.forEach(({ duplicate: _duplicate, ...photo }) => byId.set(photo.id, photo));
|
||||
return Array.from(byId.values());
|
||||
});
|
||||
patchUpload(key, {
|
||||
status: added[0]?.duplicate ? 'duplicate' : 'processing',
|
||||
progress: 1,
|
||||
photoId: added[0]?.id,
|
||||
});
|
||||
} catch (err) {
|
||||
patchUpload(key, {
|
||||
status: 'error',
|
||||
@@ -148,7 +162,8 @@ export default function AdminGalleryDetailPage() {
|
||||
};
|
||||
|
||||
// A row is "done" once its photo finished processing; the panel derives
|
||||
// this from the photos list instead of tracking it separately.
|
||||
// this from the photos list instead of tracking it separately. 'duplicate'
|
||||
// is terminal and never reconciled — its photo was already there.
|
||||
const displayStatus = (u: UploadItem): { state: string; error?: string } => {
|
||||
if (u.status === 'processing' && u.photoId) {
|
||||
const photo = photos.find((p) => p.id === u.photoId);
|
||||
@@ -579,6 +594,11 @@ export default function AdminGalleryDetailPage() {
|
||||
{ds.state === 'queued' && (es ? 'En cola' : 'Queued')}
|
||||
{ds.state === 'processing' && (es ? 'Procesando…' : 'Processing…')}
|
||||
{ds.state === 'done' && formatBytes(u.sizeBytes)}
|
||||
{ds.state === 'duplicate' && (
|
||||
<span className="text-amber-600">
|
||||
{es ? 'Ya está en esta galería' : 'Already in this gallery'}
|
||||
</span>
|
||||
)}
|
||||
{ds.state === 'error' && (
|
||||
<span className="text-red-600" title={ds.error}>
|
||||
{ds.error}
|
||||
@@ -588,6 +608,9 @@ export default function AdminGalleryDetailPage() {
|
||||
</div>
|
||||
<span className="shrink-0">
|
||||
{ds.state === 'done' && <CheckCircleIcon className="w-6 h-6 text-green-600" />}
|
||||
{ds.state === 'duplicate' && (
|
||||
<DocumentDuplicateIcon className="w-6 h-6 text-amber-600" />
|
||||
)}
|
||||
{ds.state === 'error' && <ExclamationCircleIcon className="w-6 h-6 text-red-600" />}
|
||||
{ds.state === 'processing' && (
|
||||
<div className="animate-spin w-5 h-5 border-2 border-primary-yellow border-t-transparent rounded-full" />
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
ChevronRightIcon,
|
||||
ArrowDownTrayIcon,
|
||||
} from '@heroicons/react/24/outline';
|
||||
import Spinner from '@/components/ui/Spinner';
|
||||
|
||||
export interface LightboxItem {
|
||||
id: string;
|
||||
@@ -24,17 +25,35 @@ interface LightboxProps {
|
||||
onNavigate: (index: number) => void;
|
||||
/** Extra per-item action buttons rendered in the top bar (admin use). */
|
||||
renderActions?: (item: LightboxItem, index: number) => React.ReactNode;
|
||||
/**
|
||||
* Handles the download in JS instead of navigating, so the button can show
|
||||
* progress. Without it the button stays a plain <a download>.
|
||||
*/
|
||||
onDownload?: (item: LightboxItem) => void;
|
||||
/** Id of the item currently downloading (pairs with onDownload). */
|
||||
downloadingId?: string | null;
|
||||
downloadLabels?: { download: string; downloading: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-screen photo lightbox with keyboard and swipe navigation, in the
|
||||
* style of the admin gallery preview modal (fixed inset-0 bg-black/90).
|
||||
*/
|
||||
export default function Lightbox({ items, index, onClose, onNavigate, renderActions }: LightboxProps) {
|
||||
export default function Lightbox({
|
||||
items,
|
||||
index,
|
||||
onClose,
|
||||
onNavigate,
|
||||
renderActions,
|
||||
onDownload,
|
||||
downloadingId,
|
||||
downloadLabels,
|
||||
}: LightboxProps) {
|
||||
const touchStart = useRef<{ x: number; y: number } | null>(null);
|
||||
const activeThumbRef = useRef<HTMLButtonElement | null>(null);
|
||||
const item = items[index];
|
||||
const hasMultiple = items.length > 1;
|
||||
const downloading = !!downloadingId && item?.id === downloadingId;
|
||||
|
||||
const prev = useCallback(() => {
|
||||
onNavigate(index > 0 ? index - 1 : items.length - 1);
|
||||
@@ -98,14 +117,37 @@ export default function Lightbox({ items, index, onClose, onNavigate, renderActi
|
||||
className="absolute top-4 left-4 z-10 flex items-center gap-4"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<a
|
||||
href={item.downloadUrl}
|
||||
download={item.filename || true}
|
||||
className="text-white hover:text-gray-300"
|
||||
aria-label="Download"
|
||||
>
|
||||
<ArrowDownTrayIcon className="w-7 h-7" />
|
||||
</a>
|
||||
{onDownload ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onDownload(item)}
|
||||
disabled={downloading}
|
||||
aria-busy={downloading}
|
||||
aria-label={
|
||||
downloading
|
||||
? downloadLabels?.downloading || 'Downloading…'
|
||||
: downloadLabels?.download || 'Download'
|
||||
}
|
||||
className={`text-white hover:text-gray-300 flex items-center ${
|
||||
downloading ? 'cursor-wait' : ''
|
||||
}`}
|
||||
>
|
||||
{downloading ? (
|
||||
<Spinner className="w-6 h-6" />
|
||||
) : (
|
||||
<ArrowDownTrayIcon className="w-7 h-7" />
|
||||
)}
|
||||
</button>
|
||||
) : (
|
||||
<a
|
||||
href={item.downloadUrl}
|
||||
download={item.filename || true}
|
||||
className="text-white hover:text-gray-300"
|
||||
aria-label="Download"
|
||||
>
|
||||
<ArrowDownTrayIcon className="w-7 h-7" />
|
||||
</a>
|
||||
)}
|
||||
{renderActions?.(item, index)}
|
||||
</div>
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
// Layout shell shared by the public gallery page (GalleryClient) and its
|
||||
// loading placeholder (GallerySkeleton). Column count, gaps, radii and
|
||||
// container padding are defined once here, so the skeleton's geometry can
|
||||
// never drift from the grid it stands in for.
|
||||
|
||||
export function GalleryHeroFrame({
|
||||
children,
|
||||
backdrop,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
/** Cover image + scrim, absolutely positioned behind the text. */
|
||||
backdrop?: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<div className="relative bg-brand-navy overflow-hidden">
|
||||
{backdrop}
|
||||
<div className="relative container-page px-4 pt-20 pb-8 md:pt-32 md:pb-12">{children}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function GalleryContainer({ children }: { children: React.ReactNode }) {
|
||||
return <div className="container-page px-2 sm:px-4 py-4 md:py-8">{children}</div>;
|
||||
}
|
||||
|
||||
export function MasonryGrid({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
<div className="columns-2 sm:columns-3 lg:columns-4 gap-2 md:gap-3 [column-fill:_balance]">
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Per-tile geometry. Deliberately carries no background so callers can pick
|
||||
// one (photo tiles: bg-gray-100, skeleton tiles: the skeleton surface) without
|
||||
// two `bg-*` utilities fighting over CSS order.
|
||||
export const masonryTileClass = 'mb-2 md:mb-3 break-inside-avoid overflow-hidden rounded-xl';
|
||||
|
||||
/**
|
||||
* Reserves the tile's aspect ratio up front so the image can load into a box
|
||||
* that is already the right size — late arrivals never reflow the columns.
|
||||
* Returns undefined when the photo has no stored dimensions.
|
||||
*/
|
||||
export function aspectStyle(width?: number, height?: number): React.CSSProperties | undefined {
|
||||
return width && height ? { aspectRatio: `${width} / ${height}` } : undefined;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import clsx from 'clsx';
|
||||
import { Skeleton, SkeletonGroup } from '@/components/ui/Skeleton';
|
||||
import { GalleryContainer, GalleryHeroFrame, MasonryGrid, masonryTileClass } from './GalleryLayout';
|
||||
|
||||
// Loading placeholder for the public gallery page. It renders through the same
|
||||
// hero frame, container and masonry grid as the real page (see GalleryLayout),
|
||||
// so replacing it with photos changes only the pixels inside the tiles.
|
||||
|
||||
// Portrait/landscape/square mix standing in for a real event set. Fixed order
|
||||
// on purpose — a random shuffle would differ between the server and client
|
||||
// render and blow up hydration.
|
||||
const FALLBACK_RATIOS = [3 / 4, 4 / 3, 1, 2 / 3, 3 / 2, 4 / 5, 1, 3 / 4, 16 / 9, 4 / 5, 3 / 4, 4 / 3];
|
||||
|
||||
interface GallerySkeletonProps {
|
||||
/** Number of tiles to draw; ignored when `ratios` is given. */
|
||||
count?: number;
|
||||
/**
|
||||
* Real width/height ratios when the caller already knows them (e.g. a
|
||||
* cached gallery payload). Produces a grid with exactly the right geometry
|
||||
* instead of the guessed mix above.
|
||||
*/
|
||||
ratios?: number[];
|
||||
}
|
||||
|
||||
export default function GallerySkeleton({ count = 12, ratios }: GallerySkeletonProps) {
|
||||
const tiles =
|
||||
ratios && ratios.length > 0
|
||||
? ratios
|
||||
: Array.from({ length: count }, (_, i) => FALLBACK_RATIOS[i % FALLBACK_RATIOS.length]);
|
||||
|
||||
return (
|
||||
<SkeletonGroup>
|
||||
<GalleryHeroFrame>
|
||||
{/* Matches the h1 (text-3xl / md:text-5xl) and the pill row below it. */}
|
||||
<Skeleton tone="on-dark" className="h-9 md:h-12 w-2/3 max-w-md" />
|
||||
<div className="mt-4 flex flex-wrap items-center gap-2">
|
||||
<Skeleton tone="on-dark" className="h-7 w-28 rounded-full" />
|
||||
<Skeleton tone="on-dark" className="h-7 w-56 max-w-[60%] rounded-full" />
|
||||
</div>
|
||||
</GalleryHeroFrame>
|
||||
|
||||
<GalleryContainer>
|
||||
<MasonryGrid>
|
||||
{tiles.map((ratio, i) => (
|
||||
<div
|
||||
key={i}
|
||||
aria-hidden="true"
|
||||
className={clsx(
|
||||
masonryTileClass,
|
||||
'bg-secondary-light-gray/70 animate-pulse motion-reduce:animate-none'
|
||||
)}
|
||||
style={{ aspectRatio: `${ratio}` }}
|
||||
/>
|
||||
))}
|
||||
</MasonryGrid>
|
||||
</GalleryContainer>
|
||||
</SkeletonGroup>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import clsx from 'clsx';
|
||||
import { ArrowDownTrayIcon } from '@heroicons/react/24/outline';
|
||||
import type { Photo } from '@/lib/api';
|
||||
import Spinner from '@/components/ui/Spinner';
|
||||
import { aspectStyle, masonryTileClass } from './GalleryLayout';
|
||||
|
||||
interface PhotoTileProps {
|
||||
photo: Photo;
|
||||
/** Above-the-fold tiles load eagerly, the rest lazily. */
|
||||
eager?: boolean;
|
||||
onOpen: () => void;
|
||||
onDownload: () => void;
|
||||
downloading: boolean;
|
||||
labels: { download: string; downloading: string };
|
||||
}
|
||||
|
||||
export default function PhotoTile({
|
||||
photo,
|
||||
eager,
|
||||
onOpen,
|
||||
onDownload,
|
||||
downloading,
|
||||
labels,
|
||||
}: PhotoTileProps) {
|
||||
const imgRef = useRef<HTMLImageElement>(null);
|
||||
// 'initial' is what the server renders: fully visible, so a public gallery
|
||||
// still paints its photos if hydration is slow or JS never arrives. The
|
||||
// fade only takes over once we're mounted and know the image is still in
|
||||
// flight — a cached image reports `complete` here and skips it entirely
|
||||
// (its onLoad already fired before React attached the handler).
|
||||
const [state, setState] = useState<'initial' | 'pending' | 'loaded'>('initial');
|
||||
const pending = state === 'pending';
|
||||
|
||||
useEffect(() => {
|
||||
setState(imgRef.current?.complete ? 'loaded' : 'pending');
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onClick={onOpen}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault();
|
||||
onOpen();
|
||||
}
|
||||
}}
|
||||
className={clsx(
|
||||
masonryTileClass,
|
||||
'group relative bg-gray-100 cursor-pointer focus:outline-none focus:ring-2 focus:ring-primary-yellow'
|
||||
)}
|
||||
/* Sized from the stored dimensions before the bytes arrive, so a slow
|
||||
image never pushes its column around. */
|
||||
style={aspectStyle(photo.width, photo.height)}
|
||||
>
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
<img
|
||||
ref={imgRef}
|
||||
src={photo.urls.thumb}
|
||||
alt=""
|
||||
loading={eager ? 'eager' : 'lazy'}
|
||||
onLoad={() => setState('loaded')}
|
||||
onError={() => setState('loaded')}
|
||||
className={clsx(
|
||||
'w-full h-auto transition-[opacity,transform] duration-300 motion-reduce:transition-none group-hover:scale-[1.03]',
|
||||
pending ? 'opacity-0' : 'opacity-100'
|
||||
)}
|
||||
/>
|
||||
{pending && (
|
||||
<span
|
||||
aria-hidden="true"
|
||||
className="absolute inset-0 bg-secondary-light-gray/70 animate-pulse motion-reduce:animate-none"
|
||||
/>
|
||||
)}
|
||||
<div className="absolute inset-0 bg-black/0 group-hover:bg-black/20 transition-colors" />
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onDownload();
|
||||
}}
|
||||
disabled={downloading}
|
||||
aria-busy={downloading}
|
||||
aria-label={downloading ? labels.downloading : labels.download}
|
||||
className={clsx(
|
||||
'absolute bottom-2 right-2 hidden md:flex p-2 rounded-full bg-black/50 text-white transition-opacity hover:bg-black/80 focus:outline-none focus-visible:opacity-100 focus-visible:ring-2 focus-visible:ring-primary-yellow',
|
||||
downloading
|
||||
? 'opacity-100 cursor-wait hover:bg-black/50'
|
||||
: 'opacity-0 group-hover:opacity-100'
|
||||
)}
|
||||
>
|
||||
{downloading ? <Spinner /> : <ArrowDownTrayIcon className="w-4 h-4" />}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
'use client';
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import toast from 'react-hot-toast';
|
||||
|
||||
// Per-photo download state for the public gallery. The photo-api serves full
|
||||
// quality originals, which takes a few seconds, so downloads run through
|
||||
// fetch() instead of a bare <a download> — that gives us a spinner, a real
|
||||
// success/failure signal, a timeout and abort-on-unmount.
|
||||
|
||||
// Originals run to ~20 MB, so a single flat timeout would either be too short
|
||||
// for a phone on mobile data or useless as a stall detector. Time out on the
|
||||
// response headers instead, then give the body a generous ceiling.
|
||||
const HEADERS_TIMEOUT_MS = 30_000;
|
||||
const BODY_TIMEOUT_MS = 10 * 60_000;
|
||||
|
||||
export interface DownloadRequest {
|
||||
/** Photo id; keys the in-flight state so tiles stay independent. */
|
||||
id: string;
|
||||
url: string;
|
||||
filename?: string;
|
||||
}
|
||||
|
||||
/** Saves fetched bytes without navigating away from the gallery. */
|
||||
function saveBlob(blob: Blob, filename: string) {
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = objectUrl;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
// Give the browser time to start the save before the URL is invalidated.
|
||||
setTimeout(() => URL.revokeObjectURL(objectUrl), 10_000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Plain-anchor download, i.e. what the page did before this hook existed.
|
||||
* Used as a fallback when fetch() itself fails: with S3 storage the file
|
||||
* endpoint 302s to a presigned URL on another origin, which a cross-origin
|
||||
* fetch cannot read but a navigation downloads fine (the presigned URL
|
||||
* carries its own Content-Disposition).
|
||||
*/
|
||||
function navigateToDownload({ url, filename }: DownloadRequest) {
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
if (filename) a.download = filename;
|
||||
a.rel = 'noopener';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
}
|
||||
|
||||
export function useDownloads(es: boolean) {
|
||||
const [pending, setPending] = useState<ReadonlySet<string>>(() => new Set());
|
||||
const controllers = useRef(new Map<string, AbortController>());
|
||||
const unmounted = useRef(false);
|
||||
// Lets the retry button in the error toast call the latest `start`.
|
||||
const startRef = useRef<(req: DownloadRequest) => void>(() => {});
|
||||
|
||||
useEffect(() => {
|
||||
unmounted.current = false;
|
||||
return () => {
|
||||
unmounted.current = true;
|
||||
controllers.current.forEach((c) => c.abort());
|
||||
controllers.current.clear();
|
||||
};
|
||||
}, []);
|
||||
|
||||
const start = useCallback(
|
||||
async (req: DownloadRequest) => {
|
||||
// Repeat clicks while the same photo is in flight are no-ops; other
|
||||
// photos are unaffected because state is keyed by id.
|
||||
if (controllers.current.has(req.id)) return;
|
||||
|
||||
const controller = new AbortController();
|
||||
controllers.current.set(req.id, controller);
|
||||
setPending((prev) => new Set(prev).add(req.id));
|
||||
|
||||
let timer = setTimeout(() => controller.abort(), HEADERS_TIMEOUT_MS);
|
||||
const timedOut = () => controller.signal.aborted && !unmounted.current;
|
||||
|
||||
try {
|
||||
const res = await fetch(req.url, {
|
||||
credentials: 'same-origin',
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(() => controller.abort(), BODY_TIMEOUT_MS);
|
||||
if (!res.ok) throw new Error(`Download failed (${res.status})`);
|
||||
const blob = await res.blob();
|
||||
saveBlob(blob, req.filename || `${req.id}.jpg`);
|
||||
} catch (err) {
|
||||
if (unmounted.current) return; // page gone, nothing to report
|
||||
if (err instanceof TypeError) {
|
||||
// Network-level failure — most likely a cross-origin presigned
|
||||
// redirect. Hand it to the browser, which can follow it.
|
||||
navigateToDownload(req);
|
||||
} else {
|
||||
const message = timedOut()
|
||||
? es
|
||||
? 'La descarga tardó demasiado.'
|
||||
: 'The download timed out.'
|
||||
: es
|
||||
? 'No se pudo descargar la foto.'
|
||||
: 'Could not download the photo.';
|
||||
toast.error(
|
||||
(t) => (
|
||||
<span className="flex items-center gap-3">
|
||||
{message}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
toast.dismiss(t.id);
|
||||
startRef.current(req);
|
||||
}}
|
||||
className="font-medium underline underline-offset-2 whitespace-nowrap"
|
||||
>
|
||||
{es ? 'Reintentar' : 'Retry'}
|
||||
</button>
|
||||
</span>
|
||||
),
|
||||
{ duration: 6000 }
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
controllers.current.delete(req.id);
|
||||
if (!unmounted.current) {
|
||||
setPending((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(req.id);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
[es]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
startRef.current = start;
|
||||
}, [start]);
|
||||
|
||||
const isPending = useCallback((id: string) => pending.has(id), [pending]);
|
||||
|
||||
return { start, isPending };
|
||||
}
|
||||
@@ -9,11 +9,21 @@ interface SkeletonProps {
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function Skeleton({ className }: SkeletonProps) {
|
||||
export function Skeleton({
|
||||
className,
|
||||
tone = 'default',
|
||||
}: SkeletonProps & {
|
||||
/** `on-dark` lightens the surface for placeholders over a dark hero. */
|
||||
tone?: 'default' | 'on-dark';
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
aria-hidden="true"
|
||||
className={clsx('animate-pulse rounded-lg bg-secondary-light-gray/70', className)}
|
||||
className={clsx(
|
||||
'animate-pulse motion-reduce:animate-none rounded-lg',
|
||||
tone === 'on-dark' ? 'bg-white/20' : 'bg-secondary-light-gray/70',
|
||||
className
|
||||
)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import clsx from 'clsx';
|
||||
|
||||
// Inline busy indicator in the same style as the full-page loaders
|
||||
// (animate-spin ring with one contrasting edge). Purely decorative: callers
|
||||
// own the accessible state via aria-busy / aria-label. Defaults to the
|
||||
// white-on-dark ring used by the photo overlay controls; pass `border-*`
|
||||
// classes to restyle it elsewhere.
|
||||
export default function Spinner({ className }: { className?: string }) {
|
||||
return (
|
||||
<span
|
||||
aria-hidden="true"
|
||||
className={clsx(
|
||||
'inline-block rounded-full border-2 animate-spin border-white/40 border-t-white',
|
||||
className || 'w-4 h-4'
|
||||
)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -52,6 +52,9 @@ export interface Photo {
|
||||
preview?: string;
|
||||
original: string;
|
||||
};
|
||||
// Upload responses only: these bytes were already in the gallery, so nothing
|
||||
// was stored and the rest of this object describes the existing photo.
|
||||
duplicate?: boolean;
|
||||
}
|
||||
|
||||
export interface CreateGalleryInput {
|
||||
|
||||
@@ -14,6 +14,10 @@
|
||||
"build:photos": "cd photo-api && go build -o bin/photo-api ./cmd/photo-api",
|
||||
"test:photos": "cd photo-api && go test ./...",
|
||||
"migrate:photos": "cd photo-api && go run ./cmd/photo-api migrate",
|
||||
"sync:photos": "cd photo-api && go run ./cmd/photo-api sync",
|
||||
"sync:photos:to-s3": "cd photo-api && go run ./cmd/photo-api sync to-s3",
|
||||
"sync:photos:to-local": "cd photo-api && go run ./cmd/photo-api sync to-local",
|
||||
"backfill:photos:checksums": "cd photo-api && go run ./cmd/photo-api backfill-checksums",
|
||||
"start": "concurrently \"npm run start:backend\" \"npm run start:frontend\"",
|
||||
"start:backend": "npm run start --workspace=backend",
|
||||
"start:frontend": "npm run start --workspace=frontend",
|
||||
|
||||
+12
-3
@@ -29,10 +29,19 @@ JWT_SECRET=
|
||||
# Public site origin, used to build share links and allow dev CORS
|
||||
FRONTEND_URL=https://spanglishcommunity.com
|
||||
|
||||
# Photo storage. Local disk by default; setting BOTH S3_ENDPOINT and
|
||||
# S3_BUCKET switches to S3 (same convention as the backend's media storage).
|
||||
# Use a photos-specific bucket — do not reuse the backend's media bucket.
|
||||
# Photo storage. Which backend serves requests:
|
||||
# auto (default) S3 when S3_ENDPOINT + S3_BUCKET are set, local otherwise
|
||||
# — the backend's convention (backend/src/lib/storage.ts)
|
||||
# local always local disk, even with the S3 settings filled in
|
||||
# s3 always S3
|
||||
# Keep both sides filled in and flip this one line to switch backends. Both
|
||||
# are also required by the library sync (`npm run sync:photos:to-s3` /
|
||||
# `:to-local`), which copies every photo from one backend to the other.
|
||||
STORAGE_BACKEND=auto
|
||||
|
||||
STORAGE_PATH=./data/photos
|
||||
|
||||
# Use a photos-specific bucket — do not reuse the backend's media bucket.
|
||||
#S3_ENDPOINT=
|
||||
#S3_REGION=auto
|
||||
#S3_BUCKET=spanglish-photos
|
||||
|
||||
+11
-2
@@ -1,11 +1,11 @@
|
||||
.PHONY: start build test migrate clean help
|
||||
.PHONY: start build test migrate sync-to-s3 sync-to-local backfill-checksums clean help
|
||||
|
||||
BINARY := bin/photo-api
|
||||
CMD := ./cmd/photo-api
|
||||
|
||||
help: ## Show available targets
|
||||
@grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \
|
||||
awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}'
|
||||
awk 'BEGIN {FS = ":.*?## "}; {printf " %-19s %s\n", $$1, $$2}'
|
||||
|
||||
start: ## Run the photo-api server (go run)
|
||||
go run $(CMD)
|
||||
@@ -19,5 +19,14 @@ test: ## Run all Go tests
|
||||
migrate: ## Apply pending photos_* migrations
|
||||
go run $(CMD) migrate
|
||||
|
||||
sync-to-s3: ## Copy the photo library from local disk to S3 (needs both in .env)
|
||||
go run $(CMD) sync to-s3
|
||||
|
||||
sync-to-local: ## Copy the photo library from S3 back to local disk
|
||||
go run $(CMD) sync to-local
|
||||
|
||||
backfill-checksums: ## Hash photos uploaded before duplicate detection existed
|
||||
go run $(CMD) backfill-checksums
|
||||
|
||||
clean: ## Remove built binary
|
||||
rm -rf bin
|
||||
|
||||
+1
-1
@@ -131,7 +131,7 @@ All routes under `/api/photos`. Auth = `Authorization: Bearer <existing JWT>`. "
|
||||
| `GET /api/photos/galleries/:id` | Full gallery detail incl. photos in position order with per-variant URLs and processing status. | `{ gallery, photos: [...] }` |
|
||||
| `PATCH /api/photos/galleries/:id` | Update title/description/eventId/visibility/coverPhotoId. | `{ gallery }` |
|
||||
| `DELETE /api/photos/galleries/:id` | Delete gallery + photos + stored objects. | `{ message }` |
|
||||
| `POST /api/photos/galleries/:id/photos` | Multipart upload, field `files` (repeatable). Sniffs magic bytes (JPEG/PNG/WebP/GIF/AVIF; explicit `415 { error: "HEIC is not supported, please upload JPEG" }` for HEIC). Stores original, inserts row `status='queued'`, appends at end position. | `201 { photos: [...] }` |
|
||||
| `POST /api/photos/galleries/:id/photos` | Multipart upload, field `files` (repeatable). Sniffs magic bytes (JPEG/PNG/WebP/GIF/AVIF; explicit `415 { error: "HEIC is not supported, please upload JPEG" }` for HEIC). Stores original, inserts row `status='queued'`, appends at end position. Hashes the bytes into `checksum`: if the gallery already holds them, nothing is stored and the existing photo is echoed with `duplicate: true` (the batch continues, no position consumed). | `201 { photos: [...] }` |
|
||||
| `PATCH /api/photos/galleries/:id/order` | Body `{ photoIds: [uuid,...] }` — full ordering; positions rewritten in one transaction. | `{ message }` |
|
||||
| `DELETE /api/photos/photos/:photoId` | Delete one photo + its objects; compacts positions. | `{ message }` |
|
||||
| `POST /api/photos/galleries/:id/share-token` | Rotate share token (invalidate old links). | `{ gallery }` |
|
||||
|
||||
+81
-4
@@ -14,9 +14,13 @@ Design/decisions: [PLAN.md](./PLAN.md).
|
||||
- Originals are stored byte-identical for download; a worker generates JPEG
|
||||
variants (thumb 512px q78, preview 2048px q85, EXIF stripped/orientation
|
||||
applied) queued in the DB with retries.
|
||||
- Uploads are deduplicated per gallery: see
|
||||
[Duplicate detection](#duplicate-detection).
|
||||
- Storage: local disk (`STORAGE_PATH`) or S3/Garage/MinIO (set `S3_ENDPOINT`
|
||||
+ `S3_BUCKET`), same selection convention as the backend. S3 downloads use
|
||||
short-lived presigned URLs; nothing in the bucket is public.
|
||||
+ `S3_BUCKET`), same selection convention as the backend, overridable with
|
||||
`STORAGE_BACKEND`. S3 downloads use short-lived presigned URLs; nothing in
|
||||
the bucket is public. Switching backends later: see
|
||||
[Move the library between backends](#move-the-library-between-backends).
|
||||
- Auth: validates the backend's HS256 JWTs with the shared `JWT_SECRET`
|
||||
(issuer `spanglish`, audience `spanglish-app`) including the DB-backed
|
||||
tokenVersion/account-status revocation check. Admin surface is
|
||||
@@ -37,7 +41,8 @@ go test ./... # SQLite; add PHOTO_TEST_PG=<url> to also run on Post
|
||||
```
|
||||
|
||||
From the repo root: `npm run dev:photos`, `npm run build:photos`,
|
||||
`npm run test:photos`. The Next dev server rewrites `/api/photos/*` to
|
||||
`npm run test:photos`, `npm run migrate:photos`, `npm run sync:photos`,
|
||||
`npm run backfill:photos:checksums`. The Next dev server rewrites `/api/photos/*` to
|
||||
`PHOTO_API_URL` (default `http://localhost:3003`), so the frontend needs no
|
||||
extra config in dev.
|
||||
|
||||
@@ -45,6 +50,78 @@ HEIC uploads require a converter CLI on the host: `apt install libvips-tools`
|
||||
(or `libheif-examples`). Without one, HEIC uploads are rejected with a clear
|
||||
message and a startup warning is logged.
|
||||
|
||||
## Move the library between backends
|
||||
|
||||
`photo-api sync` copies the whole photo library one way between local disk and
|
||||
S3, so local↔S3 is a config switch rather than a migration project. **Both
|
||||
backends must be configured in `photo-api/.env`** (`STORAGE_PATH` *and* the
|
||||
`S3_*` values); `STORAGE_BACKEND` decides which one actually serves requests,
|
||||
so filling in S3 does not switch anything by itself.
|
||||
|
||||
Local disk → S3:
|
||||
|
||||
```bash
|
||||
npm run sync:photos -- to-s3 --dry-run # see what would be copied
|
||||
npm run sync:photos:to-s3 # copy it
|
||||
# then set STORAGE_BACKEND=s3 in photo-api/.env and restart the service
|
||||
```
|
||||
|
||||
S3 → local disk is the same with `to-local` / `STORAGE_BACKEND=local`
|
||||
(`npm run sync:photos:to-local`). From `photo-api/`: `make sync-to-s3`,
|
||||
`make sync-to-local`.
|
||||
|
||||
Flags (`npm run sync:photos -- to-s3 --overwrite`, or after the direction on
|
||||
the direct scripts): `--dry-run`, `--overwrite` (re-copy objects already
|
||||
present with the same size), `--concurrency=N` (default 4), `--gallery=<id>`.
|
||||
|
||||
How it behaves:
|
||||
|
||||
- The `photos_photos` rows are the inventory — for each photo the original
|
||||
plus, once processed, the thumb and preview. Objects with no row (worker
|
||||
scratch files, leftovers of deleted galleries) are not copied.
|
||||
- Only the destination is written. The source stays as a fallback; delete it
|
||||
by hand once the switch is verified.
|
||||
- Reruns are cheap and safe: objects already on the destination with the same
|
||||
size are skipped, so an interrupted or partly failed sync just needs
|
||||
rerunning. A failed object is logged and the exit code is non-zero.
|
||||
- Keys are identical on both backends, so nothing in the database changes and
|
||||
no re-processing is triggered.
|
||||
- Photos uploaded *after* the copy but *before* the restart land on the old
|
||||
backend. For a clean cutover, stop the service, sync, flip
|
||||
`STORAGE_BACKEND`, start again — or sync a second time after the switch to
|
||||
pick up stragglers.
|
||||
|
||||
## Duplicate detection
|
||||
|
||||
Every upload is hashed (sha256 of the original bytes) into
|
||||
`photos_photos.checksum`, unique per `(gallery_id, checksum)`. If a gallery
|
||||
already holds those exact bytes, the incoming copy is **discarded**: no object
|
||||
is stored, no row is inserted, and the upload response echoes the existing
|
||||
photo with `"duplicate": true`. The rest of the batch continues normally — a
|
||||
duplicate is not an error and does not consume a position.
|
||||
|
||||
Scope is one gallery. The same image can still live in several galleries, each
|
||||
with its own row and its own stored object, so deleting a gallery never orphans
|
||||
another one's photos.
|
||||
|
||||
The admin uploader panel shows those rows as *"Already in this gallery"*; no
|
||||
second tile appears in the grid.
|
||||
|
||||
Photos uploaded before this existed have no checksum, so they are not matched
|
||||
until hashed once:
|
||||
|
||||
```bash
|
||||
npm run backfill:photos:checksums -- --dry-run # what would be hashed
|
||||
npm run backfill:photos:checksums # hash it
|
||||
```
|
||||
|
||||
From `photo-api/`: `make backfill-checksums`. Flags: `--dry-run`,
|
||||
`--concurrency=N` (default 4), `--gallery=<id>`. It reads from the active
|
||||
`STORAGE_BACKEND`, only ever writes the checksum column, and is idempotent —
|
||||
rerun it after a sync or a restore. Photos whose content already matches an
|
||||
earlier one in the same gallery are **reported and left unhashed**; the command
|
||||
never deletes anything, so removing the extras is an admin's call.
|
||||
|
||||
## API
|
||||
|
||||
Everything under `/api/photos`. Errors are `{"error": string}`.
|
||||
@@ -58,7 +135,7 @@ Admin (Bearer token, role admin/organizer):
|
||||
| GET | `/api/photos/galleries/:id` | gallery + photos (all statuses) |
|
||||
| PATCH | `/api/photos/galleries/:id` | update title/visibility/event/cover |
|
||||
| DELETE | `/api/photos/galleries/:id` | delete gallery + objects |
|
||||
| POST | `/api/photos/galleries/:id/photos` | multipart upload (`files`) |
|
||||
| POST | `/api/photos/galleries/:id/photos` | multipart upload (`files`), deduplicated |
|
||||
| PATCH | `/api/photos/galleries/:id/order` | reorder (`{photoIds}`) |
|
||||
| POST | `/api/photos/galleries/:id/share-token` | rotate share token |
|
||||
| DELETE | `/api/photos/photos/:photoId` | delete photo |
|
||||
|
||||
+135
-15
@@ -1,12 +1,15 @@
|
||||
// photo-api serves event photo galleries for the Spanglish platform.
|
||||
//
|
||||
// photo-api start the HTTP server
|
||||
// photo-api migrate apply pending photos_* migrations and exit
|
||||
// photo-api start the HTTP server
|
||||
// photo-api migrate apply pending photos_* migrations and exit
|
||||
// photo-api sync to-s3|to-local copy the photo library between backends
|
||||
// photo-api backfill-checksums hash photos uploaded before duplicate detection
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -16,9 +19,11 @@ import (
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/httpapi"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/photosync"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/worker"
|
||||
@@ -38,15 +43,21 @@ func main() {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "migrate" {
|
||||
if err := db.Migrate(context.Background()); err != nil {
|
||||
log.Fatalf("migrate: %v", err)
|
||||
}
|
||||
log.Println("migrations up to date")
|
||||
return
|
||||
}
|
||||
if len(os.Args) > 1 {
|
||||
log.Fatalf("unknown subcommand %q (expected: migrate)", os.Args[1])
|
||||
switch os.Args[1] {
|
||||
case "migrate":
|
||||
if err := db.Migrate(context.Background()); err != nil {
|
||||
log.Fatalf("migrate: %v", err)
|
||||
}
|
||||
log.Println("migrations up to date")
|
||||
case "sync":
|
||||
runSync(cfg, db, os.Args[2:])
|
||||
case "backfill-checksums":
|
||||
runBackfillChecksums(cfg, db, os.Args[2:])
|
||||
default:
|
||||
log.Fatalf("unknown subcommand %q (expected: migrate, sync, backfill-checksums)", os.Args[1])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
@@ -64,11 +75,7 @@ func main() {
|
||||
if err != nil {
|
||||
log.Fatalf("storage: %v", err)
|
||||
}
|
||||
if cfg.S3Enabled() {
|
||||
log.Printf("storage: S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint)
|
||||
} else {
|
||||
log.Printf("storage: local disk at %s", cfg.StoragePath)
|
||||
}
|
||||
log.Printf("storage: %s (STORAGE_BACKEND=%s)", storage.Name(cfg, st), cfg.StorageBackend)
|
||||
|
||||
heic := imaging.DetectHeicConverter(cfg.HeicConverter)
|
||||
if heic == nil {
|
||||
@@ -104,3 +111,116 @@ func main() {
|
||||
log.Printf("shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
const syncUsage = `usage: photo-api sync to-s3|to-local [flags]
|
||||
|
||||
Copies every object of every photo (original, thumb, preview) from one storage
|
||||
backend to the other. Both must be configured in photo-api/.env; the source is
|
||||
never modified, and reruns skip objects already present on the destination.
|
||||
|
||||
Flags:
|
||||
`
|
||||
|
||||
// runSync handles `photo-api sync <direction> [flags]`, the storage migration
|
||||
// used when moving the library between local disk and S3.
|
||||
func runSync(cfg config.Config, db *store.DB, args []string) {
|
||||
fs := flag.NewFlagSet("sync", flag.ExitOnError)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprint(fs.Output(), syncUsage)
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
var (
|
||||
dryRun = fs.Bool("dry-run", false, "report what would be copied without writing")
|
||||
overwrite = fs.Bool("overwrite", false, "re-copy objects already present with the same size")
|
||||
workers = fs.Int("concurrency", 4, "objects copied in parallel")
|
||||
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
||||
)
|
||||
if len(args) == 0 {
|
||||
fs.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
direction, err := photosync.ParseDirection(args[0])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%v\n\n", err)
|
||||
fs.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if _, err := photosync.Run(ctx, cfg, db, photosync.Options{
|
||||
Direction: direction,
|
||||
GalleryID: *gallery,
|
||||
Concurrency: *workers,
|
||||
DryRun: *dryRun,
|
||||
Overwrite: *overwrite,
|
||||
}); err != nil {
|
||||
log.Fatalf("sync: %v", err)
|
||||
}
|
||||
if *dryRun {
|
||||
return
|
||||
}
|
||||
target := "s3"
|
||||
if direction == photosync.ToLocal {
|
||||
target = "local"
|
||||
}
|
||||
log.Printf("sync: set STORAGE_BACKEND=%s in photo-api/.env and restart the service to serve from it "+
|
||||
"(the source copy is left in place — delete it once the switch is verified)", target)
|
||||
}
|
||||
|
||||
const backfillUsage = `usage: photo-api backfill-checksums [flags]
|
||||
|
||||
Hashes the stored original of every photo that has no checksum yet, so uploads
|
||||
of a photo already in a gallery are recognised as duplicates. Runs against the
|
||||
active storage backend (STORAGE_BACKEND) and only ever writes the checksum
|
||||
column — no photo is deleted. Photos whose content already matches an earlier
|
||||
one in the same gallery are reported and left unhashed.
|
||||
|
||||
Flags:
|
||||
`
|
||||
|
||||
// runBackfillChecksums handles `photo-api backfill-checksums [flags]`, the
|
||||
// one-off pass needed after the checksum column is added to an existing
|
||||
// library.
|
||||
func runBackfillChecksums(cfg config.Config, db *store.DB, args []string) {
|
||||
fs := flag.NewFlagSet("backfill-checksums", flag.ExitOnError)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprint(fs.Output(), backfillUsage)
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
var (
|
||||
dryRun = fs.Bool("dry-run", false, "report what would be hashed without writing")
|
||||
workers = fs.Int("concurrency", 4, "originals hashed in parallel")
|
||||
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
||||
)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
st, err := storage.New(cfg)
|
||||
if err != nil {
|
||||
log.Fatalf("storage: %v", err)
|
||||
}
|
||||
log.Printf("backfill-checksums: reading from %s", storage.Name(cfg, st))
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
res, err := checksum.Backfill(ctx, db, st, checksum.Options{
|
||||
GalleryID: *gallery,
|
||||
Concurrency: *workers,
|
||||
DryRun: *dryRun,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("backfill-checksums: %v", err)
|
||||
}
|
||||
if res.Duplicates > 0 {
|
||||
log.Printf("backfill-checksums: %d existing photos duplicate an earlier one in their gallery "+
|
||||
"(listed above); they still show in the gallery — delete the unwanted ones from the admin page",
|
||||
res.Duplicates)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package checksum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// Options configures a backfill run.
|
||||
type Options struct {
|
||||
// GalleryID limits the run to one gallery; empty means the whole library.
|
||||
GalleryID string
|
||||
// Concurrency is how many originals are hashed at a time.
|
||||
Concurrency int
|
||||
// DryRun reports what would be hashed without writing to the database.
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// Result counts what a run did. Duplicates are rows whose bytes match an
|
||||
// earlier photo in the same gallery; they keep a NULL checksum and are listed
|
||||
// so an admin can decide what to do with them.
|
||||
type Result struct {
|
||||
Total int // rows without a checksum
|
||||
Hashed int // hashed and recorded (or, with DryRun, would be)
|
||||
Duplicates int
|
||||
Missing int // original object absent from storage
|
||||
Failed int
|
||||
}
|
||||
|
||||
// Backfill hashes the stored original of every photo that has no checksum yet,
|
||||
// so duplicate detection also catches re-uploads of photos from before the
|
||||
// checksum column existed. It only ever writes the checksum column; no photo,
|
||||
// row or object is deleted.
|
||||
//
|
||||
// It is idempotent: rerunning it finds only what the previous run left behind.
|
||||
func Backfill(ctx context.Context, db *store.DB, st storage.Storage, opts Options) (Result, error) {
|
||||
if opts.Concurrency < 1 {
|
||||
opts.Concurrency = 4
|
||||
}
|
||||
photos, err := db.PhotosMissingChecksum(ctx, opts.GalleryID)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("list photos without a checksum: %w", err)
|
||||
}
|
||||
|
||||
prefix := ""
|
||||
if opts.DryRun {
|
||||
prefix = "[dry-run] "
|
||||
}
|
||||
log.Printf("backfill-checksums: %s%d photos to hash, concurrency %d", prefix, len(photos), opts.Concurrency)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
res = Result{Total: len(photos)}
|
||||
done int64
|
||||
jobs = make(chan store.Photo)
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
// Sequential hashing per worker, but the database update is what can
|
||||
// collide: two photos of the same gallery with identical bytes race, and
|
||||
// the unique index decides which one keeps the checksum.
|
||||
for i := 0; i < opts.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for p := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
sum, err := hashOriginal(ctx, st, p)
|
||||
if err == nil && !opts.DryRun {
|
||||
err = db.SetPhotoChecksum(ctx, p.ID, sum)
|
||||
}
|
||||
n := atomic.AddInt64(&done, 1)
|
||||
|
||||
mu.Lock()
|
||||
switch {
|
||||
case errors.Is(err, storage.ErrNotExist):
|
||||
res.Missing++
|
||||
log.Printf("backfill-checksums: [%d/%d] MISSING object, skipped: %s (photo %s)",
|
||||
n, len(photos), p.OriginalKey, p.ID)
|
||||
case store.IsUniqueViolation(err):
|
||||
res.Duplicates++
|
||||
other := "an earlier photo"
|
||||
if existing, findErr := db.FindPhotoByChecksum(ctx, p.GalleryID, sum); findErr == nil {
|
||||
other = "photo " + existing.ID
|
||||
}
|
||||
log.Printf("backfill-checksums: [%d/%d] DUPLICATE: photo %s (%s) has the same content as %s "+
|
||||
"in gallery %s — left without a checksum, delete it by hand if unwanted",
|
||||
n, len(photos), p.ID, p.OriginalFilename, other, p.GalleryID)
|
||||
case err != nil:
|
||||
res.Failed++
|
||||
log.Printf("backfill-checksums: [%d/%d] FAILED %s (photo %s): %v",
|
||||
n, len(photos), p.OriginalKey, p.ID, err)
|
||||
default:
|
||||
res.Hashed++
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, p := range photos {
|
||||
select {
|
||||
case jobs <- p:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
|
||||
log.Printf("backfill-checksums: %sdone — %d hashed, %d duplicates left unhashed, %d objects missing, %d failed",
|
||||
prefix, res.Hashed, res.Duplicates, res.Missing, res.Failed)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, fmt.Errorf("interrupted after %d/%d photos: %w", res.Hashed, res.Total, err)
|
||||
}
|
||||
if res.Failed > 0 {
|
||||
return res, fmt.Errorf("%d of %d photos failed to hash (rerun to retry; already-hashed photos are skipped)",
|
||||
res.Failed, res.Total)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// hashOriginal reads a photo's stored original. Stat runs first because it is
|
||||
// the only call that reports a missing object as storage.ErrNotExist on both
|
||||
// backends — Open surfaces the driver's own error.
|
||||
func hashOriginal(ctx context.Context, st storage.Storage, p store.Photo) (string, error) {
|
||||
if _, err := st.Stat(ctx, p.OriginalKey); err != nil {
|
||||
return "", err
|
||||
}
|
||||
r, _, err := st.Open(ctx, p.OriginalKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer r.Close()
|
||||
return Sum(r)
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package checksum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
const galleryID = "11111111-1111-1111-1111-111111111111"
|
||||
|
||||
// env builds a migrated scratch SQLite database with one gallery, plus a local
|
||||
// storage backend rooted next to it.
|
||||
func env(t *testing.T) (*store.DB, storage.Storage) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
db, err := store.Open(config.Config{DBType: "sqlite", DatabaseURL: filepath.Join(dir, "test.db")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
ctx := context.Background()
|
||||
if err := db.Migrate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
if err := db.CreateGallery(ctx, store.Gallery{
|
||||
ID: galleryID, Slug: "g", Title: "G", Visibility: "private", ShareToken: "tok",
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := storage.NewLocal(filepath.Join(dir, "photos"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, st
|
||||
}
|
||||
|
||||
// seed inserts an unhashed photo whose original holds body, mimicking a row
|
||||
// uploaded before the checksum column existed.
|
||||
func seed(t *testing.T, db *store.DB, st storage.Storage, id, body string) store.Photo {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
key := "galleries/" + galleryID + "/orig/" + id + ".jpg"
|
||||
if body != "" {
|
||||
if err := st.Put(ctx, key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
p := store.Photo{
|
||||
ID: id, GalleryID: galleryID, OriginalKey: key, OriginalFilename: id + ".jpg",
|
||||
ContentType: "image/jpeg", SizeBytes: int64(len(body)), Status: "ready",
|
||||
NextAttemptAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}
|
||||
if err := db.InsertPhoto(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func checksumOf(t *testing.T, db *store.DB, id string) string {
|
||||
t.Helper()
|
||||
p, err := db.GetPhoto(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p.Checksum
|
||||
}
|
||||
|
||||
func TestBackfill(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("hashes unhashed photos and is idempotent", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
seed(t, db, st, "p2", "beta")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 2 || res.Hashed != 2 || res.Duplicates != 0 || res.Failed != 0 {
|
||||
t.Fatalf("first run: %+v", res)
|
||||
}
|
||||
want, err := Sum(strings.NewReader("alpha"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := checksumOf(t, db, "p1"); got != want {
|
||||
t.Fatalf("p1 checksum = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
res, err = Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 0 || res.Hashed != 0 {
|
||||
t.Fatalf("rerun should find nothing to do: %+v", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reports pre-existing duplicates and leaves them unhashed", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "same")
|
||||
seed(t, db, st, "p2", "same")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{Concurrency: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Hashed != 1 || res.Duplicates != 1 || res.Failed != 0 {
|
||||
t.Fatalf("got %+v, want 1 hashed and 1 duplicate", res)
|
||||
}
|
||||
// The earlier row keeps the checksum, the later one is left alone —
|
||||
// nothing is deleted either way.
|
||||
if checksumOf(t, db, "p1") == "" {
|
||||
t.Fatal("p1 should have been hashed")
|
||||
}
|
||||
if got := checksumOf(t, db, "p2"); got != "" {
|
||||
t.Fatalf("p2 checksum = %q, want empty", got)
|
||||
}
|
||||
if _, err := db.GetPhoto(ctx, "p2"); err != nil {
|
||||
t.Fatalf("duplicate row must survive: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips photos whose object is gone", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "") // row without a stored original
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Missing != 1 || res.Hashed != 0 || res.Failed != 0 {
|
||||
t.Fatalf("got %+v, want 1 missing", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dry run writes nothing", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Hashed != 1 {
|
||||
t.Fatalf("got %+v, want 1 hashed", res)
|
||||
}
|
||||
if got := checksumOf(t, db, "p1"); got != "" {
|
||||
t.Fatalf("dry run recorded %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("limits to one gallery", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{GalleryID: "22222222-2222-2222-2222-222222222222"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 0 {
|
||||
t.Fatalf("other gallery should have nothing to do: %+v", res)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Package checksum defines the content hash used to spot duplicate photos and
|
||||
// the backfill that fills it in for photos uploaded before it existed.
|
||||
//
|
||||
// The hash is sha256 over the untouched original bytes, hex encoded, stored in
|
||||
// photos_photos.checksum. Duplicate scope is one gallery — the unique index is
|
||||
// on (gallery_id, checksum) — so the same image may still live in several
|
||||
// galleries, each with its own row and its own stored object.
|
||||
package checksum
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"hash"
|
||||
"io"
|
||||
)
|
||||
|
||||
// New returns a fresh hasher. Upload hashes as it spools the body, so it needs
|
||||
// the writer rather than a finished reader.
|
||||
func New() hash.Hash { return sha256.New() }
|
||||
|
||||
// Format renders a hasher's digest the way it is stored.
|
||||
func Format(h hash.Hash) string { return hex.EncodeToString(h.Sum(nil)) }
|
||||
|
||||
// Sum reads r to EOF and returns its digest.
|
||||
func Sum(r io.Reader) (string, error) {
|
||||
h := New()
|
||||
if _, err := io.Copy(h, r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return Format(h), nil
|
||||
}
|
||||
@@ -20,6 +20,11 @@ type Config struct {
|
||||
// JWT_SECRET during the Better Auth migration.
|
||||
ViewTokenSecret string
|
||||
|
||||
// StorageBackend selects the active backend: "auto" (S3 when it is
|
||||
// configured, local otherwise), "local" or "s3". Explicit values let both
|
||||
// backends stay configured — required to run `photo-api sync`, and the
|
||||
// one-line switch after a sync.
|
||||
StorageBackend string
|
||||
StoragePath string
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
@@ -34,12 +39,26 @@ type Config struct {
|
||||
HeicConverter string // optional explicit converter command; autodetected when empty
|
||||
}
|
||||
|
||||
// S3Enabled mirrors backend/src/lib/storage.ts: S3 is active when both
|
||||
// S3_ENDPOINT and S3_BUCKET are set.
|
||||
func (c Config) S3Enabled() bool {
|
||||
// S3Configured reports whether the S3 credentials are present at all,
|
||||
// mirroring backend/src/lib/storage.ts: both S3_ENDPOINT and S3_BUCKET set.
|
||||
// Kept separate from S3Enabled so a configured-but-inactive S3 backend can
|
||||
// still be reached by `photo-api sync`.
|
||||
func (c Config) S3Configured() bool {
|
||||
return c.S3Endpoint != "" && c.S3Bucket != ""
|
||||
}
|
||||
|
||||
// S3Enabled reports whether S3 is the backend serving requests.
|
||||
func (c Config) S3Enabled() bool {
|
||||
switch c.StorageBackend {
|
||||
case "s3":
|
||||
return true
|
||||
case "local":
|
||||
return false
|
||||
default: // auto
|
||||
return c.S3Configured()
|
||||
}
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
loadDotenv(".env")
|
||||
|
||||
@@ -48,6 +67,7 @@ func Load() (Config, error) {
|
||||
DBType: strings.ToLower(env("DB_TYPE", "sqlite")),
|
||||
DatabaseURL: env("DATABASE_URL", ""),
|
||||
ViewTokenSecret: env("PHOTO_VIEW_SECRET", env("JWT_SECRET", "")),
|
||||
StorageBackend: strings.ToLower(env("STORAGE_BACKEND", "auto")),
|
||||
StoragePath: env("STORAGE_PATH", "./data/photos"),
|
||||
S3Endpoint: env("S3_ENDPOINT", ""),
|
||||
S3Region: env("S3_REGION", "auto"),
|
||||
@@ -70,6 +90,14 @@ func Load() (Config, error) {
|
||||
if cfg.ViewTokenSecret == "" {
|
||||
return cfg, fmt.Errorf("PHOTO_VIEW_SECRET is required (or legacy JWT_SECRET as fallback)")
|
||||
}
|
||||
switch cfg.StorageBackend {
|
||||
case "auto", "local", "s3":
|
||||
default:
|
||||
return cfg, fmt.Errorf("STORAGE_BACKEND must be auto, local or s3, got %q", cfg.StorageBackend)
|
||||
}
|
||||
if cfg.StorageBackend == "s3" && !cfg.S3Configured() {
|
||||
return cfg, fmt.Errorf("STORAGE_BACKEND=s3 requires S3_ENDPOINT and S3_BUCKET")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"image"
|
||||
"image/color"
|
||||
@@ -38,10 +39,11 @@ const (
|
||||
)
|
||||
|
||||
type testEnv struct {
|
||||
handler http.Handler
|
||||
db *store.DB
|
||||
worker *worker.Worker
|
||||
pg bool
|
||||
handler http.Handler
|
||||
db *store.DB
|
||||
worker *worker.Worker
|
||||
storagePath string
|
||||
pg bool
|
||||
}
|
||||
|
||||
// setup migrates a scratch DB (SQLite by default; Postgres when
|
||||
@@ -123,7 +125,13 @@ func setup(t *testing.T) *testEnv {
|
||||
}
|
||||
wrk := worker.New(db, st, nil, cfg.StoragePath, 1)
|
||||
srv := New(cfg, db, st, auth.NewVerifier(db), wrk)
|
||||
return &testEnv{handler: srv.Handler(), db: db, worker: wrk, pg: cfg.DBType == "postgres"}
|
||||
return &testEnv{
|
||||
handler: srv.Handler(),
|
||||
db: db,
|
||||
worker: wrk,
|
||||
storagePath: cfg.StoragePath,
|
||||
pg: cfg.DBType == "postgres",
|
||||
}
|
||||
}
|
||||
|
||||
// makeToken inserts a Better Auth session row for the user and returns a
|
||||
@@ -175,12 +183,16 @@ func decode[T any](t *testing.T, w *httptest.ResponseRecorder) T {
|
||||
return v
|
||||
}
|
||||
|
||||
func testJPEG(t *testing.T) []byte {
|
||||
func testJPEG(t *testing.T) []byte { return testJPEGTinted(t, 128) }
|
||||
|
||||
// testJPEGTinted varies the blue channel so tests that need two *different*
|
||||
// images (duplicate detection) can get them without a fixture file.
|
||||
func testJPEGTinted(t *testing.T, blue uint8) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, 800, 600))
|
||||
for x := 0; x < 800; x += 10 {
|
||||
for y := 0; y < 600; y++ {
|
||||
img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: 128, A: 255})
|
||||
img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: blue, A: 255})
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
@@ -296,6 +308,130 @@ func uploadPhoto(t *testing.T, e *testEnv, admin, galleryID string, file []byte)
|
||||
return photos[0]
|
||||
}
|
||||
|
||||
// uploadFiles posts several parts in one request, the way the API allows even
|
||||
// though the admin UI sends one file per request.
|
||||
func uploadFiles(t *testing.T, e *testEnv, admin, galleryID string, files ...[]byte) []photoJSON {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
mw := multipart.NewWriter(&buf)
|
||||
for i, f := range files {
|
||||
fw, _ := mw.CreateFormFile("files", fmt.Sprintf("photo-%d.jpg", i))
|
||||
fw.Write(f)
|
||||
}
|
||||
mw.Close()
|
||||
req := httptest.NewRequest("POST", "/api/photos/galleries/"+galleryID+"/photos", &buf)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Bearer "+admin)
|
||||
w := httptest.NewRecorder()
|
||||
e.handler.ServeHTTP(w, req)
|
||||
if w.Code != 201 {
|
||||
t.Fatalf("upload: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
return decode[struct {
|
||||
Photos []photoJSON `json:"photos"`
|
||||
}](t, w).Photos
|
||||
}
|
||||
|
||||
// countOriginals is how many objects actually landed on disk for a gallery —
|
||||
// a duplicate must not add one.
|
||||
func countOriginals(t *testing.T, e *testEnv, galleryID string) int {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(filepath.Join(e.storagePath, "galleries", galleryID, "orig"))
|
||||
if os.IsNotExist(err) {
|
||||
return 0
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(entries)
|
||||
}
|
||||
|
||||
func TestUploadSkipsDuplicates(t *testing.T) {
|
||||
e := setup(t)
|
||||
admin := e.makeToken(t, uAdmin)
|
||||
g := createGallery(t, e, admin, map[string]any{"title": "Dupes"})
|
||||
img := testJPEG(t)
|
||||
|
||||
first := uploadPhoto(t, e, admin, g.ID, img)
|
||||
if first.Duplicate {
|
||||
t.Fatalf("first upload flagged as duplicate: %+v", first)
|
||||
}
|
||||
|
||||
// Same bytes again: echoed back as the existing photo, nothing stored.
|
||||
second := uploadPhoto(t, e, admin, g.ID, img)
|
||||
if !second.Duplicate {
|
||||
t.Fatalf("second upload not flagged as duplicate: %+v", second)
|
||||
}
|
||||
if second.ID != first.ID {
|
||||
t.Fatalf("duplicate should echo the existing photo: got %s, want %s", second.ID, first.ID)
|
||||
}
|
||||
photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(photos) != 1 {
|
||||
t.Fatalf("want 1 row after re-upload, got %d", len(photos))
|
||||
}
|
||||
if n := countOriginals(t, e, g.ID); n != 1 {
|
||||
t.Fatalf("want 1 stored original after re-upload, got %d", n)
|
||||
}
|
||||
if photos[0].Checksum == "" {
|
||||
t.Fatal("checksum was not recorded")
|
||||
}
|
||||
|
||||
// A different image is unaffected and takes the next position.
|
||||
other := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))
|
||||
if other.Duplicate || other.ID == first.ID {
|
||||
t.Fatalf("distinct image treated as duplicate: %+v", other)
|
||||
}
|
||||
if other.Position != 1 {
|
||||
t.Fatalf("want position 1 for the second distinct photo, got %d", other.Position)
|
||||
}
|
||||
|
||||
// Duplicate scope is one gallery: the same bytes elsewhere upload normally.
|
||||
g2 := createGallery(t, e, admin, map[string]any{"title": "Other gallery"})
|
||||
elsewhere := uploadPhoto(t, e, admin, g2.ID, img)
|
||||
if elsewhere.Duplicate {
|
||||
t.Fatalf("same image in another gallery must not be a duplicate: %+v", elsewhere)
|
||||
}
|
||||
if elsewhere.ID == first.ID {
|
||||
t.Fatal("second gallery should get its own photo row")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadBatchContinuesPastDuplicate(t *testing.T) {
|
||||
e := setup(t)
|
||||
admin := e.makeToken(t, uAdmin)
|
||||
g := createGallery(t, e, admin, map[string]any{"title": "Batch"})
|
||||
a, b, c := testJPEG(t), testJPEGTinted(t, 32), testJPEGTinted(t, 200)
|
||||
|
||||
if got := uploadFiles(t, e, admin, g.ID, a); len(got) != 1 {
|
||||
t.Fatalf("seed upload: %d photos", len(got))
|
||||
}
|
||||
// b duplicates nothing, a is already there, c is new: the batch must not
|
||||
// abort, and the duplicate must not consume a position.
|
||||
batch := uploadFiles(t, e, admin, g.ID, b, a, c)
|
||||
if len(batch) != 3 {
|
||||
t.Fatalf("want 3 results, got %d", len(batch))
|
||||
}
|
||||
if batch[0].Duplicate || !batch[1].Duplicate || batch[2].Duplicate {
|
||||
t.Fatalf("duplicate flags: %v %v %v", batch[0].Duplicate, batch[1].Duplicate, batch[2].Duplicate)
|
||||
}
|
||||
if batch[0].Position != 1 || batch[2].Position != 2 {
|
||||
t.Fatalf("positions should stay dense: %d, %d", batch[0].Position, batch[2].Position)
|
||||
}
|
||||
photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(photos) != 3 {
|
||||
t.Fatalf("want 3 rows, got %d", len(photos))
|
||||
}
|
||||
if n := countOriginals(t, e, g.ID); n != 3 {
|
||||
t.Fatalf("want 3 stored originals, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// processQueue runs the worker until the photo is ready or failed.
|
||||
func processQueue(t *testing.T, e *testEnv, photoID string) store.Photo {
|
||||
t.Helper()
|
||||
@@ -465,9 +601,9 @@ func TestReorderAndVisibilityUpdate(t *testing.T) {
|
||||
e := setup(t)
|
||||
admin := e.makeToken(t, uAdmin)
|
||||
g := createGallery(t, e, admin, map[string]any{"title": "Orden", "visibility": "public"})
|
||||
jpg := testJPEG(t)
|
||||
p1 := uploadPhoto(t, e, admin, g.ID, jpg)
|
||||
p2 := uploadPhoto(t, e, admin, g.ID, jpg)
|
||||
// Two distinct images: the same bytes twice would be deduplicated.
|
||||
p1 := uploadPhoto(t, e, admin, g.ID, testJPEG(t))
|
||||
p2 := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))
|
||||
|
||||
if w := e.request(t, "PATCH", "/api/photos/galleries/"+g.ID+"/order", admin,
|
||||
map[string]any{"photoIds": []string{p2.ID, p1.ID}}); w.Code != 200 {
|
||||
|
||||
@@ -58,6 +58,10 @@ type photoJSON struct {
|
||||
LastError string `json:"lastError,omitempty"` // admin only
|
||||
CreatedAt string `json:"createdAt"`
|
||||
URLs photoURLs `json:"urls"`
|
||||
// Duplicate marks an upload that was skipped because the gallery already
|
||||
// held these bytes; the rest of the object describes the existing photo.
|
||||
// Set by the upload handler only, never persisted.
|
||||
Duplicate bool `json:"duplicate,omitempty"`
|
||||
}
|
||||
|
||||
// viewTokenFor returns the token to embed in a gallery's file URLs: none
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
@@ -20,6 +21,10 @@ import (
|
||||
// Each file is sniffed by magic bytes (client filename/Content-Type are
|
||||
// untrusted, same policy as /api/media/upload), stored as the untouched
|
||||
// original, and queued for variant processing.
|
||||
//
|
||||
// A file whose bytes are already in this gallery is not stored a second time:
|
||||
// the incoming copy is discarded, the existing photo is echoed back with
|
||||
// "duplicate": true, and the rest of the batch carries on.
|
||||
func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.User) {
|
||||
galleryID := r.PathValue("id")
|
||||
g, err := s.db.GetGallery(r.Context(), galleryID)
|
||||
@@ -57,7 +62,7 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
|
||||
part.Close()
|
||||
continue
|
||||
}
|
||||
photo, uploadErr := s.saveUpload(r, g, part, position, maxFile)
|
||||
photo, duplicate, uploadErr := s.saveUpload(r, g, part, position, maxFile)
|
||||
part.Close()
|
||||
if uploadErr != nil {
|
||||
// One bad file fails the request explicitly rather than silently
|
||||
@@ -65,7 +70,12 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
|
||||
writeError(w, uploadErr.status, uploadErr.msg)
|
||||
return
|
||||
}
|
||||
created = append(created, s.photoToJSON(photo, s.viewTokenFor(g), true))
|
||||
out := s.photoToJSON(photo, s.viewTokenFor(g), true)
|
||||
out.Duplicate = duplicate
|
||||
created = append(created, out)
|
||||
if duplicate {
|
||||
continue // nothing was stored, so the next file keeps this position
|
||||
}
|
||||
position++
|
||||
}
|
||||
|
||||
@@ -82,51 +92,67 @@ type uploadError struct {
|
||||
msg string
|
||||
}
|
||||
|
||||
func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, *uploadError) {
|
||||
// saveUpload stores one part. The bool it returns reports a duplicate: the
|
||||
// gallery already holds these bytes, so nothing was written and the photo
|
||||
// returned is the existing one.
|
||||
func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, bool, *uploadError) {
|
||||
head := make([]byte, 16)
|
||||
n, err := io.ReadFull(part, head)
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
return store.Photo{}, &uploadError{http.StatusBadRequest, "Could not read file"}
|
||||
return store.Photo{}, false, &uploadError{http.StatusBadRequest, "Could not read file"}
|
||||
}
|
||||
head = head[:n]
|
||||
contentType, ext, ok, reason := imaging.Sniff(head)
|
||||
if !ok {
|
||||
return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType, reason}
|
||||
return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType, reason}
|
||||
}
|
||||
if contentType == "image/heic" && imaging.DetectHeicConverter(s.cfg.HeicConverter) == nil {
|
||||
return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType,
|
||||
return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType,
|
||||
"HEIC uploads need an image converter on the server (install libvips-tools); please upload JPEG instead"}
|
||||
}
|
||||
|
||||
// Spool to a temp file to learn the size before handing to storage
|
||||
// (S3 wants a length; local rename wants a file anyway).
|
||||
// (S3 wants a length; local rename wants a file anyway). The same pass
|
||||
// hashes the bytes for the duplicate check below.
|
||||
tmp, err := os.CreateTemp(s.cfg.StoragePath, ".incoming-*")
|
||||
if err != nil {
|
||||
log.Printf("Error: %v", err)
|
||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
defer tmp.Close()
|
||||
|
||||
size, err := io.Copy(tmp, io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
|
||||
hasher := checksum.New()
|
||||
size, err := io.Copy(io.MultiWriter(tmp, hasher),
|
||||
io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
|
||||
if err != nil {
|
||||
log.Printf("Error: %v", err)
|
||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
}
|
||||
if size > maxFile {
|
||||
return store.Photo{}, &uploadError{http.StatusRequestEntityTooLarge,
|
||||
return store.Photo{}, false, &uploadError{http.StatusRequestEntityTooLarge,
|
||||
fmt.Sprintf("File exceeds the %d MB limit", s.cfg.MaxUploadMB)}
|
||||
}
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
log.Printf("Error: %v", err)
|
||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
}
|
||||
sum := checksum.Format(hasher)
|
||||
|
||||
// Duplicate check before anything is written: the temp copy is dropped by
|
||||
// the deferred Remove, no object is stored and no row is inserted.
|
||||
if existing, err := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); err == nil {
|
||||
return existing, true, nil
|
||||
} else if err != store.ErrNotFound {
|
||||
log.Printf("Error: %v", err)
|
||||
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
}
|
||||
|
||||
photoID := newID()
|
||||
key := fmt.Sprintf("galleries/%s/orig/%s%s", g.ID, photoID, ext)
|
||||
if err := s.storage.Put(r.Context(), key, tmp, size, contentType); err != nil {
|
||||
log.Printf("Error: %v", err)
|
||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
@@ -142,13 +168,22 @@ func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Pa
|
||||
NextAttemptAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
Checksum: sum,
|
||||
}
|
||||
if err := s.db.InsertPhoto(r.Context(), photo); err != nil {
|
||||
s.storage.Delete(r.Context(), key)
|
||||
// A concurrent upload of the same bytes won the race between the
|
||||
// check above and this insert; the unique index caught it, so report
|
||||
// the winner as the duplicate instead of failing.
|
||||
if store.IsUniqueViolation(err) {
|
||||
if existing, findErr := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); findErr == nil {
|
||||
return existing, true, nil
|
||||
}
|
||||
}
|
||||
log.Printf("Error: %v", err)
|
||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||
}
|
||||
return photo, nil
|
||||
return photo, false, nil
|
||||
}
|
||||
|
||||
type reorderBody struct {
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// Package photosync copies the photo library between the local-disk and the
|
||||
// S3 backend so the active one (STORAGE_BACKEND) can be switched without
|
||||
// losing photos. Both backends must be configured in photo-api/.env for a
|
||||
// sync to run — the direction picks which is the source.
|
||||
//
|
||||
// The photos_photos rows are the inventory: every row contributes its
|
||||
// original key and, once the worker has processed it, its thumb and preview
|
||||
// key. Objects on disk or in the bucket with no row (worker scratch dirs,
|
||||
// leftovers of deleted galleries) are deliberately not copied.
|
||||
//
|
||||
// Sync only ever writes to the destination. The source is left untouched, so
|
||||
// a sync is repeatable, safe to interrupt, and leaves the old backend as a
|
||||
// fallback until it is cleaned up by hand.
|
||||
package photosync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// Direction is which way objects move.
|
||||
type Direction string
|
||||
|
||||
const (
|
||||
ToS3 Direction = "to-s3"
|
||||
ToLocal Direction = "to-local"
|
||||
)
|
||||
|
||||
// ParseDirection accepts the short and the spelled-out form.
|
||||
func ParseDirection(s string) (Direction, error) {
|
||||
switch s {
|
||||
case "to-s3", "local-to-s3":
|
||||
return ToS3, nil
|
||||
case "to-local", "s3-to-local":
|
||||
return ToLocal, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown direction %q (expected to-s3 or to-local)", s)
|
||||
}
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
Direction Direction
|
||||
// GalleryID limits the sync to one gallery; empty means the whole library.
|
||||
GalleryID string
|
||||
// Concurrency is how many objects are copied at a time.
|
||||
Concurrency int
|
||||
// DryRun reports what would be copied without writing anything.
|
||||
DryRun bool
|
||||
// Overwrite re-copies objects that already exist on the destination with
|
||||
// the same size (default: those are skipped, which makes reruns cheap).
|
||||
Overwrite bool
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Total int // objects in the inventory
|
||||
Copied int // copied (or, with DryRun, would be copied)
|
||||
Skipped int // already on the destination
|
||||
Missing int // absent from the source — nothing to copy
|
||||
Failed int // copy attempted and errored
|
||||
Bytes int64 // bytes copied
|
||||
}
|
||||
|
||||
// Run copies the library in the requested direction. It returns a Result even
|
||||
// on error, and an error if any object failed.
|
||||
func Run(ctx context.Context, cfg config.Config, db *store.DB, opts Options) (Result, error) {
|
||||
if !cfg.S3Configured() {
|
||||
return Result{}, errors.New("sync needs both backends configured: set S3_ENDPOINT, S3_BUCKET, " +
|
||||
"S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY (next to STORAGE_PATH) in photo-api/.env")
|
||||
}
|
||||
if opts.Concurrency < 1 {
|
||||
opts.Concurrency = 4
|
||||
}
|
||||
|
||||
local, err := storage.NewLocal(cfg.StoragePath)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("local storage: %w", err)
|
||||
}
|
||||
s3, err := storage.NewS3(cfg)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("s3 storage: %w", err)
|
||||
}
|
||||
src, dst := local, s3
|
||||
srcName, dstName := "local disk "+cfg.StoragePath, "s3 bucket "+cfg.S3Bucket
|
||||
if opts.Direction == ToLocal {
|
||||
src, dst = s3, local
|
||||
srcName, dstName = dstName, srcName
|
||||
}
|
||||
|
||||
objects, err := db.AllPhotoObjects(ctx, opts.GalleryID)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("list photo objects: %w", err)
|
||||
}
|
||||
|
||||
prefix := ""
|
||||
if opts.DryRun {
|
||||
prefix = "[dry-run] "
|
||||
}
|
||||
log.Printf("sync: %s%s → %s: %d objects, concurrency %d",
|
||||
prefix, srcName, dstName, len(objects), opts.Concurrency)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
res = Result{Total: len(objects)}
|
||||
done int64
|
||||
jobs = make(chan store.PhotoObject)
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for i := 0; i < opts.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for o := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
act, size, err := copyObject(ctx, src, dst, o, opts)
|
||||
n := atomic.AddInt64(&done, 1)
|
||||
|
||||
mu.Lock()
|
||||
switch {
|
||||
case err != nil:
|
||||
res.Failed++
|
||||
log.Printf("sync: [%d/%d] FAILED %s (photo %s %s): %v",
|
||||
n, len(objects), o.Key, o.PhotoID, o.Variant, err)
|
||||
case act == actionCopied:
|
||||
res.Copied++
|
||||
res.Bytes += size
|
||||
log.Printf("sync: [%d/%d] %scopied %s (%s)", n, len(objects), prefix, o.Key, humanBytes(size))
|
||||
case act == actionMissing:
|
||||
res.Missing++
|
||||
log.Printf("sync: [%d/%d] MISSING on source, skipped: %s (photo %s %s)",
|
||||
n, len(objects), o.Key, o.PhotoID, o.Variant)
|
||||
default:
|
||||
res.Skipped++
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, o := range objects {
|
||||
select {
|
||||
case jobs <- o:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
|
||||
log.Printf("sync: %sdone — %d copied (%s), %d already present, %d missing on source, %d failed",
|
||||
prefix, res.Copied, humanBytes(res.Bytes), res.Skipped, res.Missing, res.Failed)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, fmt.Errorf("interrupted after %d/%d objects: %w", res.Copied+res.Skipped, res.Total, err)
|
||||
}
|
||||
if res.Failed > 0 {
|
||||
return res, fmt.Errorf("%d of %d objects failed to copy (rerun to retry; already-copied objects are skipped)",
|
||||
res.Failed, res.Total)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
type action int
|
||||
|
||||
const (
|
||||
actionCopied action = iota
|
||||
actionSkipped
|
||||
actionMissing
|
||||
)
|
||||
|
||||
func copyObject(ctx context.Context, src, dst storage.Storage, o store.PhotoObject, opts Options) (action, int64, error) {
|
||||
srcSize, err := src.Stat(ctx, o.Key)
|
||||
if errors.Is(err, storage.ErrNotExist) {
|
||||
return actionMissing, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return actionSkipped, 0, fmt.Errorf("stat source: %w", err)
|
||||
}
|
||||
if !opts.Overwrite {
|
||||
if dstSize, err := dst.Stat(ctx, o.Key); err == nil && dstSize == srcSize {
|
||||
return actionSkipped, 0, nil
|
||||
} else if err != nil && !errors.Is(err, storage.ErrNotExist) {
|
||||
return actionSkipped, 0, fmt.Errorf("stat destination: %w", err)
|
||||
}
|
||||
}
|
||||
if opts.DryRun {
|
||||
return actionCopied, srcSize, nil
|
||||
}
|
||||
|
||||
r, size, err := src.Open(ctx, o.Key)
|
||||
if err != nil {
|
||||
return actionSkipped, 0, fmt.Errorf("read source: %w", err)
|
||||
}
|
||||
defer r.Close()
|
||||
if size <= 0 {
|
||||
size = srcSize // local Open reports the stat size; be defensive anyway
|
||||
}
|
||||
contentType := o.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
if err := dst.Put(ctx, o.Key, r, size, contentType); err != nil {
|
||||
return actionSkipped, 0, fmt.Errorf("write destination: %w", err)
|
||||
}
|
||||
return actionCopied, size, nil
|
||||
}
|
||||
|
||||
func humanBytes(n int64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
v, exp := float64(n), 0
|
||||
for v >= unit && exp < 4 {
|
||||
v /= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", v, "KMGT"[exp-1])
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package photosync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// Two local backends stand in for the real pair: copyObject only talks to the
|
||||
// storage.Storage interface, so the S3 side needs no fake here.
|
||||
func backends(t *testing.T) (src, dst storage.Storage, srcRoot string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
srcRoot = filepath.Join(dir, "src")
|
||||
src, err := storage.NewLocal(srcRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dst, err = storage.NewLocal(filepath.Join(dir, "dst"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return src, dst, srcRoot
|
||||
}
|
||||
|
||||
func put(t *testing.T, s storage.Storage, key, body string) {
|
||||
t.Helper()
|
||||
if err := s.Put(context.Background(), key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
const testKey = "galleries/g1/original/p1.jpg"
|
||||
|
||||
func testObject() store.PhotoObject {
|
||||
return store.PhotoObject{PhotoID: "p1", GalleryID: "g1", Variant: "original", Key: testKey, ContentType: "image/jpeg"}
|
||||
}
|
||||
|
||||
func TestCopyObject(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("copies to destination", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "hello-photo")
|
||||
|
||||
act, size, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||
if err != nil || act != actionCopied || size != 11 {
|
||||
t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err)
|
||||
}
|
||||
if got, err := dst.Stat(ctx, testKey); err != nil || got != 11 {
|
||||
t.Fatalf("destination stat: (%d, %v)", got, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips same-size object already present", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "hello-photo")
|
||||
put(t, dst, testKey, "hello-photo")
|
||||
|
||||
act, _, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||
if err != nil || act != actionSkipped {
|
||||
t.Fatalf("got (%v, %v), want (skipped, nil)", act, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overwrite re-copies", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "new-content")
|
||||
put(t, dst, testKey, "old-content")
|
||||
|
||||
act, _, err := copyObject(ctx, src, dst, testObject(), Options{Overwrite: true})
|
||||
if err != nil || act != actionCopied {
|
||||
t.Fatalf("got (%v, %v), want (copied, nil)", act, err)
|
||||
}
|
||||
r, _, err := dst.Open(ctx, testKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "new-content" {
|
||||
t.Fatalf("destination body = %q, want new-content", body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reports objects missing on the source", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
|
||||
act, _, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||
if err != nil || act != actionMissing {
|
||||
t.Fatalf("got (%v, %v), want (missing, nil)", act, err)
|
||||
}
|
||||
if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist {
|
||||
t.Fatalf("destination stat err = %v, want ErrNotExist", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dry run writes nothing", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "hello-photo")
|
||||
|
||||
act, size, err := copyObject(ctx, src, dst, testObject(), Options{DryRun: true})
|
||||
if err != nil || act != actionCopied || size != 11 {
|
||||
t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err)
|
||||
}
|
||||
if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist {
|
||||
t.Fatalf("destination stat err = %v, want ErrNotExist", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseDirection(t *testing.T) {
|
||||
for in, want := range map[string]Direction{
|
||||
"to-s3": ToS3,
|
||||
"local-to-s3": ToS3,
|
||||
"to-local": ToLocal,
|
||||
"s3-to-local": ToLocal,
|
||||
} {
|
||||
got, err := ParseDirection(in)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("ParseDirection(%q) = (%v, %v), want %v", in, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseDirection("sideways"); err == nil {
|
||||
t.Error("ParseDirection(\"sideways\") should fail")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package photosync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// stubS3 is a minimal path-style S3 (PUT/HEAD/GET on /bucket/key) so the sync
|
||||
// can be driven end to end over the real aws-sdk client.
|
||||
type stubS3 struct {
|
||||
mu sync.Mutex
|
||||
objects map[string][]byte
|
||||
puts int
|
||||
}
|
||||
|
||||
func newStubS3(t *testing.T) (*stubS3, string) {
|
||||
t.Helper()
|
||||
s := &stubS3{objects: map[string][]byte{}}
|
||||
srv := httptest.NewServer(s)
|
||||
t.Cleanup(srv.Close)
|
||||
return s, srv.URL
|
||||
}
|
||||
|
||||
func (s *stubS3) get(key string) ([]byte, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
b, ok := s.objects[key]
|
||||
return b, ok
|
||||
}
|
||||
|
||||
func (s *stubS3) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
key := strings.TrimPrefix(r.URL.Path, "/test-bucket/")
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
body, err := readS3Body(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.objects[key] = body
|
||||
s.puts++
|
||||
s.mu.Unlock()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodHead:
|
||||
body, ok := s.get(key)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound) // a HEAD carries no error body
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodGet:
|
||||
body, ok := s.get(key)
|
||||
if !ok {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
fmt.Fprint(w, `<Error><Code>NoSuchKey</Code></Error>`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.Write(body)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
// readS3Body undoes the SDK's aws-chunked framing when it streams with a
|
||||
// trailing checksum (what it does for the non-seekable S3-to-S3 style reader).
|
||||
func readS3Body(r *http.Request) ([]byte, error) {
|
||||
raw, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") {
|
||||
return raw, nil
|
||||
}
|
||||
var out []byte
|
||||
rest := raw
|
||||
for {
|
||||
nl := strings.Index(string(rest), "\r\n")
|
||||
if nl < 0 {
|
||||
return out, nil
|
||||
}
|
||||
header := string(rest[:nl])
|
||||
rest = rest[nl+2:]
|
||||
size, err := strconv.ParseInt(strings.SplitN(header, ";", 2)[0], 16, 64)
|
||||
if err != nil || size == 0 {
|
||||
return out, nil // trailer section or malformed: body is complete
|
||||
}
|
||||
if int64(len(rest)) < size {
|
||||
return nil, fmt.Errorf("truncated aws-chunked body")
|
||||
}
|
||||
out = append(out, rest[:size]...)
|
||||
rest = rest[size:]
|
||||
if len(rest) >= 2 {
|
||||
rest = rest[2:] // chunk CRLF
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// syncEnv seeds a scratch SQLite DB with one gallery holding one ready photo
|
||||
// (original + thumb + preview) and returns a config wired to the stub bucket.
|
||||
func syncEnv(t *testing.T) (config.Config, *store.DB, *stubS3, []string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
stub, endpoint := newStubS3(t)
|
||||
cfg := config.Config{
|
||||
DBType: "sqlite",
|
||||
DatabaseURL: filepath.Join(dir, "test.db"),
|
||||
ViewTokenSecret: "test-secret",
|
||||
StoragePath: filepath.Join(dir, "photos"),
|
||||
S3Endpoint: endpoint,
|
||||
S3Region: "auto",
|
||||
S3Bucket: "test-bucket",
|
||||
S3AccessKeyID: "key",
|
||||
S3SecretKey: "secret",
|
||||
S3ForcePathStyle: true,
|
||||
}
|
||||
db, err := store.Open(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
ctx := context.Background()
|
||||
if err := db.Migrate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const (
|
||||
galleryID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
|
||||
photoID = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
|
||||
)
|
||||
now := time.Now()
|
||||
if err := db.CreateGallery(ctx, store.Gallery{
|
||||
ID: galleryID, Slug: "trip", Title: "Trip", Visibility: store.VisibilityPublic,
|
||||
ShareToken: "tok", CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
origKey := "galleries/" + galleryID + "/original/" + photoID + ".jpg"
|
||||
thumbKey := "galleries/" + galleryID + "/thumb/" + photoID + ".jpg"
|
||||
previewKey := "galleries/" + galleryID + "/preview/" + photoID + ".jpg"
|
||||
if err := db.InsertPhoto(ctx, store.Photo{
|
||||
ID: photoID, GalleryID: galleryID, OriginalKey: origKey, ContentType: "image/jpeg",
|
||||
SizeBytes: 11, NextAttemptAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.MarkPhotoReady(ctx, photoID, thumbKey, previewKey, 100, 80, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cfg, db, stub, []string{origKey, thumbKey, previewKey}
|
||||
}
|
||||
|
||||
func TestRunToS3(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg, db, stub, keys := syncEnv(t)
|
||||
|
||||
src, err := storage.NewLocal(cfg.StoragePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, k := range keys {
|
||||
put(t, src, k, fmt.Sprintf("photo-bytes-%d", i))
|
||||
}
|
||||
|
||||
res, err := Run(ctx, cfg, db, Options{Direction: ToS3, Concurrency: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 3 || res.Copied != 3 || res.Skipped != 0 || res.Failed != 0 || res.Missing != 0 {
|
||||
t.Fatalf("first run = %+v, want 3 total / 3 copied", res)
|
||||
}
|
||||
for i, k := range keys {
|
||||
body, ok := stub.get(k)
|
||||
if !ok {
|
||||
t.Fatalf("%s not uploaded", k)
|
||||
}
|
||||
if want := fmt.Sprintf("photo-bytes-%d", i); string(body) != want {
|
||||
t.Errorf("%s = %q, want %q", k, body, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Rerunning is a no-op: everything is already there at the same size.
|
||||
res, err = Run(ctx, cfg, db, Options{Direction: ToS3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Copied != 0 || res.Skipped != 3 {
|
||||
t.Fatalf("rerun = %+v, want 0 copied / 3 skipped", res)
|
||||
}
|
||||
|
||||
// --overwrite re-uploads them.
|
||||
before := stub.puts
|
||||
res, err = Run(ctx, cfg, db, Options{Direction: ToS3, Overwrite: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Copied != 3 || stub.puts != before+3 {
|
||||
t.Fatalf("overwrite run = %+v, puts %d → %d", res, before, stub.puts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunToLocal(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg, db, stub, keys := syncEnv(t)
|
||||
for i, k := range keys {
|
||||
stub.objects[k] = []byte(fmt.Sprintf("s3-bytes-%d", i))
|
||||
}
|
||||
|
||||
res, err := Run(ctx, cfg, db, Options{Direction: ToLocal})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 3 || res.Copied != 3 || res.Failed != 0 {
|
||||
t.Fatalf("run = %+v, want 3 total / 3 copied", res)
|
||||
}
|
||||
for i, k := range keys {
|
||||
body, err := os.ReadFile(filepath.Join(cfg.StoragePath, k))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", k, err)
|
||||
}
|
||||
if want := fmt.Sprintf("s3-bytes-%d", i); string(body) != want {
|
||||
t.Errorf("%s = %q, want %q", k, body, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingOnSource(t *testing.T) {
|
||||
cfg, db, _, _ := syncEnv(t)
|
||||
// Nothing on local disk: every object is reported missing, none fail.
|
||||
res, err := Run(context.Background(), cfg, db, Options{Direction: ToS3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Missing != 3 || res.Copied != 0 || res.Failed != 0 {
|
||||
t.Fatalf("run = %+v, want 3 missing", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRequiresS3Config(t *testing.T) {
|
||||
cfg, db, _, _ := syncEnv(t)
|
||||
cfg.S3Endpoint, cfg.S3Bucket = "", ""
|
||||
if _, err := Run(context.Background(), cfg, db, Options{Direction: ToS3}); err == nil {
|
||||
t.Fatal("sync without S3 configured should fail")
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,12 @@ type local struct {
|
||||
root string
|
||||
}
|
||||
|
||||
// NewLocal builds the local-disk backend explicitly, whichever backend is
|
||||
// active — `photo-api sync` needs both sides at once.
|
||||
func NewLocal(root string) (Storage, error) {
|
||||
return newLocal(root)
|
||||
}
|
||||
|
||||
func newLocal(root string) (*local, error) {
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create storage dir %s: %w", root, err)
|
||||
@@ -74,6 +80,21 @@ func (l *local) Open(_ context.Context, key string) (io.ReadCloser, int64, error
|
||||
return f, info.Size(), nil
|
||||
}
|
||||
|
||||
func (l *local) Stat(_ context.Context, key string) (int64, error) {
|
||||
p, err := l.path(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if os.IsNotExist(err) {
|
||||
return 0, ErrNotExist
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), nil
|
||||
}
|
||||
|
||||
func (l *local) Delete(_ context.Context, key string) error {
|
||||
p, err := l.path(key)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,14 +2,19 @@ package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
)
|
||||
@@ -23,6 +28,12 @@ type s3Store struct {
|
||||
bucket string
|
||||
}
|
||||
|
||||
// NewS3 builds the S3 backend explicitly, whichever backend is active —
|
||||
// `photo-api sync` needs both sides at once.
|
||||
func NewS3(cfg config.Config) (Storage, error) {
|
||||
return newS3(cfg)
|
||||
}
|
||||
|
||||
func newS3(cfg config.Config) (*s3Store, error) {
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
|
||||
awsconfig.WithRegion(cfg.S3Region),
|
||||
@@ -65,6 +76,28 @@ func (s *s3Store) Open(ctx context.Context, key string) (io.ReadCloser, int64, e
|
||||
return out.Body, aws.ToInt64(out.ContentLength), nil
|
||||
}
|
||||
|
||||
func (s *s3Store) Stat(ctx context.Context, key string) (int64, error) {
|
||||
out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
// Compatibles differ: some answer NotFound, some NoSuchKey, and a
|
||||
// HEAD carries no body to parse, so fall back to the status code.
|
||||
var notFound *types.NotFound
|
||||
var noKey *types.NoSuchKey
|
||||
var apiErr smithy.APIError
|
||||
var respErr *awshttp.ResponseError
|
||||
if errors.As(err, ¬Found) || errors.As(err, &noKey) ||
|
||||
(errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey")) ||
|
||||
(errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusNotFound) {
|
||||
return 0, ErrNotExist
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
return aws.ToInt64(out.ContentLength), nil
|
||||
}
|
||||
|
||||
func (s *s3Store) Delete(ctx context.Context, key string) error {
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
|
||||
@@ -7,6 +7,7 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
@@ -17,16 +18,30 @@ import (
|
||||
// callers then stream the object through the API instead.
|
||||
var ErrNoPresign = errors.New("presigned URLs not supported")
|
||||
|
||||
// ErrNotExist is what Stat reports for a missing object on either backend.
|
||||
var ErrNotExist = errors.New("object does not exist")
|
||||
|
||||
type Storage interface {
|
||||
Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error
|
||||
Open(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
// Stat returns the object's size without reading it, or ErrNotExist.
|
||||
Stat(ctx context.Context, key string) (int64, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
PresignGet(ctx context.Context, key, downloadFilename, contentType string, expiry time.Duration) (string, error)
|
||||
}
|
||||
|
||||
// New builds the backend that serves requests (see config.S3Enabled).
|
||||
func New(cfg config.Config) (Storage, error) {
|
||||
if cfg.S3Enabled() {
|
||||
return newS3(cfg)
|
||||
return NewS3(cfg)
|
||||
}
|
||||
return newLocal(cfg.StoragePath)
|
||||
return NewLocal(cfg.StoragePath)
|
||||
}
|
||||
|
||||
// Name describes a backend for log lines.
|
||||
func Name(cfg config.Config, s Storage) string {
|
||||
if _, ok := s.(*s3Store); ok {
|
||||
return fmt.Sprintf("S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint)
|
||||
}
|
||||
return fmt.Sprintf("local disk at %s", cfg.StoragePath)
|
||||
}
|
||||
|
||||
@@ -54,6 +54,19 @@ func Open(cfg config.Config) (*DB, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// IsUniqueViolation reports whether err is a duplicate-key error. The two
|
||||
// drivers wrap it in unrelated types (pgconn.PgError vs sqlite.Error), and
|
||||
// neither is worth importing here just for one check, so this matches on the
|
||||
// message: pgx renders "(SQLSTATE 23505)", modernc/sqlite "UNIQUE constraint
|
||||
// failed: ...".
|
||||
func IsUniqueViolation(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
return strings.Contains(msg, "SQLSTATE 23505") || strings.Contains(msg, "UNIQUE constraint failed")
|
||||
}
|
||||
|
||||
// Rebind converts ?-style placeholders to $n for Postgres. Queries in this
|
||||
// package never contain literal question marks in strings.
|
||||
func (db *DB) Rebind(query string) string {
|
||||
|
||||
@@ -29,14 +29,17 @@ type Photo struct {
|
||||
LastError string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
// Checksum is the sha256 of the original bytes, empty when the row has
|
||||
// not been hashed yet (uploaded before duplicate detection existed).
|
||||
Checksum string
|
||||
}
|
||||
|
||||
const photoColumns = `id, gallery_id, position, original_key, original_filename, content_type,
|
||||
size_bytes, width, height, thumb_key, preview_key, taken_at, status, attempts, next_attempt_at,
|
||||
last_error, created_at, updated_at`
|
||||
last_error, created_at, updated_at, checksum`
|
||||
|
||||
func scanPhoto(s scanner) (Photo, error) {
|
||||
var v [18]any
|
||||
var v [19]any
|
||||
dest := make([]any, len(v))
|
||||
for i := range v {
|
||||
dest[i] = &v[i]
|
||||
@@ -63,20 +66,82 @@ func scanPhoto(s scanner) (Photo, error) {
|
||||
LastError: asString(v[15]),
|
||||
CreatedAt: asTime(v[16]),
|
||||
UpdatedAt: asTime(v[17]),
|
||||
Checksum: asString(v[18]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// InsertPhoto returns an error satisfying IsUniqueViolation when the gallery
|
||||
// already holds a photo with the same checksum; callers treat that as a
|
||||
// duplicate rather than a failure.
|
||||
func (db *DB) InsertPhoto(ctx context.Context, p Photo) error {
|
||||
_, err := db.ExecContext(ctx, db.Rebind(`
|
||||
INSERT INTO photos_photos
|
||||
(id, gallery_id, position, original_key, original_filename, content_type, size_bytes,
|
||||
status, attempts, next_attempt_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`),
|
||||
status, attempts, next_attempt_at, created_at, updated_at, checksum)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?, ?)`),
|
||||
p.ID, p.GalleryID, p.Position, p.OriginalKey, nullable(p.OriginalFilename), p.ContentType,
|
||||
p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt))
|
||||
p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt),
|
||||
nullable(p.Checksum))
|
||||
return err
|
||||
}
|
||||
|
||||
// FindPhotoByChecksum looks for an existing photo with the same content in one
|
||||
// gallery — the duplicate check the upload path runs before storing anything.
|
||||
// Duplicate scope is per gallery: the same image in another gallery keeps its
|
||||
// own row and its own stored object.
|
||||
func (db *DB) FindPhotoByChecksum(ctx context.Context, galleryID, checksum string) (Photo, error) {
|
||||
if checksum == "" {
|
||||
return Photo{}, ErrNotFound // never match the not-yet-hashed rows
|
||||
}
|
||||
row := db.QueryRowContext(ctx, db.Rebind(
|
||||
"SELECT "+photoColumns+" FROM photos_photos WHERE gallery_id = ? AND checksum = ?"),
|
||||
galleryID, checksum)
|
||||
p, err := scanPhoto(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Photo{}, ErrNotFound
|
||||
}
|
||||
return p, err
|
||||
}
|
||||
|
||||
// SetPhotoChecksum fills in the hash of an already-stored photo. Used by the
|
||||
// backfill; returns a unique-violation error when the row turns out to
|
||||
// duplicate one already hashed in the same gallery.
|
||||
func (db *DB) SetPhotoChecksum(ctx context.Context, id, checksum string) error {
|
||||
res, err := db.ExecContext(ctx, db.Rebind(
|
||||
"UPDATE photos_photos SET checksum = ? WHERE id = ?"), checksum, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return errIfNoRows(res)
|
||||
}
|
||||
|
||||
// PhotosMissingChecksum lists rows still lacking a hash (optionally of one
|
||||
// gallery only), oldest first so backfill keeps the earliest upload of a
|
||||
// duplicate pair as the one that gets the checksum.
|
||||
func (db *DB) PhotosMissingChecksum(ctx context.Context, galleryID string) ([]Photo, error) {
|
||||
q := "SELECT " + photoColumns + " FROM photos_photos WHERE checksum IS NULL"
|
||||
var args []any
|
||||
if galleryID != "" {
|
||||
q += " AND gallery_id = ?"
|
||||
args = append(args, galleryID)
|
||||
}
|
||||
q += " ORDER BY gallery_id, position, created_at"
|
||||
rows, err := db.QueryContext(ctx, db.Rebind(q), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
photos := []Photo{}
|
||||
for rows.Next() {
|
||||
p, err := scanPhoto(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
photos = append(photos, p)
|
||||
}
|
||||
return photos, rows.Err()
|
||||
}
|
||||
|
||||
func (db *DB) GetPhoto(ctx context.Context, id string) (Photo, error) {
|
||||
row := db.QueryRowContext(ctx,
|
||||
db.Rebind("SELECT "+photoColumns+" FROM photos_photos WHERE id = ?"), id)
|
||||
@@ -169,6 +234,64 @@ func (db *DB) PhotoKeys(ctx context.Context, galleryID string) ([]string, error)
|
||||
return keys, rows.Err()
|
||||
}
|
||||
|
||||
// PhotoObject is one stored object: a photo variant plus the content type it
|
||||
// should be written with. Used by the storage sync, which treats the rows as
|
||||
// the inventory of what exists.
|
||||
type PhotoObject struct {
|
||||
PhotoID string
|
||||
GalleryID string
|
||||
Variant string // original | thumb | preview
|
||||
Key string
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// AllPhotoObjects lists every object of every photo (optionally of one
|
||||
// gallery only), oldest first. Variant keys are empty until the worker has
|
||||
// processed the photo; those are left out.
|
||||
func (db *DB) AllPhotoObjects(ctx context.Context, galleryID string) ([]PhotoObject, error) {
|
||||
q := `SELECT id, gallery_id, content_type, original_key, thumb_key, preview_key
|
||||
FROM photos_photos`
|
||||
var args []any
|
||||
if galleryID != "" {
|
||||
q += " WHERE gallery_id = ?"
|
||||
args = append(args, galleryID)
|
||||
}
|
||||
q += " ORDER BY gallery_id, position, created_at"
|
||||
rows, err := db.QueryContext(ctx, db.Rebind(q), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
objects := []PhotoObject{}
|
||||
for rows.Next() {
|
||||
var id, galID, ctype, orig, thumb, preview any
|
||||
if err := rows.Scan(&id, &galID, &ctype, &orig, &thumb, &preview); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
variants := []struct {
|
||||
name, key, contentType string
|
||||
}{
|
||||
{"original", asString(orig), asString(ctype)},
|
||||
{"thumb", asString(thumb), "image/jpeg"},
|
||||
{"preview", asString(preview), "image/jpeg"},
|
||||
}
|
||||
for _, v := range variants {
|
||||
if v.key == "" {
|
||||
continue
|
||||
}
|
||||
objects = append(objects, PhotoObject{
|
||||
PhotoID: asString(id),
|
||||
GalleryID: asString(galID),
|
||||
Variant: v.name,
|
||||
Key: v.key,
|
||||
ContentType: v.contentType,
|
||||
})
|
||||
}
|
||||
}
|
||||
return objects, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimNextPhoto picks the oldest due queued/failed photo and marks it
|
||||
// processing. Optimistic claim (RowsAffected check) works identically on
|
||||
// Postgres and SQLite; returns ErrNotFound when the queue is empty.
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Content hash of the stored original, for per-gallery duplicate detection on
|
||||
-- upload. NULL means "not hashed yet" — rows predating this migration, until
|
||||
-- `photo-api backfill-checksums` runs. NULLs are distinct under a unique
|
||||
-- index, so those rows never collide with each other.
|
||||
-- (No semicolons in these comments: the migration runner splits on them.)
|
||||
ALTER TABLE photos_photos ADD COLUMN IF NOT EXISTS checksum varchar(64);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS photos_photos_gallery_checksum_idx ON photos_photos (gallery_id, checksum);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Content hash of the stored original, for per-gallery duplicate detection on
|
||||
-- upload. NULL means "not hashed yet" — rows predating this migration, until
|
||||
-- `photo-api backfill-checksums` runs. NULLs are distinct under a unique
|
||||
-- index, so those rows never collide with each other.
|
||||
-- (No semicolons in these comments: the migration runner splits on them.)
|
||||
ALTER TABLE photos_photos ADD COLUMN checksum text;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS photos_photos_gallery_checksum_idx ON photos_photos (gallery_id, checksum);
|
||||
Reference in New Issue
Block a user