From 498d7d8a7da47066083db6b83323d4a97287c390 Mon Sep 17 00:00:00 2001 From: Michilis Date: Wed, 5 Aug 2026 05:41:12 +0000 Subject: [PATCH 1/3] =?UTF-8?q?Add=20photo=20content-hash=20dedup=20and=20?= =?UTF-8?q?local=E2=86=94S3=20library=20sync.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe. --- README.md | 16 ++ deploy/photos.nginx.conf | 109 ++++++++ deploy/prod_nginx/backend.nginx.conf | 146 ++++++++++ deploy/prod_nginx/frontend.nginx.conf | 115 ++++++++ deploy/prod_nginx/photos.ngnx.conf | 108 ++++++++ deploy/prod_nginx/spanglish_upstreams.conf | 12 + frontend/src/app/admin/photos/[id]/page.tsx | 31 ++- frontend/src/lib/api/photos.ts | 3 + package.json | 4 + photo-api/.env.example | 15 +- photo-api/Makefile | 13 +- photo-api/PLAN.md | 2 +- photo-api/README.md | 85 +++++- photo-api/cmd/photo-api/main.go | 150 +++++++++- photo-api/internal/checksum/backfill.go | 145 ++++++++++ photo-api/internal/checksum/backfill_test.go | 175 ++++++++++++ photo-api/internal/checksum/checksum.go | 31 +++ photo-api/internal/config/config.go | 34 ++- photo-api/internal/httpapi/api_test.go | 156 ++++++++++- photo-api/internal/httpapi/dto.go | 4 + photo-api/internal/httpapi/photos.go | 65 ++++- photo-api/internal/photosync/photosync.go | 229 +++++++++++++++ .../internal/photosync/photosync_test.go | 135 +++++++++ photo-api/internal/photosync/run_test.go | 262 ++++++++++++++++++ photo-api/internal/storage/local.go | 21 ++ photo-api/internal/storage/s3.go | 33 +++ photo-api/internal/storage/storage.go | 19 +- photo-api/internal/store/db.go | 13 + photo-api/internal/store/photos.go | 133 ++++++++- .../migrations/0002_photo_checksum.pg.sql | 8 + .../migrations/0002_photo_checksum.sqlite.sql | 8 + 31 files changed, 2216 insertions(+), 64 deletions(-) create mode 100644 deploy/photos.nginx.conf create mode 100644 deploy/prod_nginx/backend.nginx.conf create mode 100644 deploy/prod_nginx/frontend.nginx.conf create mode 100644 deploy/prod_nginx/photos.ngnx.conf create mode 100644 deploy/prod_nginx/spanglish_upstreams.conf create mode 100644 photo-api/internal/checksum/backfill.go create mode 100644 photo-api/internal/checksum/backfill_test.go create mode 100644 photo-api/internal/checksum/checksum.go create mode 100644 photo-api/internal/photosync/photosync.go create mode 100644 photo-api/internal/photosync/photosync_test.go create mode 100644 photo-api/internal/photosync/run_test.go create mode 100644 photo-api/migrations/0002_photo_checksum.pg.sql create mode 100644 photo-api/migrations/0002_photo_checksum.sqlite.sql diff --git a/README.md b/README.md index 4f48ba7..c1cdb84 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,23 @@ npm run dev:photos # go run ./cmd/photo-api npm run build:photos # go build -o bin/photo-api npm run migrate:photos # apply photos_* migrations npm run test:photos # go test ./... + +# Move the photo library between storage backends (both must be configured +# in photo-api/.env; the source is left untouched, reruns skip what is there) +npm run sync:photos:to-s3 # local disk -> S3 +npm run sync:photos:to-local # S3 -> local disk +npm run sync:photos -- to-s3 --dry-run # flags: --dry-run --overwrite + # --concurrency=N --gallery= + +# Uploads are deduplicated per gallery by content hash. Photos uploaded before +# that existed need hashing once (idempotent, deletes nothing): +npm run backfill:photos:checksums ``` +After a sync, set `STORAGE_BACKEND=s3` (or `local`) in `photo-api/.env` and +restart the service to serve from the new backend. See +[`photo-api/README.md`](photo-api/README.md#move-the-library-between-backends). + You can also run per workspace: ```bash @@ -117,6 +132,7 @@ Key settings (see `photo-api/.env.example`): - **DB**: `DB_TYPE` and `DATABASE_URL` — point at the **same** database as the backend - **Auth**: none needed for user auth — the service validates Better Auth session cookies against the shared database. `PHOTO_VIEW_SECRET` signs gallery image view tokens (falls back to `JWT_SECRET` during migration). - **Storage**: `STORAGE_PATH` (local disk) or `S3_ENDPOINT` + `S3_BUCKET` (S3/Garage/MinIO); S3 downloads use short-lived presigned URLs +- **Storage switch**: `STORAGE_BACKEND=auto|local|s3` (`auto` = S3 when it is configured). Keep both sides configured and flip this one line to move between them; `npm run sync:photos:to-s3` / `:to-local` copies the existing library first - **Uploads/worker**: `MAX_UPLOAD_MB`, `WORKER_CONCURRENCY` (a worker generates thumb/preview JPEG variants with EXIF stripped) ### Frontend (`frontend/.env`) diff --git a/deploy/photos.nginx.conf b/deploy/photos.nginx.conf new file mode 100644 index 0000000..8693202 --- /dev/null +++ b/deploy/photos.nginx.conf @@ -0,0 +1,109 @@ +# ============================================================ +# Spanglish Community - Photo Gallery API +# photos.spanglishcommunity.com +# ============================================================ + +server { + listen 80; + server_name photos.spanglishcommunity.com; + + location /.well-known/acme-challenge/ { + root /var/www/html; + } + + location / { + return 301 https://photos.spanglishcommunity.com$request_uri; + } +} + +server { + listen 443 ssl; + http2 on; + + server_name photos.spanglishcommunity.com; + + # Photos can be larger than typical JSON payloads + client_max_body_size 25m; + + # SSL + ssl_certificate /etc/letsencrypt/live/photos.spanglishcommunity.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/photos.spanglishcommunity.com/privkey.pem; + + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # Security + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # Logs + access_log /var/log/nginx/spanglish_photo_access.log; + error_log /var/log/nginx/spanglish_photo_error.log; + + # CORS Configuration + set $cors_origin ""; + if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") { + set $cors_origin $http_origin; + } + + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + + # Ensure 413 returns JSON + CORS + error_page 413 = @payload_too_large; + location @payload_too_large { + default_type application/json; + return 413 '{"error":"Payload too large (413). Please upload a smaller file."}'; + } + + # Ensure 429 (rate limited) returns JSON + CORS + error_page 429 = @rate_limited; + location @rate_limited { + default_type application/json; + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + return 429 '{"error":"Too many requests. Please slow down."}'; + } + + location / { + limit_req zone=spanglish_photo_limit burst=40 nodelay; + + # Preflight + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + add_header 'Access-Control-Max-Age' 86400 always; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + + proxy_pass http://spanglish_photo_api; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_hide_header 'Access-Control-Allow-Origin'; + proxy_hide_header 'Access-Control-Allow-Methods'; + proxy_hide_header 'Access-Control-Allow-Headers'; + proxy_hide_header 'Access-Control-Allow-Credentials'; + proxy_hide_header 'Access-Control-Expose-Headers'; + + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + + # Buffer large image uploads to disk rather than memory + proxy_request_buffering on; + proxy_max_temp_file_size 1024m; + } +} \ No newline at end of file diff --git a/deploy/prod_nginx/backend.nginx.conf b/deploy/prod_nginx/backend.nginx.conf new file mode 100644 index 0000000..fcb68da --- /dev/null +++ b/deploy/prod_nginx/backend.nginx.conf @@ -0,0 +1,146 @@ +# ============================================================ +# Spanglish Community - Backend API +# api.spanglishcommunity.com +# ============================================================ + +server { + listen 80; + server_name api.spanglishcommunity.com; + + # ACME + location /.well-known/acme-challenge/ { + root /var/www/html; + } + + # Force HTTPS + location / { + return 301 https://api.spanglishcommunity.com$request_uri; + } +} + +server { + listen 443 ssl; + http2 on; + + server_name api.spanglishcommunity.com; + + # Upload size limit (avoid nginx 413 on media uploads) + # Keep this >= backend MEDIA_MAX_UPLOAD_MB (default 10MB). + client_max_body_size 20m; + + # SSL + ssl_certificate /etc/letsencrypt/live/spanglishcommunity.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/spanglishcommunity.com/privkey.pem; + + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # Security (API) + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + + # Logs + access_log /var/log/nginx/spanglish_api_access.log; + error_log /var/log/nginx/spanglish_api_error.log; + + # CORS Configuration (set once, used everywhere) + set $cors_origin ""; + if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") { + set $cors_origin $http_origin; + } + + # Add CORS headers to all responses (including nginx-generated errors) + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + + # Ensure 413 returns JSON + CORS (browser otherwise reports "CORS blocked") + error_page 413 = @payload_too_large; + location @payload_too_large { + default_type application/json; + return 413 '{"error":"Payload too large (413). Please upload a smaller file."}'; + } + + # Photo gallery service (photo-api, port 3003). ^~ wins over the "/" + # prefix below, so /api/photos/* reaches the Go photo-api instead of the + # Node backend (which has no photo routes and would 404). The admin UI + # calls this cross-origin via NEXT_PUBLIC_API_URL, so preflight + CORS + # must be handled here just like location / below. + location ^~ /api/photos/ { + limit_req zone=spanglish_api_limit burst=50 nodelay; + + # Handle preflight OPTIONS requests (add_header inside if{} does NOT + # inherit server-level headers, so repeat all CORS headers here). + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + add_header 'Access-Control-Max-Age' 86400 always; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + + proxy_pass http://spanglish_photo_api; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Strip CORS headers from the service (nginx handles CORS here) + proxy_hide_header 'Access-Control-Allow-Origin'; + proxy_hide_header 'Access-Control-Allow-Methods'; + proxy_hide_header 'Access-Control-Allow-Headers'; + proxy_hide_header 'Access-Control-Allow-Credentials'; + proxy_hide_header 'Access-Control-Expose-Headers'; + + # Photo batches can be large; allow bigger bodies + unbuffered upload. + client_max_body_size 100m; + proxy_request_buffering off; + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + } + + location / { + limit_req zone=spanglish_api_limit burst=50 nodelay; + + # Handle preflight OPTIONS requests + # NOTE: add_header inside if{} does NOT inherit server-level headers, + # so we must repeat all CORS headers here. + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + add_header 'Access-Control-Max-Age' 86400 always; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + + proxy_pass http://spanglish_backend; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Strip CORS headers from backend (nginx handles CORS at server level) + proxy_hide_header 'Access-Control-Allow-Origin'; + proxy_hide_header 'Access-Control-Allow-Methods'; + proxy_hide_header 'Access-Control-Allow-Headers'; + proxy_hide_header 'Access-Control-Allow-Credentials'; + proxy_hide_header 'Access-Control-Expose-Headers'; + + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + } +} diff --git a/deploy/prod_nginx/frontend.nginx.conf b/deploy/prod_nginx/frontend.nginx.conf new file mode 100644 index 0000000..32c9500 --- /dev/null +++ b/deploy/prod_nginx/frontend.nginx.conf @@ -0,0 +1,115 @@ +# ============================================================ +# Spanglish Community - Frontend +# spanglishcommunity.com / www +# ============================================================ + +server { + listen 80; + server_name spanglishcommunity.com www.spanglishcommunity.com; + + # ACME + location /.well-known/acme-challenge/ { + root /var/www/html; + } + + # Force HTTPS + location / { + return 301 https://spanglishcommunity.com$request_uri; + } +} + +server { + listen 443 ssl; + http2 on; + + server_name spanglishcommunity.com www.spanglishcommunity.com; + + # Upload size limit (covers same-origin /api uploads via this vhost) + client_max_body_size 20m; + + # SSL + ssl_certificate /etc/letsencrypt/live/spanglishcommunity.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/spanglishcommunity.com/privkey.pem; + + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # Security + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # Logs + access_log /var/log/nginx/spanglish_frontend_access.log; + error_log /var/log/nginx/spanglish_frontend_error.log; + + # Proxy /api/photos to the photo-api (Go service, port 3003). ^~ wins over + # the /api prefix below so same-origin image/gallery requests reach the + # photo-api instead of the Node backend (which has no photo routes -> 404). + location ^~ /api/photos/ { + proxy_pass http://spanglish_photo_api; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Photo uploads/batches can be large; allow bigger bodies. + client_max_body_size 100m; + proxy_request_buffering off; + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + + # Let the photo-api set Cache-Control per image visibility (public vs. + # token-gated) rather than forcing a cache policy here. + } + + # Proxy /api to backend + location /api { + proxy_pass http://spanglish_backend; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + } + + # Proxy /uploads to backend + location /uploads { + proxy_pass http://spanglish_backend; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Cache static files + proxy_cache_valid 200 1d; + expires 1d; + add_header Cache-Control "public, immutable"; + } + + # Frontend App + location / { + proxy_pass http://spanglish_frontend; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket / HMR + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_read_timeout 60s; + proxy_connect_timeout 60s; + } +} diff --git a/deploy/prod_nginx/photos.ngnx.conf b/deploy/prod_nginx/photos.ngnx.conf new file mode 100644 index 0000000..2897366 --- /dev/null +++ b/deploy/prod_nginx/photos.ngnx.conf @@ -0,0 +1,108 @@ +# ============================================================ +# Spanglish Community - Photo Gallery API +# photos.spanglishcommunity.com +# ============================================================ + +server { + listen 80; + server_name photos.spanglishcommunity.com; + + location /.well-known/acme-challenge/ { + } + + location / { + return 301 https://photos.spanglishcommunity.com$request_uri; + } +} + +server { + listen 443 ssl; + http2 on; + + server_name photos.spanglishcommunity.com; + + # Photos can be larger than typical JSON payloads + client_max_body_size 25m; + + # SSL + ssl_certificate /etc/letsencrypt/live/photos.spanglishcommunity.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/photos.spanglishcommunity.com/privkey.pem; + + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # Security + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # Logs + access_log /var/log/nginx/spanglish_photo_access.log; + error_log /var/log/nginx/spanglish_photo_error.log; + + # CORS Configuration + set $cors_origin ""; + if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") { + set $cors_origin $http_origin; + } + + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + + # Ensure 413 returns JSON + CORS + error_page 413 = @payload_too_large; + location @payload_too_large { + default_type application/json; + return 413 '{"error":"Payload too large (413). Please upload a smaller file."}'; + } + + # Ensure 429 (rate limited) returns JSON + CORS + error_page 429 = @rate_limited; + location @rate_limited { + default_type application/json; + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + return 429 '{"error":"Too many requests. Please slow down."}'; + } + + location / { + limit_req zone=spanglish_photo_limit burst=40 nodelay; + + # Preflight + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' $cors_origin always; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always; + add_header 'Access-Control-Allow-Credentials' 'true' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + add_header 'Access-Control-Max-Age' 86400 always; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + + proxy_pass http://spanglish_photo_api; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_hide_header 'Access-Control-Allow-Origin'; + proxy_hide_header 'Access-Control-Allow-Methods'; + proxy_hide_header 'Access-Control-Allow-Headers'; + proxy_hide_header 'Access-Control-Allow-Credentials'; + proxy_hide_header 'Access-Control-Expose-Headers'; + + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + + # Buffer large image uploads to disk rather than memory + proxy_request_buffering on; + proxy_max_temp_file_size 1024m; + } +} diff --git a/deploy/prod_nginx/spanglish_upstreams.conf b/deploy/prod_nginx/spanglish_upstreams.conf new file mode 100644 index 0000000..a60eda8 --- /dev/null +++ b/deploy/prod_nginx/spanglish_upstreams.conf @@ -0,0 +1,12 @@ +upstream spanglish_frontend { + server 127.0.0.1:3019; +} +upstream spanglish_backend { + server 127.0.0.1:3018; +} +upstream spanglish_photo_api { + server 127.0.0.1:3003; +} + +limit_req_zone $binary_remote_addr zone=spanglish_photo_limit:10m rate=20r/s; +limit_req_zone $binary_remote_addr zone=spanglish_api_limit:10m rate=30r/s; \ No newline at end of file diff --git a/frontend/src/app/admin/photos/[id]/page.tsx b/frontend/src/app/admin/photos/[id]/page.tsx index 3f1dd3d..fff6d48 100644 --- a/frontend/src/app/admin/photos/[id]/page.tsx +++ b/frontend/src/app/admin/photos/[id]/page.tsx @@ -18,6 +18,7 @@ import { CheckCircleIcon, ChevronDownIcon, ChevronUpIcon, + DocumentDuplicateIcon, ExclamationCircleIcon, ExclamationTriangleIcon, LinkIcon, @@ -34,7 +35,9 @@ interface UploadItem { name: string; sizeBytes: number; progress: number; // 0..1 while uploading - status: 'queued' | 'uploading' | 'processing' | 'error'; + // 'duplicate' is terminal: the gallery already held these bytes, so nothing + // was stored and no new tile appears in the grid. + status: 'queued' | 'uploading' | 'processing' | 'error' | 'duplicate'; error?: string; photoId?: string; } @@ -117,8 +120,19 @@ export default function AdminGalleryDetailPage() { const { photos: added } = await photosApi.uploadPhotoWithProgress(id, file, (fraction) => patchUpload(key, { progress: fraction }) ); - setPhotos((prev) => [...prev, ...added]); - patchUpload(key, { status: 'processing', progress: 1, photoId: added[0]?.id }); + // A duplicate echoes back a photo already in the grid, so merge by id + // instead of appending (`duplicate` is an upload outcome, not photo + // state, so it is dropped here). + setPhotos((prev) => { + const byId = new Map(prev.map((p) => [p.id, p])); + added.forEach(({ duplicate: _duplicate, ...photo }) => byId.set(photo.id, photo)); + return Array.from(byId.values()); + }); + patchUpload(key, { + status: added[0]?.duplicate ? 'duplicate' : 'processing', + progress: 1, + photoId: added[0]?.id, + }); } catch (err) { patchUpload(key, { status: 'error', @@ -148,7 +162,8 @@ export default function AdminGalleryDetailPage() { }; // A row is "done" once its photo finished processing; the panel derives - // this from the photos list instead of tracking it separately. + // this from the photos list instead of tracking it separately. 'duplicate' + // is terminal and never reconciled — its photo was already there. const displayStatus = (u: UploadItem): { state: string; error?: string } => { if (u.status === 'processing' && u.photoId) { const photo = photos.find((p) => p.id === u.photoId); @@ -579,6 +594,11 @@ export default function AdminGalleryDetailPage() { {ds.state === 'queued' && (es ? 'En cola' : 'Queued')} {ds.state === 'processing' && (es ? 'Procesando…' : 'Processing…')} {ds.state === 'done' && formatBytes(u.sizeBytes)} + {ds.state === 'duplicate' && ( + + {es ? 'Ya está en esta galería' : 'Already in this gallery'} + + )} {ds.state === 'error' && ( {ds.error} @@ -588,6 +608,9 @@ export default function AdminGalleryDetailPage() { {ds.state === 'done' && } + {ds.state === 'duplicate' && ( + + )} {ds.state === 'error' && } {ds.state === 'processing' && (
diff --git a/frontend/src/lib/api/photos.ts b/frontend/src/lib/api/photos.ts index f2ce238..e2e9a39 100644 --- a/frontend/src/lib/api/photos.ts +++ b/frontend/src/lib/api/photos.ts @@ -52,6 +52,9 @@ export interface Photo { preview?: string; original: string; }; + // Upload responses only: these bytes were already in the gallery, so nothing + // was stored and the rest of this object describes the existing photo. + duplicate?: boolean; } export interface CreateGalleryInput { diff --git a/package.json b/package.json index a8a0825..25238b9 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,10 @@ "build:photos": "cd photo-api && go build -o bin/photo-api ./cmd/photo-api", "test:photos": "cd photo-api && go test ./...", "migrate:photos": "cd photo-api && go run ./cmd/photo-api migrate", + "sync:photos": "cd photo-api && go run ./cmd/photo-api sync", + "sync:photos:to-s3": "cd photo-api && go run ./cmd/photo-api sync to-s3", + "sync:photos:to-local": "cd photo-api && go run ./cmd/photo-api sync to-local", + "backfill:photos:checksums": "cd photo-api && go run ./cmd/photo-api backfill-checksums", "start": "concurrently \"npm run start:backend\" \"npm run start:frontend\"", "start:backend": "npm run start --workspace=backend", "start:frontend": "npm run start --workspace=frontend", diff --git a/photo-api/.env.example b/photo-api/.env.example index f312e0e..9ec4861 100644 --- a/photo-api/.env.example +++ b/photo-api/.env.example @@ -29,10 +29,19 @@ JWT_SECRET= # Public site origin, used to build share links and allow dev CORS FRONTEND_URL=https://spanglishcommunity.com -# Photo storage. Local disk by default; setting BOTH S3_ENDPOINT and -# S3_BUCKET switches to S3 (same convention as the backend's media storage). -# Use a photos-specific bucket — do not reuse the backend's media bucket. +# Photo storage. Which backend serves requests: +# auto (default) S3 when S3_ENDPOINT + S3_BUCKET are set, local otherwise +# — the backend's convention (backend/src/lib/storage.ts) +# local always local disk, even with the S3 settings filled in +# s3 always S3 +# Keep both sides filled in and flip this one line to switch backends. Both +# are also required by the library sync (`npm run sync:photos:to-s3` / +# `:to-local`), which copies every photo from one backend to the other. +STORAGE_BACKEND=auto + STORAGE_PATH=./data/photos + +# Use a photos-specific bucket — do not reuse the backend's media bucket. #S3_ENDPOINT= #S3_REGION=auto #S3_BUCKET=spanglish-photos diff --git a/photo-api/Makefile b/photo-api/Makefile index 6fcfefb..fa7ee42 100644 --- a/photo-api/Makefile +++ b/photo-api/Makefile @@ -1,11 +1,11 @@ -.PHONY: start build test migrate clean help +.PHONY: start build test migrate sync-to-s3 sync-to-local backfill-checksums clean help BINARY := bin/photo-api CMD := ./cmd/photo-api help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \ - awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}' + awk 'BEGIN {FS = ":.*?## "}; {printf " %-19s %s\n", $$1, $$2}' start: ## Run the photo-api server (go run) go run $(CMD) @@ -19,5 +19,14 @@ test: ## Run all Go tests migrate: ## Apply pending photos_* migrations go run $(CMD) migrate +sync-to-s3: ## Copy the photo library from local disk to S3 (needs both in .env) + go run $(CMD) sync to-s3 + +sync-to-local: ## Copy the photo library from S3 back to local disk + go run $(CMD) sync to-local + +backfill-checksums: ## Hash photos uploaded before duplicate detection existed + go run $(CMD) backfill-checksums + clean: ## Remove built binary rm -rf bin diff --git a/photo-api/PLAN.md b/photo-api/PLAN.md index fb002eb..86bc000 100644 --- a/photo-api/PLAN.md +++ b/photo-api/PLAN.md @@ -131,7 +131,7 @@ All routes under `/api/photos`. Auth = `Authorization: Bearer `. " | `GET /api/photos/galleries/:id` | Full gallery detail incl. photos in position order with per-variant URLs and processing status. | `{ gallery, photos: [...] }` | | `PATCH /api/photos/galleries/:id` | Update title/description/eventId/visibility/coverPhotoId. | `{ gallery }` | | `DELETE /api/photos/galleries/:id` | Delete gallery + photos + stored objects. | `{ message }` | -| `POST /api/photos/galleries/:id/photos` | Multipart upload, field `files` (repeatable). Sniffs magic bytes (JPEG/PNG/WebP/GIF/AVIF; explicit `415 { error: "HEIC is not supported, please upload JPEG" }` for HEIC). Stores original, inserts row `status='queued'`, appends at end position. | `201 { photos: [...] }` | +| `POST /api/photos/galleries/:id/photos` | Multipart upload, field `files` (repeatable). Sniffs magic bytes (JPEG/PNG/WebP/GIF/AVIF; explicit `415 { error: "HEIC is not supported, please upload JPEG" }` for HEIC). Stores original, inserts row `status='queued'`, appends at end position. Hashes the bytes into `checksum`: if the gallery already holds them, nothing is stored and the existing photo is echoed with `duplicate: true` (the batch continues, no position consumed). | `201 { photos: [...] }` | | `PATCH /api/photos/galleries/:id/order` | Body `{ photoIds: [uuid,...] }` — full ordering; positions rewritten in one transaction. | `{ message }` | | `DELETE /api/photos/photos/:photoId` | Delete one photo + its objects; compacts positions. | `{ message }` | | `POST /api/photos/galleries/:id/share-token` | Rotate share token (invalidate old links). | `{ gallery }` | diff --git a/photo-api/README.md b/photo-api/README.md index 624e6f9..80977e0 100644 --- a/photo-api/README.md +++ b/photo-api/README.md @@ -14,9 +14,13 @@ Design/decisions: [PLAN.md](./PLAN.md). - Originals are stored byte-identical for download; a worker generates JPEG variants (thumb 512px q78, preview 2048px q85, EXIF stripped/orientation applied) queued in the DB with retries. +- Uploads are deduplicated per gallery: see + [Duplicate detection](#duplicate-detection). - Storage: local disk (`STORAGE_PATH`) or S3/Garage/MinIO (set `S3_ENDPOINT` - + `S3_BUCKET`), same selection convention as the backend. S3 downloads use - short-lived presigned URLs; nothing in the bucket is public. + + `S3_BUCKET`), same selection convention as the backend, overridable with + `STORAGE_BACKEND`. S3 downloads use short-lived presigned URLs; nothing in + the bucket is public. Switching backends later: see + [Move the library between backends](#move-the-library-between-backends). - Auth: validates the backend's HS256 JWTs with the shared `JWT_SECRET` (issuer `spanglish`, audience `spanglish-app`) including the DB-backed tokenVersion/account-status revocation check. Admin surface is @@ -37,7 +41,8 @@ go test ./... # SQLite; add PHOTO_TEST_PG= to also run on Post ``` From the repo root: `npm run dev:photos`, `npm run build:photos`, -`npm run test:photos`. The Next dev server rewrites `/api/photos/*` to +`npm run test:photos`, `npm run migrate:photos`, `npm run sync:photos`, +`npm run backfill:photos:checksums`. The Next dev server rewrites `/api/photos/*` to `PHOTO_API_URL` (default `http://localhost:3003`), so the frontend needs no extra config in dev. @@ -45,6 +50,78 @@ HEIC uploads require a converter CLI on the host: `apt install libvips-tools` (or `libheif-examples`). Without one, HEIC uploads are rejected with a clear message and a startup warning is logged. +## Move the library between backends + +`photo-api sync` copies the whole photo library one way between local disk and +S3, so local↔S3 is a config switch rather than a migration project. **Both +backends must be configured in `photo-api/.env`** (`STORAGE_PATH` *and* the +`S3_*` values); `STORAGE_BACKEND` decides which one actually serves requests, +so filling in S3 does not switch anything by itself. + +Local disk → S3: + +```bash +npm run sync:photos -- to-s3 --dry-run # see what would be copied +npm run sync:photos:to-s3 # copy it +# then set STORAGE_BACKEND=s3 in photo-api/.env and restart the service +``` + +S3 → local disk is the same with `to-local` / `STORAGE_BACKEND=local` +(`npm run sync:photos:to-local`). From `photo-api/`: `make sync-to-s3`, +`make sync-to-local`. + +Flags (`npm run sync:photos -- to-s3 --overwrite`, or after the direction on +the direct scripts): `--dry-run`, `--overwrite` (re-copy objects already +present with the same size), `--concurrency=N` (default 4), `--gallery=`. + +How it behaves: + +- The `photos_photos` rows are the inventory — for each photo the original + plus, once processed, the thumb and preview. Objects with no row (worker + scratch files, leftovers of deleted galleries) are not copied. +- Only the destination is written. The source stays as a fallback; delete it + by hand once the switch is verified. +- Reruns are cheap and safe: objects already on the destination with the same + size are skipped, so an interrupted or partly failed sync just needs + rerunning. A failed object is logged and the exit code is non-zero. +- Keys are identical on both backends, so nothing in the database changes and + no re-processing is triggered. +- Photos uploaded *after* the copy but *before* the restart land on the old + backend. For a clean cutover, stop the service, sync, flip + `STORAGE_BACKEND`, start again — or sync a second time after the switch to + pick up stragglers. + +## Duplicate detection + +Every upload is hashed (sha256 of the original bytes) into +`photos_photos.checksum`, unique per `(gallery_id, checksum)`. If a gallery +already holds those exact bytes, the incoming copy is **discarded**: no object +is stored, no row is inserted, and the upload response echoes the existing +photo with `"duplicate": true`. The rest of the batch continues normally — a +duplicate is not an error and does not consume a position. + +Scope is one gallery. The same image can still live in several galleries, each +with its own row and its own stored object, so deleting a gallery never orphans +another one's photos. + +The admin uploader panel shows those rows as *"Already in this gallery"*; no +second tile appears in the grid. + +Photos uploaded before this existed have no checksum, so they are not matched +until hashed once: + +```bash +npm run backfill:photos:checksums -- --dry-run # what would be hashed +npm run backfill:photos:checksums # hash it +``` + +From `photo-api/`: `make backfill-checksums`. Flags: `--dry-run`, +`--concurrency=N` (default 4), `--gallery=`. It reads from the active +`STORAGE_BACKEND`, only ever writes the checksum column, and is idempotent — +rerun it after a sync or a restore. Photos whose content already matches an +earlier one in the same gallery are **reported and left unhashed**; the command +never deletes anything, so removing the extras is an admin's call. + ## API Everything under `/api/photos`. Errors are `{"error": string}`. @@ -58,7 +135,7 @@ Admin (Bearer token, role admin/organizer): | GET | `/api/photos/galleries/:id` | gallery + photos (all statuses) | | PATCH | `/api/photos/galleries/:id` | update title/visibility/event/cover | | DELETE | `/api/photos/galleries/:id` | delete gallery + objects | -| POST | `/api/photos/galleries/:id/photos` | multipart upload (`files`) | +| POST | `/api/photos/galleries/:id/photos` | multipart upload (`files`), deduplicated | | PATCH | `/api/photos/galleries/:id/order` | reorder (`{photoIds}`) | | POST | `/api/photos/galleries/:id/share-token` | rotate share token | | DELETE | `/api/photos/photos/:photoId` | delete photo | diff --git a/photo-api/cmd/photo-api/main.go b/photo-api/cmd/photo-api/main.go index 406563c..cf8475d 100644 --- a/photo-api/cmd/photo-api/main.go +++ b/photo-api/cmd/photo-api/main.go @@ -1,12 +1,15 @@ // photo-api serves event photo galleries for the Spanglish platform. // -// photo-api start the HTTP server -// photo-api migrate apply pending photos_* migrations and exit +// photo-api start the HTTP server +// photo-api migrate apply pending photos_* migrations and exit +// photo-api sync to-s3|to-local copy the photo library between backends +// photo-api backfill-checksums hash photos uploaded before duplicate detection package main import ( "context" "errors" + "flag" "fmt" "log" "net/http" @@ -16,9 +19,11 @@ import ( "time" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/config" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/httpapi" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/photosync" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/store" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/worker" @@ -38,15 +43,21 @@ func main() { } defer db.Close() - if len(os.Args) > 1 && os.Args[1] == "migrate" { - if err := db.Migrate(context.Background()); err != nil { - log.Fatalf("migrate: %v", err) - } - log.Println("migrations up to date") - return - } if len(os.Args) > 1 { - log.Fatalf("unknown subcommand %q (expected: migrate)", os.Args[1]) + switch os.Args[1] { + case "migrate": + if err := db.Migrate(context.Background()); err != nil { + log.Fatalf("migrate: %v", err) + } + log.Println("migrations up to date") + case "sync": + runSync(cfg, db, os.Args[2:]) + case "backfill-checksums": + runBackfillChecksums(cfg, db, os.Args[2:]) + default: + log.Fatalf("unknown subcommand %q (expected: migrate, sync, backfill-checksums)", os.Args[1]) + } + return } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -64,11 +75,7 @@ func main() { if err != nil { log.Fatalf("storage: %v", err) } - if cfg.S3Enabled() { - log.Printf("storage: S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint) - } else { - log.Printf("storage: local disk at %s", cfg.StoragePath) - } + log.Printf("storage: %s (STORAGE_BACKEND=%s)", storage.Name(cfg, st), cfg.StorageBackend) heic := imaging.DetectHeicConverter(cfg.HeicConverter) if heic == nil { @@ -104,3 +111,116 @@ func main() { log.Printf("shutdown: %v", err) } } + +const syncUsage = `usage: photo-api sync to-s3|to-local [flags] + +Copies every object of every photo (original, thumb, preview) from one storage +backend to the other. Both must be configured in photo-api/.env; the source is +never modified, and reruns skip objects already present on the destination. + +Flags: +` + +// runSync handles `photo-api sync [flags]`, the storage migration +// used when moving the library between local disk and S3. +func runSync(cfg config.Config, db *store.DB, args []string) { + fs := flag.NewFlagSet("sync", flag.ExitOnError) + fs.Usage = func() { + fmt.Fprint(fs.Output(), syncUsage) + fs.PrintDefaults() + } + var ( + dryRun = fs.Bool("dry-run", false, "report what would be copied without writing") + overwrite = fs.Bool("overwrite", false, "re-copy objects already present with the same size") + workers = fs.Int("concurrency", 4, "objects copied in parallel") + gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)") + ) + if len(args) == 0 { + fs.Usage() + os.Exit(2) + } + direction, err := photosync.ParseDirection(args[0]) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n\n", err) + fs.Usage() + os.Exit(2) + } + if err := fs.Parse(args[1:]); err != nil { + os.Exit(2) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if _, err := photosync.Run(ctx, cfg, db, photosync.Options{ + Direction: direction, + GalleryID: *gallery, + Concurrency: *workers, + DryRun: *dryRun, + Overwrite: *overwrite, + }); err != nil { + log.Fatalf("sync: %v", err) + } + if *dryRun { + return + } + target := "s3" + if direction == photosync.ToLocal { + target = "local" + } + log.Printf("sync: set STORAGE_BACKEND=%s in photo-api/.env and restart the service to serve from it "+ + "(the source copy is left in place — delete it once the switch is verified)", target) +} + +const backfillUsage = `usage: photo-api backfill-checksums [flags] + +Hashes the stored original of every photo that has no checksum yet, so uploads +of a photo already in a gallery are recognised as duplicates. Runs against the +active storage backend (STORAGE_BACKEND) and only ever writes the checksum +column — no photo is deleted. Photos whose content already matches an earlier +one in the same gallery are reported and left unhashed. + +Flags: +` + +// runBackfillChecksums handles `photo-api backfill-checksums [flags]`, the +// one-off pass needed after the checksum column is added to an existing +// library. +func runBackfillChecksums(cfg config.Config, db *store.DB, args []string) { + fs := flag.NewFlagSet("backfill-checksums", flag.ExitOnError) + fs.Usage = func() { + fmt.Fprint(fs.Output(), backfillUsage) + fs.PrintDefaults() + } + var ( + dryRun = fs.Bool("dry-run", false, "report what would be hashed without writing") + workers = fs.Int("concurrency", 4, "originals hashed in parallel") + gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)") + ) + if err := fs.Parse(args); err != nil { + os.Exit(2) + } + + st, err := storage.New(cfg) + if err != nil { + log.Fatalf("storage: %v", err) + } + log.Printf("backfill-checksums: reading from %s", storage.Name(cfg, st)) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + res, err := checksum.Backfill(ctx, db, st, checksum.Options{ + GalleryID: *gallery, + Concurrency: *workers, + DryRun: *dryRun, + }) + if err != nil { + log.Fatalf("backfill-checksums: %v", err) + } + if res.Duplicates > 0 { + log.Printf("backfill-checksums: %d existing photos duplicate an earlier one in their gallery "+ + "(listed above); they still show in the gallery — delete the unwanted ones from the admin page", + res.Duplicates) + } +} diff --git a/photo-api/internal/checksum/backfill.go b/photo-api/internal/checksum/backfill.go new file mode 100644 index 0000000..18ccece --- /dev/null +++ b/photo-api/internal/checksum/backfill.go @@ -0,0 +1,145 @@ +package checksum + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + "sync/atomic" + + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/store" +) + +// Options configures a backfill run. +type Options struct { + // GalleryID limits the run to one gallery; empty means the whole library. + GalleryID string + // Concurrency is how many originals are hashed at a time. + Concurrency int + // DryRun reports what would be hashed without writing to the database. + DryRun bool +} + +// Result counts what a run did. Duplicates are rows whose bytes match an +// earlier photo in the same gallery; they keep a NULL checksum and are listed +// so an admin can decide what to do with them. +type Result struct { + Total int // rows without a checksum + Hashed int // hashed and recorded (or, with DryRun, would be) + Duplicates int + Missing int // original object absent from storage + Failed int +} + +// Backfill hashes the stored original of every photo that has no checksum yet, +// so duplicate detection also catches re-uploads of photos from before the +// checksum column existed. It only ever writes the checksum column; no photo, +// row or object is deleted. +// +// It is idempotent: rerunning it finds only what the previous run left behind. +func Backfill(ctx context.Context, db *store.DB, st storage.Storage, opts Options) (Result, error) { + if opts.Concurrency < 1 { + opts.Concurrency = 4 + } + photos, err := db.PhotosMissingChecksum(ctx, opts.GalleryID) + if err != nil { + return Result{}, fmt.Errorf("list photos without a checksum: %w", err) + } + + prefix := "" + if opts.DryRun { + prefix = "[dry-run] " + } + log.Printf("backfill-checksums: %s%d photos to hash, concurrency %d", prefix, len(photos), opts.Concurrency) + + var ( + mu sync.Mutex + res = Result{Total: len(photos)} + done int64 + jobs = make(chan store.Photo) + wg sync.WaitGroup + ) + // Sequential hashing per worker, but the database update is what can + // collide: two photos of the same gallery with identical bytes race, and + // the unique index decides which one keeps the checksum. + for i := 0; i < opts.Concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for p := range jobs { + if ctx.Err() != nil { + return + } + sum, err := hashOriginal(ctx, st, p) + if err == nil && !opts.DryRun { + err = db.SetPhotoChecksum(ctx, p.ID, sum) + } + n := atomic.AddInt64(&done, 1) + + mu.Lock() + switch { + case errors.Is(err, storage.ErrNotExist): + res.Missing++ + log.Printf("backfill-checksums: [%d/%d] MISSING object, skipped: %s (photo %s)", + n, len(photos), p.OriginalKey, p.ID) + case store.IsUniqueViolation(err): + res.Duplicates++ + other := "an earlier photo" + if existing, findErr := db.FindPhotoByChecksum(ctx, p.GalleryID, sum); findErr == nil { + other = "photo " + existing.ID + } + log.Printf("backfill-checksums: [%d/%d] DUPLICATE: photo %s (%s) has the same content as %s "+ + "in gallery %s — left without a checksum, delete it by hand if unwanted", + n, len(photos), p.ID, p.OriginalFilename, other, p.GalleryID) + case err != nil: + res.Failed++ + log.Printf("backfill-checksums: [%d/%d] FAILED %s (photo %s): %v", + n, len(photos), p.OriginalKey, p.ID, err) + default: + res.Hashed++ + } + mu.Unlock() + } + }() + } + for _, p := range photos { + select { + case jobs <- p: + case <-ctx.Done(): + } + if ctx.Err() != nil { + break + } + } + close(jobs) + wg.Wait() + + log.Printf("backfill-checksums: %sdone — %d hashed, %d duplicates left unhashed, %d objects missing, %d failed", + prefix, res.Hashed, res.Duplicates, res.Missing, res.Failed) + + if err := ctx.Err(); err != nil { + return res, fmt.Errorf("interrupted after %d/%d photos: %w", res.Hashed, res.Total, err) + } + if res.Failed > 0 { + return res, fmt.Errorf("%d of %d photos failed to hash (rerun to retry; already-hashed photos are skipped)", + res.Failed, res.Total) + } + return res, nil +} + +// hashOriginal reads a photo's stored original. Stat runs first because it is +// the only call that reports a missing object as storage.ErrNotExist on both +// backends — Open surfaces the driver's own error. +func hashOriginal(ctx context.Context, st storage.Storage, p store.Photo) (string, error) { + if _, err := st.Stat(ctx, p.OriginalKey); err != nil { + return "", err + } + r, _, err := st.Open(ctx, p.OriginalKey) + if err != nil { + return "", err + } + defer r.Close() + return Sum(r) +} diff --git a/photo-api/internal/checksum/backfill_test.go b/photo-api/internal/checksum/backfill_test.go new file mode 100644 index 0000000..3425a14 --- /dev/null +++ b/photo-api/internal/checksum/backfill_test.go @@ -0,0 +1,175 @@ +package checksum + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/config" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/store" +) + +const galleryID = "11111111-1111-1111-1111-111111111111" + +// env builds a migrated scratch SQLite database with one gallery, plus a local +// storage backend rooted next to it. +func env(t *testing.T) (*store.DB, storage.Storage) { + t.Helper() + dir := t.TempDir() + db, err := store.Open(config.Config{DBType: "sqlite", DatabaseURL: filepath.Join(dir, "test.db")}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + ctx := context.Background() + if err := db.Migrate(ctx); err != nil { + t.Fatal(err) + } + now := time.Now() + if err := db.CreateGallery(ctx, store.Gallery{ + ID: galleryID, Slug: "g", Title: "G", Visibility: "private", ShareToken: "tok", + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + st, err := storage.NewLocal(filepath.Join(dir, "photos")) + if err != nil { + t.Fatal(err) + } + return db, st +} + +// seed inserts an unhashed photo whose original holds body, mimicking a row +// uploaded before the checksum column existed. +func seed(t *testing.T, db *store.DB, st storage.Storage, id, body string) store.Photo { + t.Helper() + ctx := context.Background() + key := "galleries/" + galleryID + "/orig/" + id + ".jpg" + if body != "" { + if err := st.Put(ctx, key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil { + t.Fatal(err) + } + } + now := time.Now() + p := store.Photo{ + ID: id, GalleryID: galleryID, OriginalKey: key, OriginalFilename: id + ".jpg", + ContentType: "image/jpeg", SizeBytes: int64(len(body)), Status: "ready", + NextAttemptAt: now, CreatedAt: now, UpdatedAt: now, + } + if err := db.InsertPhoto(ctx, p); err != nil { + t.Fatal(err) + } + return p +} + +func checksumOf(t *testing.T, db *store.DB, id string) string { + t.Helper() + p, err := db.GetPhoto(context.Background(), id) + if err != nil { + t.Fatal(err) + } + return p.Checksum +} + +func TestBackfill(t *testing.T) { + ctx := context.Background() + + t.Run("hashes unhashed photos and is idempotent", func(t *testing.T) { + db, st := env(t) + seed(t, db, st, "p1", "alpha") + seed(t, db, st, "p2", "beta") + + res, err := Backfill(ctx, db, st, Options{}) + if err != nil { + t.Fatal(err) + } + if res.Total != 2 || res.Hashed != 2 || res.Duplicates != 0 || res.Failed != 0 { + t.Fatalf("first run: %+v", res) + } + want, err := Sum(strings.NewReader("alpha")) + if err != nil { + t.Fatal(err) + } + if got := checksumOf(t, db, "p1"); got != want { + t.Fatalf("p1 checksum = %q, want %q", got, want) + } + + res, err = Backfill(ctx, db, st, Options{}) + if err != nil { + t.Fatal(err) + } + if res.Total != 0 || res.Hashed != 0 { + t.Fatalf("rerun should find nothing to do: %+v", res) + } + }) + + t.Run("reports pre-existing duplicates and leaves them unhashed", func(t *testing.T) { + db, st := env(t) + seed(t, db, st, "p1", "same") + seed(t, db, st, "p2", "same") + + res, err := Backfill(ctx, db, st, Options{Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if res.Hashed != 1 || res.Duplicates != 1 || res.Failed != 0 { + t.Fatalf("got %+v, want 1 hashed and 1 duplicate", res) + } + // The earlier row keeps the checksum, the later one is left alone — + // nothing is deleted either way. + if checksumOf(t, db, "p1") == "" { + t.Fatal("p1 should have been hashed") + } + if got := checksumOf(t, db, "p2"); got != "" { + t.Fatalf("p2 checksum = %q, want empty", got) + } + if _, err := db.GetPhoto(ctx, "p2"); err != nil { + t.Fatalf("duplicate row must survive: %v", err) + } + }) + + t.Run("skips photos whose object is gone", func(t *testing.T) { + db, st := env(t) + seed(t, db, st, "p1", "") // row without a stored original + + res, err := Backfill(ctx, db, st, Options{}) + if err != nil { + t.Fatal(err) + } + if res.Missing != 1 || res.Hashed != 0 || res.Failed != 0 { + t.Fatalf("got %+v, want 1 missing", res) + } + }) + + t.Run("dry run writes nothing", func(t *testing.T) { + db, st := env(t) + seed(t, db, st, "p1", "alpha") + + res, err := Backfill(ctx, db, st, Options{DryRun: true}) + if err != nil { + t.Fatal(err) + } + if res.Hashed != 1 { + t.Fatalf("got %+v, want 1 hashed", res) + } + if got := checksumOf(t, db, "p1"); got != "" { + t.Fatalf("dry run recorded %q", got) + } + }) + + t.Run("limits to one gallery", func(t *testing.T) { + db, st := env(t) + seed(t, db, st, "p1", "alpha") + + res, err := Backfill(ctx, db, st, Options{GalleryID: "22222222-2222-2222-2222-222222222222"}) + if err != nil { + t.Fatal(err) + } + if res.Total != 0 { + t.Fatalf("other gallery should have nothing to do: %+v", res) + } + }) +} diff --git a/photo-api/internal/checksum/checksum.go b/photo-api/internal/checksum/checksum.go new file mode 100644 index 0000000..daa899a --- /dev/null +++ b/photo-api/internal/checksum/checksum.go @@ -0,0 +1,31 @@ +// Package checksum defines the content hash used to spot duplicate photos and +// the backfill that fills it in for photos uploaded before it existed. +// +// The hash is sha256 over the untouched original bytes, hex encoded, stored in +// photos_photos.checksum. Duplicate scope is one gallery — the unique index is +// on (gallery_id, checksum) — so the same image may still live in several +// galleries, each with its own row and its own stored object. +package checksum + +import ( + "crypto/sha256" + "encoding/hex" + "hash" + "io" +) + +// New returns a fresh hasher. Upload hashes as it spools the body, so it needs +// the writer rather than a finished reader. +func New() hash.Hash { return sha256.New() } + +// Format renders a hasher's digest the way it is stored. +func Format(h hash.Hash) string { return hex.EncodeToString(h.Sum(nil)) } + +// Sum reads r to EOF and returns its digest. +func Sum(r io.Reader) (string, error) { + h := New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return Format(h), nil +} diff --git a/photo-api/internal/config/config.go b/photo-api/internal/config/config.go index 42a7f5e..8f8765c 100644 --- a/photo-api/internal/config/config.go +++ b/photo-api/internal/config/config.go @@ -20,6 +20,11 @@ type Config struct { // JWT_SECRET during the Better Auth migration. ViewTokenSecret string + // StorageBackend selects the active backend: "auto" (S3 when it is + // configured, local otherwise), "local" or "s3". Explicit values let both + // backends stay configured — required to run `photo-api sync`, and the + // one-line switch after a sync. + StorageBackend string StoragePath string S3Endpoint string S3Region string @@ -34,12 +39,26 @@ type Config struct { HeicConverter string // optional explicit converter command; autodetected when empty } -// S3Enabled mirrors backend/src/lib/storage.ts: S3 is active when both -// S3_ENDPOINT and S3_BUCKET are set. -func (c Config) S3Enabled() bool { +// S3Configured reports whether the S3 credentials are present at all, +// mirroring backend/src/lib/storage.ts: both S3_ENDPOINT and S3_BUCKET set. +// Kept separate from S3Enabled so a configured-but-inactive S3 backend can +// still be reached by `photo-api sync`. +func (c Config) S3Configured() bool { return c.S3Endpoint != "" && c.S3Bucket != "" } +// S3Enabled reports whether S3 is the backend serving requests. +func (c Config) S3Enabled() bool { + switch c.StorageBackend { + case "s3": + return true + case "local": + return false + default: // auto + return c.S3Configured() + } +} + func Load() (Config, error) { loadDotenv(".env") @@ -48,6 +67,7 @@ func Load() (Config, error) { DBType: strings.ToLower(env("DB_TYPE", "sqlite")), DatabaseURL: env("DATABASE_URL", ""), ViewTokenSecret: env("PHOTO_VIEW_SECRET", env("JWT_SECRET", "")), + StorageBackend: strings.ToLower(env("STORAGE_BACKEND", "auto")), StoragePath: env("STORAGE_PATH", "./data/photos"), S3Endpoint: env("S3_ENDPOINT", ""), S3Region: env("S3_REGION", "auto"), @@ -70,6 +90,14 @@ func Load() (Config, error) { if cfg.ViewTokenSecret == "" { return cfg, fmt.Errorf("PHOTO_VIEW_SECRET is required (or legacy JWT_SECRET as fallback)") } + switch cfg.StorageBackend { + case "auto", "local", "s3": + default: + return cfg, fmt.Errorf("STORAGE_BACKEND must be auto, local or s3, got %q", cfg.StorageBackend) + } + if cfg.StorageBackend == "s3" && !cfg.S3Configured() { + return cfg, fmt.Errorf("STORAGE_BACKEND=s3 requires S3_ENDPOINT and S3_BUCKET") + } return cfg, nil } diff --git a/photo-api/internal/httpapi/api_test.go b/photo-api/internal/httpapi/api_test.go index e27e006..60aa08e 100644 --- a/photo-api/internal/httpapi/api_test.go +++ b/photo-api/internal/httpapi/api_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "image" "image/color" @@ -38,10 +39,11 @@ const ( ) type testEnv struct { - handler http.Handler - db *store.DB - worker *worker.Worker - pg bool + handler http.Handler + db *store.DB + worker *worker.Worker + storagePath string + pg bool } // setup migrates a scratch DB (SQLite by default; Postgres when @@ -123,7 +125,13 @@ func setup(t *testing.T) *testEnv { } wrk := worker.New(db, st, nil, cfg.StoragePath, 1) srv := New(cfg, db, st, auth.NewVerifier(db), wrk) - return &testEnv{handler: srv.Handler(), db: db, worker: wrk, pg: cfg.DBType == "postgres"} + return &testEnv{ + handler: srv.Handler(), + db: db, + worker: wrk, + storagePath: cfg.StoragePath, + pg: cfg.DBType == "postgres", + } } // makeToken inserts a Better Auth session row for the user and returns a @@ -175,12 +183,16 @@ func decode[T any](t *testing.T, w *httptest.ResponseRecorder) T { return v } -func testJPEG(t *testing.T) []byte { +func testJPEG(t *testing.T) []byte { return testJPEGTinted(t, 128) } + +// testJPEGTinted varies the blue channel so tests that need two *different* +// images (duplicate detection) can get them without a fixture file. +func testJPEGTinted(t *testing.T, blue uint8) []byte { t.Helper() img := image.NewRGBA(image.Rect(0, 0, 800, 600)) for x := 0; x < 800; x += 10 { for y := 0; y < 600; y++ { - img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: 128, A: 255}) + img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: blue, A: 255}) } } var buf bytes.Buffer @@ -296,6 +308,130 @@ func uploadPhoto(t *testing.T, e *testEnv, admin, galleryID string, file []byte) return photos[0] } +// uploadFiles posts several parts in one request, the way the API allows even +// though the admin UI sends one file per request. +func uploadFiles(t *testing.T, e *testEnv, admin, galleryID string, files ...[]byte) []photoJSON { + t.Helper() + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + for i, f := range files { + fw, _ := mw.CreateFormFile("files", fmt.Sprintf("photo-%d.jpg", i)) + fw.Write(f) + } + mw.Close() + req := httptest.NewRequest("POST", "/api/photos/galleries/"+galleryID+"/photos", &buf) + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+admin) + w := httptest.NewRecorder() + e.handler.ServeHTTP(w, req) + if w.Code != 201 { + t.Fatalf("upload: %d %s", w.Code, w.Body.String()) + } + return decode[struct { + Photos []photoJSON `json:"photos"` + }](t, w).Photos +} + +// countOriginals is how many objects actually landed on disk for a gallery — +// a duplicate must not add one. +func countOriginals(t *testing.T, e *testEnv, galleryID string) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(e.storagePath, "galleries", galleryID, "orig")) + if os.IsNotExist(err) { + return 0 + } + if err != nil { + t.Fatal(err) + } + return len(entries) +} + +func TestUploadSkipsDuplicates(t *testing.T) { + e := setup(t) + admin := e.makeToken(t, uAdmin) + g := createGallery(t, e, admin, map[string]any{"title": "Dupes"}) + img := testJPEG(t) + + first := uploadPhoto(t, e, admin, g.ID, img) + if first.Duplicate { + t.Fatalf("first upload flagged as duplicate: %+v", first) + } + + // Same bytes again: echoed back as the existing photo, nothing stored. + second := uploadPhoto(t, e, admin, g.ID, img) + if !second.Duplicate { + t.Fatalf("second upload not flagged as duplicate: %+v", second) + } + if second.ID != first.ID { + t.Fatalf("duplicate should echo the existing photo: got %s, want %s", second.ID, first.ID) + } + photos, err := e.db.ListPhotos(context.Background(), g.ID, false) + if err != nil { + t.Fatal(err) + } + if len(photos) != 1 { + t.Fatalf("want 1 row after re-upload, got %d", len(photos)) + } + if n := countOriginals(t, e, g.ID); n != 1 { + t.Fatalf("want 1 stored original after re-upload, got %d", n) + } + if photos[0].Checksum == "" { + t.Fatal("checksum was not recorded") + } + + // A different image is unaffected and takes the next position. + other := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32)) + if other.Duplicate || other.ID == first.ID { + t.Fatalf("distinct image treated as duplicate: %+v", other) + } + if other.Position != 1 { + t.Fatalf("want position 1 for the second distinct photo, got %d", other.Position) + } + + // Duplicate scope is one gallery: the same bytes elsewhere upload normally. + g2 := createGallery(t, e, admin, map[string]any{"title": "Other gallery"}) + elsewhere := uploadPhoto(t, e, admin, g2.ID, img) + if elsewhere.Duplicate { + t.Fatalf("same image in another gallery must not be a duplicate: %+v", elsewhere) + } + if elsewhere.ID == first.ID { + t.Fatal("second gallery should get its own photo row") + } +} + +func TestUploadBatchContinuesPastDuplicate(t *testing.T) { + e := setup(t) + admin := e.makeToken(t, uAdmin) + g := createGallery(t, e, admin, map[string]any{"title": "Batch"}) + a, b, c := testJPEG(t), testJPEGTinted(t, 32), testJPEGTinted(t, 200) + + if got := uploadFiles(t, e, admin, g.ID, a); len(got) != 1 { + t.Fatalf("seed upload: %d photos", len(got)) + } + // b duplicates nothing, a is already there, c is new: the batch must not + // abort, and the duplicate must not consume a position. + batch := uploadFiles(t, e, admin, g.ID, b, a, c) + if len(batch) != 3 { + t.Fatalf("want 3 results, got %d", len(batch)) + } + if batch[0].Duplicate || !batch[1].Duplicate || batch[2].Duplicate { + t.Fatalf("duplicate flags: %v %v %v", batch[0].Duplicate, batch[1].Duplicate, batch[2].Duplicate) + } + if batch[0].Position != 1 || batch[2].Position != 2 { + t.Fatalf("positions should stay dense: %d, %d", batch[0].Position, batch[2].Position) + } + photos, err := e.db.ListPhotos(context.Background(), g.ID, false) + if err != nil { + t.Fatal(err) + } + if len(photos) != 3 { + t.Fatalf("want 3 rows, got %d", len(photos)) + } + if n := countOriginals(t, e, g.ID); n != 3 { + t.Fatalf("want 3 stored originals, got %d", n) + } +} + // processQueue runs the worker until the photo is ready or failed. func processQueue(t *testing.T, e *testEnv, photoID string) store.Photo { t.Helper() @@ -465,9 +601,9 @@ func TestReorderAndVisibilityUpdate(t *testing.T) { e := setup(t) admin := e.makeToken(t, uAdmin) g := createGallery(t, e, admin, map[string]any{"title": "Orden", "visibility": "public"}) - jpg := testJPEG(t) - p1 := uploadPhoto(t, e, admin, g.ID, jpg) - p2 := uploadPhoto(t, e, admin, g.ID, jpg) + // Two distinct images: the same bytes twice would be deduplicated. + p1 := uploadPhoto(t, e, admin, g.ID, testJPEG(t)) + p2 := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32)) if w := e.request(t, "PATCH", "/api/photos/galleries/"+g.ID+"/order", admin, map[string]any{"photoIds": []string{p2.ID, p1.ID}}); w.Code != 200 { diff --git a/photo-api/internal/httpapi/dto.go b/photo-api/internal/httpapi/dto.go index 9f40b31..b64e482 100644 --- a/photo-api/internal/httpapi/dto.go +++ b/photo-api/internal/httpapi/dto.go @@ -58,6 +58,10 @@ type photoJSON struct { LastError string `json:"lastError,omitempty"` // admin only CreatedAt string `json:"createdAt"` URLs photoURLs `json:"urls"` + // Duplicate marks an upload that was skipped because the gallery already + // held these bytes; the rest of the object describes the existing photo. + // Set by the upload handler only, never persisted. + Duplicate bool `json:"duplicate,omitempty"` } // viewTokenFor returns the token to embed in a gallery's file URLs: none diff --git a/photo-api/internal/httpapi/photos.go b/photo-api/internal/httpapi/photos.go index 1709827..2cdd607 100644 --- a/photo-api/internal/httpapi/photos.go +++ b/photo-api/internal/httpapi/photos.go @@ -12,6 +12,7 @@ import ( "time" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/store" ) @@ -20,6 +21,10 @@ import ( // Each file is sniffed by magic bytes (client filename/Content-Type are // untrusted, same policy as /api/media/upload), stored as the untouched // original, and queued for variant processing. +// +// A file whose bytes are already in this gallery is not stored a second time: +// the incoming copy is discarded, the existing photo is echoed back with +// "duplicate": true, and the rest of the batch carries on. func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.User) { galleryID := r.PathValue("id") g, err := s.db.GetGallery(r.Context(), galleryID) @@ -57,7 +62,7 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use part.Close() continue } - photo, uploadErr := s.saveUpload(r, g, part, position, maxFile) + photo, duplicate, uploadErr := s.saveUpload(r, g, part, position, maxFile) part.Close() if uploadErr != nil { // One bad file fails the request explicitly rather than silently @@ -65,7 +70,12 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use writeError(w, uploadErr.status, uploadErr.msg) return } - created = append(created, s.photoToJSON(photo, s.viewTokenFor(g), true)) + out := s.photoToJSON(photo, s.viewTokenFor(g), true) + out.Duplicate = duplicate + created = append(created, out) + if duplicate { + continue // nothing was stored, so the next file keeps this position + } position++ } @@ -82,51 +92,67 @@ type uploadError struct { msg string } -func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, *uploadError) { +// saveUpload stores one part. The bool it returns reports a duplicate: the +// gallery already holds these bytes, so nothing was written and the photo +// returned is the existing one. +func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, bool, *uploadError) { head := make([]byte, 16) n, err := io.ReadFull(part, head) if err != nil && err != io.ErrUnexpectedEOF { - return store.Photo{}, &uploadError{http.StatusBadRequest, "Could not read file"} + return store.Photo{}, false, &uploadError{http.StatusBadRequest, "Could not read file"} } head = head[:n] contentType, ext, ok, reason := imaging.Sniff(head) if !ok { - return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType, reason} + return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType, reason} } if contentType == "image/heic" && imaging.DetectHeicConverter(s.cfg.HeicConverter) == nil { - return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType, + return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType, "HEIC uploads need an image converter on the server (install libvips-tools); please upload JPEG instead"} } // Spool to a temp file to learn the size before handing to storage - // (S3 wants a length; local rename wants a file anyway). + // (S3 wants a length; local rename wants a file anyway). The same pass + // hashes the bytes for the duplicate check below. tmp, err := os.CreateTemp(s.cfg.StoragePath, ".incoming-*") if err != nil { log.Printf("Error: %v", err) - return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"} + return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"} } defer os.Remove(tmp.Name()) defer tmp.Close() - size, err := io.Copy(tmp, io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1))) + hasher := checksum.New() + size, err := io.Copy(io.MultiWriter(tmp, hasher), + io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1))) if err != nil { log.Printf("Error: %v", err) - return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"} + return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"} } if size > maxFile { - return store.Photo{}, &uploadError{http.StatusRequestEntityTooLarge, + return store.Photo{}, false, &uploadError{http.StatusRequestEntityTooLarge, fmt.Sprintf("File exceeds the %d MB limit", s.cfg.MaxUploadMB)} } if _, err := tmp.Seek(0, io.SeekStart); err != nil { log.Printf("Error: %v", err) - return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"} + return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"} + } + sum := checksum.Format(hasher) + + // Duplicate check before anything is written: the temp copy is dropped by + // the deferred Remove, no object is stored and no row is inserted. + if existing, err := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); err == nil { + return existing, true, nil + } else if err != store.ErrNotFound { + log.Printf("Error: %v", err) + return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"} } photoID := newID() key := fmt.Sprintf("galleries/%s/orig/%s%s", g.ID, photoID, ext) if err := s.storage.Put(r.Context(), key, tmp, size, contentType); err != nil { log.Printf("Error: %v", err) - return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"} + return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"} } now := time.Now() @@ -142,13 +168,22 @@ func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Pa NextAttemptAt: now, CreatedAt: now, UpdatedAt: now, + Checksum: sum, } if err := s.db.InsertPhoto(r.Context(), photo); err != nil { s.storage.Delete(r.Context(), key) + // A concurrent upload of the same bytes won the race between the + // check above and this insert; the unique index caught it, so report + // the winner as the duplicate instead of failing. + if store.IsUniqueViolation(err) { + if existing, findErr := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); findErr == nil { + return existing, true, nil + } + } log.Printf("Error: %v", err) - return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"} + return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"} } - return photo, nil + return photo, false, nil } type reorderBody struct { diff --git a/photo-api/internal/photosync/photosync.go b/photo-api/internal/photosync/photosync.go new file mode 100644 index 0000000..9fc52af --- /dev/null +++ b/photo-api/internal/photosync/photosync.go @@ -0,0 +1,229 @@ +// Package photosync copies the photo library between the local-disk and the +// S3 backend so the active one (STORAGE_BACKEND) can be switched without +// losing photos. Both backends must be configured in photo-api/.env for a +// sync to run — the direction picks which is the source. +// +// The photos_photos rows are the inventory: every row contributes its +// original key and, once the worker has processed it, its thumb and preview +// key. Objects on disk or in the bucket with no row (worker scratch dirs, +// leftovers of deleted galleries) are deliberately not copied. +// +// Sync only ever writes to the destination. The source is left untouched, so +// a sync is repeatable, safe to interrupt, and leaves the old backend as a +// fallback until it is cleaned up by hand. +package photosync + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + "sync/atomic" + + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/config" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/store" +) + +// Direction is which way objects move. +type Direction string + +const ( + ToS3 Direction = "to-s3" + ToLocal Direction = "to-local" +) + +// ParseDirection accepts the short and the spelled-out form. +func ParseDirection(s string) (Direction, error) { + switch s { + case "to-s3", "local-to-s3": + return ToS3, nil + case "to-local", "s3-to-local": + return ToLocal, nil + default: + return "", fmt.Errorf("unknown direction %q (expected to-s3 or to-local)", s) + } +} + +type Options struct { + Direction Direction + // GalleryID limits the sync to one gallery; empty means the whole library. + GalleryID string + // Concurrency is how many objects are copied at a time. + Concurrency int + // DryRun reports what would be copied without writing anything. + DryRun bool + // Overwrite re-copies objects that already exist on the destination with + // the same size (default: those are skipped, which makes reruns cheap). + Overwrite bool +} + +type Result struct { + Total int // objects in the inventory + Copied int // copied (or, with DryRun, would be copied) + Skipped int // already on the destination + Missing int // absent from the source — nothing to copy + Failed int // copy attempted and errored + Bytes int64 // bytes copied +} + +// Run copies the library in the requested direction. It returns a Result even +// on error, and an error if any object failed. +func Run(ctx context.Context, cfg config.Config, db *store.DB, opts Options) (Result, error) { + if !cfg.S3Configured() { + return Result{}, errors.New("sync needs both backends configured: set S3_ENDPOINT, S3_BUCKET, " + + "S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY (next to STORAGE_PATH) in photo-api/.env") + } + if opts.Concurrency < 1 { + opts.Concurrency = 4 + } + + local, err := storage.NewLocal(cfg.StoragePath) + if err != nil { + return Result{}, fmt.Errorf("local storage: %w", err) + } + s3, err := storage.NewS3(cfg) + if err != nil { + return Result{}, fmt.Errorf("s3 storage: %w", err) + } + src, dst := local, s3 + srcName, dstName := "local disk "+cfg.StoragePath, "s3 bucket "+cfg.S3Bucket + if opts.Direction == ToLocal { + src, dst = s3, local + srcName, dstName = dstName, srcName + } + + objects, err := db.AllPhotoObjects(ctx, opts.GalleryID) + if err != nil { + return Result{}, fmt.Errorf("list photo objects: %w", err) + } + + prefix := "" + if opts.DryRun { + prefix = "[dry-run] " + } + log.Printf("sync: %s%s → %s: %d objects, concurrency %d", + prefix, srcName, dstName, len(objects), opts.Concurrency) + + var ( + mu sync.Mutex + res = Result{Total: len(objects)} + done int64 + jobs = make(chan store.PhotoObject) + wg sync.WaitGroup + ) + for i := 0; i < opts.Concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for o := range jobs { + if ctx.Err() != nil { + return + } + act, size, err := copyObject(ctx, src, dst, o, opts) + n := atomic.AddInt64(&done, 1) + + mu.Lock() + switch { + case err != nil: + res.Failed++ + log.Printf("sync: [%d/%d] FAILED %s (photo %s %s): %v", + n, len(objects), o.Key, o.PhotoID, o.Variant, err) + case act == actionCopied: + res.Copied++ + res.Bytes += size + log.Printf("sync: [%d/%d] %scopied %s (%s)", n, len(objects), prefix, o.Key, humanBytes(size)) + case act == actionMissing: + res.Missing++ + log.Printf("sync: [%d/%d] MISSING on source, skipped: %s (photo %s %s)", + n, len(objects), o.Key, o.PhotoID, o.Variant) + default: + res.Skipped++ + } + mu.Unlock() + } + }() + } + for _, o := range objects { + select { + case jobs <- o: + case <-ctx.Done(): + } + if ctx.Err() != nil { + break + } + } + close(jobs) + wg.Wait() + + log.Printf("sync: %sdone — %d copied (%s), %d already present, %d missing on source, %d failed", + prefix, res.Copied, humanBytes(res.Bytes), res.Skipped, res.Missing, res.Failed) + + if err := ctx.Err(); err != nil { + return res, fmt.Errorf("interrupted after %d/%d objects: %w", res.Copied+res.Skipped, res.Total, err) + } + if res.Failed > 0 { + return res, fmt.Errorf("%d of %d objects failed to copy (rerun to retry; already-copied objects are skipped)", + res.Failed, res.Total) + } + return res, nil +} + +type action int + +const ( + actionCopied action = iota + actionSkipped + actionMissing +) + +func copyObject(ctx context.Context, src, dst storage.Storage, o store.PhotoObject, opts Options) (action, int64, error) { + srcSize, err := src.Stat(ctx, o.Key) + if errors.Is(err, storage.ErrNotExist) { + return actionMissing, 0, nil + } + if err != nil { + return actionSkipped, 0, fmt.Errorf("stat source: %w", err) + } + if !opts.Overwrite { + if dstSize, err := dst.Stat(ctx, o.Key); err == nil && dstSize == srcSize { + return actionSkipped, 0, nil + } else if err != nil && !errors.Is(err, storage.ErrNotExist) { + return actionSkipped, 0, fmt.Errorf("stat destination: %w", err) + } + } + if opts.DryRun { + return actionCopied, srcSize, nil + } + + r, size, err := src.Open(ctx, o.Key) + if err != nil { + return actionSkipped, 0, fmt.Errorf("read source: %w", err) + } + defer r.Close() + if size <= 0 { + size = srcSize // local Open reports the stat size; be defensive anyway + } + contentType := o.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + if err := dst.Put(ctx, o.Key, r, size, contentType); err != nil { + return actionSkipped, 0, fmt.Errorf("write destination: %w", err) + } + return actionCopied, size, nil +} + +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + v, exp := float64(n), 0 + for v >= unit && exp < 4 { + v /= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", v, "KMGT"[exp-1]) +} diff --git a/photo-api/internal/photosync/photosync_test.go b/photo-api/internal/photosync/photosync_test.go new file mode 100644 index 0000000..6b2a6b6 --- /dev/null +++ b/photo-api/internal/photosync/photosync_test.go @@ -0,0 +1,135 @@ +package photosync + +import ( + "context" + "io" + "path/filepath" + "strings" + "testing" + + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/store" +) + +// Two local backends stand in for the real pair: copyObject only talks to the +// storage.Storage interface, so the S3 side needs no fake here. +func backends(t *testing.T) (src, dst storage.Storage, srcRoot string) { + t.Helper() + dir := t.TempDir() + srcRoot = filepath.Join(dir, "src") + src, err := storage.NewLocal(srcRoot) + if err != nil { + t.Fatal(err) + } + dst, err = storage.NewLocal(filepath.Join(dir, "dst")) + if err != nil { + t.Fatal(err) + } + return src, dst, srcRoot +} + +func put(t *testing.T, s storage.Storage, key, body string) { + t.Helper() + if err := s.Put(context.Background(), key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil { + t.Fatal(err) + } +} + +const testKey = "galleries/g1/original/p1.jpg" + +func testObject() store.PhotoObject { + return store.PhotoObject{PhotoID: "p1", GalleryID: "g1", Variant: "original", Key: testKey, ContentType: "image/jpeg"} +} + +func TestCopyObject(t *testing.T) { + ctx := context.Background() + + t.Run("copies to destination", func(t *testing.T) { + src, dst, _ := backends(t) + put(t, src, testKey, "hello-photo") + + act, size, err := copyObject(ctx, src, dst, testObject(), Options{}) + if err != nil || act != actionCopied || size != 11 { + t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err) + } + if got, err := dst.Stat(ctx, testKey); err != nil || got != 11 { + t.Fatalf("destination stat: (%d, %v)", got, err) + } + }) + + t.Run("skips same-size object already present", func(t *testing.T) { + src, dst, _ := backends(t) + put(t, src, testKey, "hello-photo") + put(t, dst, testKey, "hello-photo") + + act, _, err := copyObject(ctx, src, dst, testObject(), Options{}) + if err != nil || act != actionSkipped { + t.Fatalf("got (%v, %v), want (skipped, nil)", act, err) + } + }) + + t.Run("overwrite re-copies", func(t *testing.T) { + src, dst, _ := backends(t) + put(t, src, testKey, "new-content") + put(t, dst, testKey, "old-content") + + act, _, err := copyObject(ctx, src, dst, testObject(), Options{Overwrite: true}) + if err != nil || act != actionCopied { + t.Fatalf("got (%v, %v), want (copied, nil)", act, err) + } + r, _, err := dst.Open(ctx, testKey) + if err != nil { + t.Fatal(err) + } + defer r.Close() + body, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if string(body) != "new-content" { + t.Fatalf("destination body = %q, want new-content", body) + } + }) + + t.Run("reports objects missing on the source", func(t *testing.T) { + src, dst, _ := backends(t) + + act, _, err := copyObject(ctx, src, dst, testObject(), Options{}) + if err != nil || act != actionMissing { + t.Fatalf("got (%v, %v), want (missing, nil)", act, err) + } + if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist { + t.Fatalf("destination stat err = %v, want ErrNotExist", err) + } + }) + + t.Run("dry run writes nothing", func(t *testing.T) { + src, dst, _ := backends(t) + put(t, src, testKey, "hello-photo") + + act, size, err := copyObject(ctx, src, dst, testObject(), Options{DryRun: true}) + if err != nil || act != actionCopied || size != 11 { + t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err) + } + if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist { + t.Fatalf("destination stat err = %v, want ErrNotExist", err) + } + }) +} + +func TestParseDirection(t *testing.T) { + for in, want := range map[string]Direction{ + "to-s3": ToS3, + "local-to-s3": ToS3, + "to-local": ToLocal, + "s3-to-local": ToLocal, + } { + got, err := ParseDirection(in) + if err != nil || got != want { + t.Errorf("ParseDirection(%q) = (%v, %v), want %v", in, got, err, want) + } + } + if _, err := ParseDirection("sideways"); err == nil { + t.Error("ParseDirection(\"sideways\") should fail") + } +} diff --git a/photo-api/internal/photosync/run_test.go b/photo-api/internal/photosync/run_test.go new file mode 100644 index 0000000..73aca30 --- /dev/null +++ b/photo-api/internal/photosync/run_test.go @@ -0,0 +1,262 @@ +package photosync + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/config" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage" + "git.azzamo.net/Michilis/Spanglish/photo-api/internal/store" +) + +// stubS3 is a minimal path-style S3 (PUT/HEAD/GET on /bucket/key) so the sync +// can be driven end to end over the real aws-sdk client. +type stubS3 struct { + mu sync.Mutex + objects map[string][]byte + puts int +} + +func newStubS3(t *testing.T) (*stubS3, string) { + t.Helper() + s := &stubS3{objects: map[string][]byte{}} + srv := httptest.NewServer(s) + t.Cleanup(srv.Close) + return s, srv.URL +} + +func (s *stubS3) get(key string) ([]byte, bool) { + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.objects[key] + return b, ok +} + +func (s *stubS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/test-bucket/") + switch r.Method { + case http.MethodPut: + body, err := readS3Body(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + s.mu.Lock() + s.objects[key] = body + s.puts++ + s.mu.Unlock() + w.WriteHeader(http.StatusOK) + case http.MethodHead: + body, ok := s.get(key) + if !ok { + w.WriteHeader(http.StatusNotFound) // a HEAD carries no error body + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + case http.MethodGet: + body, ok := s.get(key) + if !ok { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, `NoSuchKey`) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.Write(body) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +// readS3Body undoes the SDK's aws-chunked framing when it streams with a +// trailing checksum (what it does for the non-seekable S3-to-S3 style reader). +func readS3Body(r *http.Request) ([]byte, error) { + raw, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + if !strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") { + return raw, nil + } + var out []byte + rest := raw + for { + nl := strings.Index(string(rest), "\r\n") + if nl < 0 { + return out, nil + } + header := string(rest[:nl]) + rest = rest[nl+2:] + size, err := strconv.ParseInt(strings.SplitN(header, ";", 2)[0], 16, 64) + if err != nil || size == 0 { + return out, nil // trailer section or malformed: body is complete + } + if int64(len(rest)) < size { + return nil, fmt.Errorf("truncated aws-chunked body") + } + out = append(out, rest[:size]...) + rest = rest[size:] + if len(rest) >= 2 { + rest = rest[2:] // chunk CRLF + } + } +} + +// syncEnv seeds a scratch SQLite DB with one gallery holding one ready photo +// (original + thumb + preview) and returns a config wired to the stub bucket. +func syncEnv(t *testing.T) (config.Config, *store.DB, *stubS3, []string) { + t.Helper() + dir := t.TempDir() + stub, endpoint := newStubS3(t) + cfg := config.Config{ + DBType: "sqlite", + DatabaseURL: filepath.Join(dir, "test.db"), + ViewTokenSecret: "test-secret", + StoragePath: filepath.Join(dir, "photos"), + S3Endpoint: endpoint, + S3Region: "auto", + S3Bucket: "test-bucket", + S3AccessKeyID: "key", + S3SecretKey: "secret", + S3ForcePathStyle: true, + } + db, err := store.Open(cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + ctx := context.Background() + if err := db.Migrate(ctx); err != nil { + t.Fatal(err) + } + + const ( + galleryID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + photoID = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + ) + now := time.Now() + if err := db.CreateGallery(ctx, store.Gallery{ + ID: galleryID, Slug: "trip", Title: "Trip", Visibility: store.VisibilityPublic, + ShareToken: "tok", CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + origKey := "galleries/" + galleryID + "/original/" + photoID + ".jpg" + thumbKey := "galleries/" + galleryID + "/thumb/" + photoID + ".jpg" + previewKey := "galleries/" + galleryID + "/preview/" + photoID + ".jpg" + if err := db.InsertPhoto(ctx, store.Photo{ + ID: photoID, GalleryID: galleryID, OriginalKey: origKey, ContentType: "image/jpeg", + SizeBytes: 11, NextAttemptAt: now, CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := db.MarkPhotoReady(ctx, photoID, thumbKey, previewKey, 100, 80, now); err != nil { + t.Fatal(err) + } + return cfg, db, stub, []string{origKey, thumbKey, previewKey} +} + +func TestRunToS3(t *testing.T) { + ctx := context.Background() + cfg, db, stub, keys := syncEnv(t) + + src, err := storage.NewLocal(cfg.StoragePath) + if err != nil { + t.Fatal(err) + } + for i, k := range keys { + put(t, src, k, fmt.Sprintf("photo-bytes-%d", i)) + } + + res, err := Run(ctx, cfg, db, Options{Direction: ToS3, Concurrency: 2}) + if err != nil { + t.Fatal(err) + } + if res.Total != 3 || res.Copied != 3 || res.Skipped != 0 || res.Failed != 0 || res.Missing != 0 { + t.Fatalf("first run = %+v, want 3 total / 3 copied", res) + } + for i, k := range keys { + body, ok := stub.get(k) + if !ok { + t.Fatalf("%s not uploaded", k) + } + if want := fmt.Sprintf("photo-bytes-%d", i); string(body) != want { + t.Errorf("%s = %q, want %q", k, body, want) + } + } + + // Rerunning is a no-op: everything is already there at the same size. + res, err = Run(ctx, cfg, db, Options{Direction: ToS3}) + if err != nil { + t.Fatal(err) + } + if res.Copied != 0 || res.Skipped != 3 { + t.Fatalf("rerun = %+v, want 0 copied / 3 skipped", res) + } + + // --overwrite re-uploads them. + before := stub.puts + res, err = Run(ctx, cfg, db, Options{Direction: ToS3, Overwrite: true}) + if err != nil { + t.Fatal(err) + } + if res.Copied != 3 || stub.puts != before+3 { + t.Fatalf("overwrite run = %+v, puts %d → %d", res, before, stub.puts) + } +} + +func TestRunToLocal(t *testing.T) { + ctx := context.Background() + cfg, db, stub, keys := syncEnv(t) + for i, k := range keys { + stub.objects[k] = []byte(fmt.Sprintf("s3-bytes-%d", i)) + } + + res, err := Run(ctx, cfg, db, Options{Direction: ToLocal}) + if err != nil { + t.Fatal(err) + } + if res.Total != 3 || res.Copied != 3 || res.Failed != 0 { + t.Fatalf("run = %+v, want 3 total / 3 copied", res) + } + for i, k := range keys { + body, err := os.ReadFile(filepath.Join(cfg.StoragePath, k)) + if err != nil { + t.Fatalf("read %s: %v", k, err) + } + if want := fmt.Sprintf("s3-bytes-%d", i); string(body) != want { + t.Errorf("%s = %q, want %q", k, body, want) + } + } +} + +func TestRunMissingOnSource(t *testing.T) { + cfg, db, _, _ := syncEnv(t) + // Nothing on local disk: every object is reported missing, none fail. + res, err := Run(context.Background(), cfg, db, Options{Direction: ToS3}) + if err != nil { + t.Fatal(err) + } + if res.Missing != 3 || res.Copied != 0 || res.Failed != 0 { + t.Fatalf("run = %+v, want 3 missing", res) + } +} + +func TestRunRequiresS3Config(t *testing.T) { + cfg, db, _, _ := syncEnv(t) + cfg.S3Endpoint, cfg.S3Bucket = "", "" + if _, err := Run(context.Background(), cfg, db, Options{Direction: ToS3}); err == nil { + t.Fatal("sync without S3 configured should fail") + } +} diff --git a/photo-api/internal/storage/local.go b/photo-api/internal/storage/local.go index 02f9168..636bb79 100644 --- a/photo-api/internal/storage/local.go +++ b/photo-api/internal/storage/local.go @@ -17,6 +17,12 @@ type local struct { root string } +// NewLocal builds the local-disk backend explicitly, whichever backend is +// active — `photo-api sync` needs both sides at once. +func NewLocal(root string) (Storage, error) { + return newLocal(root) +} + func newLocal(root string) (*local, error) { if err := os.MkdirAll(root, 0o755); err != nil { return nil, fmt.Errorf("create storage dir %s: %w", root, err) @@ -74,6 +80,21 @@ func (l *local) Open(_ context.Context, key string) (io.ReadCloser, int64, error return f, info.Size(), nil } +func (l *local) Stat(_ context.Context, key string) (int64, error) { + p, err := l.path(key) + if err != nil { + return 0, err + } + info, err := os.Stat(p) + if os.IsNotExist(err) { + return 0, ErrNotExist + } + if err != nil { + return 0, err + } + return info.Size(), nil +} + func (l *local) Delete(_ context.Context, key string) error { p, err := l.path(key) if err != nil { diff --git a/photo-api/internal/storage/s3.go b/photo-api/internal/storage/s3.go index ca05239..d9e1009 100644 --- a/photo-api/internal/storage/s3.go +++ b/photo-api/internal/storage/s3.go @@ -2,14 +2,19 @@ package storage import ( "context" + "errors" "fmt" "io" + "net/http" "time" "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "git.azzamo.net/Michilis/Spanglish/photo-api/internal/config" ) @@ -23,6 +28,12 @@ type s3Store struct { bucket string } +// NewS3 builds the S3 backend explicitly, whichever backend is active — +// `photo-api sync` needs both sides at once. +func NewS3(cfg config.Config) (Storage, error) { + return newS3(cfg) +} + func newS3(cfg config.Config) (*s3Store, error) { awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), awsconfig.WithRegion(cfg.S3Region), @@ -65,6 +76,28 @@ func (s *s3Store) Open(ctx context.Context, key string) (io.ReadCloser, int64, e return out.Body, aws.ToInt64(out.ContentLength), nil } +func (s *s3Store) Stat(ctx context.Context, key string) (int64, error) { + out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err != nil { + // Compatibles differ: some answer NotFound, some NoSuchKey, and a + // HEAD carries no body to parse, so fall back to the status code. + var notFound *types.NotFound + var noKey *types.NoSuchKey + var apiErr smithy.APIError + var respErr *awshttp.ResponseError + if errors.As(err, ¬Found) || errors.As(err, &noKey) || + (errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey")) || + (errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusNotFound) { + return 0, ErrNotExist + } + return 0, err + } + return aws.ToInt64(out.ContentLength), nil +} + func (s *s3Store) Delete(ctx context.Context, key string) error { _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(s.bucket), diff --git a/photo-api/internal/storage/storage.go b/photo-api/internal/storage/storage.go index 37e4b01..58bf75c 100644 --- a/photo-api/internal/storage/storage.go +++ b/photo-api/internal/storage/storage.go @@ -7,6 +7,7 @@ package storage import ( "context" "errors" + "fmt" "io" "time" @@ -17,16 +18,30 @@ import ( // callers then stream the object through the API instead. var ErrNoPresign = errors.New("presigned URLs not supported") +// ErrNotExist is what Stat reports for a missing object on either backend. +var ErrNotExist = errors.New("object does not exist") + type Storage interface { Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error Open(ctx context.Context, key string) (io.ReadCloser, int64, error) + // Stat returns the object's size without reading it, or ErrNotExist. + Stat(ctx context.Context, key string) (int64, error) Delete(ctx context.Context, key string) error PresignGet(ctx context.Context, key, downloadFilename, contentType string, expiry time.Duration) (string, error) } +// New builds the backend that serves requests (see config.S3Enabled). func New(cfg config.Config) (Storage, error) { if cfg.S3Enabled() { - return newS3(cfg) + return NewS3(cfg) } - return newLocal(cfg.StoragePath) + return NewLocal(cfg.StoragePath) +} + +// Name describes a backend for log lines. +func Name(cfg config.Config, s Storage) string { + if _, ok := s.(*s3Store); ok { + return fmt.Sprintf("S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint) + } + return fmt.Sprintf("local disk at %s", cfg.StoragePath) } diff --git a/photo-api/internal/store/db.go b/photo-api/internal/store/db.go index cf22ae5..7e1aa48 100644 --- a/photo-api/internal/store/db.go +++ b/photo-api/internal/store/db.go @@ -54,6 +54,19 @@ func Open(cfg config.Config) (*DB, error) { } } +// IsUniqueViolation reports whether err is a duplicate-key error. The two +// drivers wrap it in unrelated types (pgconn.PgError vs sqlite.Error), and +// neither is worth importing here just for one check, so this matches on the +// message: pgx renders "(SQLSTATE 23505)", modernc/sqlite "UNIQUE constraint +// failed: ...". +func IsUniqueViolation(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "SQLSTATE 23505") || strings.Contains(msg, "UNIQUE constraint failed") +} + // Rebind converts ?-style placeholders to $n for Postgres. Queries in this // package never contain literal question marks in strings. func (db *DB) Rebind(query string) string { diff --git a/photo-api/internal/store/photos.go b/photo-api/internal/store/photos.go index cad6391..b563aed 100644 --- a/photo-api/internal/store/photos.go +++ b/photo-api/internal/store/photos.go @@ -29,14 +29,17 @@ type Photo struct { LastError string CreatedAt time.Time UpdatedAt time.Time + // Checksum is the sha256 of the original bytes, empty when the row has + // not been hashed yet (uploaded before duplicate detection existed). + Checksum string } const photoColumns = `id, gallery_id, position, original_key, original_filename, content_type, size_bytes, width, height, thumb_key, preview_key, taken_at, status, attempts, next_attempt_at, - last_error, created_at, updated_at` + last_error, created_at, updated_at, checksum` func scanPhoto(s scanner) (Photo, error) { - var v [18]any + var v [19]any dest := make([]any, len(v)) for i := range v { dest[i] = &v[i] @@ -63,20 +66,82 @@ func scanPhoto(s scanner) (Photo, error) { LastError: asString(v[15]), CreatedAt: asTime(v[16]), UpdatedAt: asTime(v[17]), + Checksum: asString(v[18]), }, nil } +// InsertPhoto returns an error satisfying IsUniqueViolation when the gallery +// already holds a photo with the same checksum; callers treat that as a +// duplicate rather than a failure. func (db *DB) InsertPhoto(ctx context.Context, p Photo) error { _, err := db.ExecContext(ctx, db.Rebind(` INSERT INTO photos_photos (id, gallery_id, position, original_key, original_filename, content_type, size_bytes, - status, attempts, next_attempt_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`), + status, attempts, next_attempt_at, created_at, updated_at, checksum) + VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?, ?)`), p.ID, p.GalleryID, p.Position, p.OriginalKey, nullable(p.OriginalFilename), p.ContentType, - p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt)) + p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt), + nullable(p.Checksum)) return err } +// FindPhotoByChecksum looks for an existing photo with the same content in one +// gallery — the duplicate check the upload path runs before storing anything. +// Duplicate scope is per gallery: the same image in another gallery keeps its +// own row and its own stored object. +func (db *DB) FindPhotoByChecksum(ctx context.Context, galleryID, checksum string) (Photo, error) { + if checksum == "" { + return Photo{}, ErrNotFound // never match the not-yet-hashed rows + } + row := db.QueryRowContext(ctx, db.Rebind( + "SELECT "+photoColumns+" FROM photos_photos WHERE gallery_id = ? AND checksum = ?"), + galleryID, checksum) + p, err := scanPhoto(row) + if errors.Is(err, sql.ErrNoRows) { + return Photo{}, ErrNotFound + } + return p, err +} + +// SetPhotoChecksum fills in the hash of an already-stored photo. Used by the +// backfill; returns a unique-violation error when the row turns out to +// duplicate one already hashed in the same gallery. +func (db *DB) SetPhotoChecksum(ctx context.Context, id, checksum string) error { + res, err := db.ExecContext(ctx, db.Rebind( + "UPDATE photos_photos SET checksum = ? WHERE id = ?"), checksum, id) + if err != nil { + return err + } + return errIfNoRows(res) +} + +// PhotosMissingChecksum lists rows still lacking a hash (optionally of one +// gallery only), oldest first so backfill keeps the earliest upload of a +// duplicate pair as the one that gets the checksum. +func (db *DB) PhotosMissingChecksum(ctx context.Context, galleryID string) ([]Photo, error) { + q := "SELECT " + photoColumns + " FROM photos_photos WHERE checksum IS NULL" + var args []any + if galleryID != "" { + q += " AND gallery_id = ?" + args = append(args, galleryID) + } + q += " ORDER BY gallery_id, position, created_at" + rows, err := db.QueryContext(ctx, db.Rebind(q), args...) + if err != nil { + return nil, err + } + defer rows.Close() + photos := []Photo{} + for rows.Next() { + p, err := scanPhoto(rows) + if err != nil { + return nil, err + } + photos = append(photos, p) + } + return photos, rows.Err() +} + func (db *DB) GetPhoto(ctx context.Context, id string) (Photo, error) { row := db.QueryRowContext(ctx, db.Rebind("SELECT "+photoColumns+" FROM photos_photos WHERE id = ?"), id) @@ -169,6 +234,64 @@ func (db *DB) PhotoKeys(ctx context.Context, galleryID string) ([]string, error) return keys, rows.Err() } +// PhotoObject is one stored object: a photo variant plus the content type it +// should be written with. Used by the storage sync, which treats the rows as +// the inventory of what exists. +type PhotoObject struct { + PhotoID string + GalleryID string + Variant string // original | thumb | preview + Key string + ContentType string +} + +// AllPhotoObjects lists every object of every photo (optionally of one +// gallery only), oldest first. Variant keys are empty until the worker has +// processed the photo; those are left out. +func (db *DB) AllPhotoObjects(ctx context.Context, galleryID string) ([]PhotoObject, error) { + q := `SELECT id, gallery_id, content_type, original_key, thumb_key, preview_key + FROM photos_photos` + var args []any + if galleryID != "" { + q += " WHERE gallery_id = ?" + args = append(args, galleryID) + } + q += " ORDER BY gallery_id, position, created_at" + rows, err := db.QueryContext(ctx, db.Rebind(q), args...) + if err != nil { + return nil, err + } + defer rows.Close() + + objects := []PhotoObject{} + for rows.Next() { + var id, galID, ctype, orig, thumb, preview any + if err := rows.Scan(&id, &galID, &ctype, &orig, &thumb, &preview); err != nil { + return nil, err + } + variants := []struct { + name, key, contentType string + }{ + {"original", asString(orig), asString(ctype)}, + {"thumb", asString(thumb), "image/jpeg"}, + {"preview", asString(preview), "image/jpeg"}, + } + for _, v := range variants { + if v.key == "" { + continue + } + objects = append(objects, PhotoObject{ + PhotoID: asString(id), + GalleryID: asString(galID), + Variant: v.name, + Key: v.key, + ContentType: v.contentType, + }) + } + } + return objects, rows.Err() +} + // ClaimNextPhoto picks the oldest due queued/failed photo and marks it // processing. Optimistic claim (RowsAffected check) works identically on // Postgres and SQLite; returns ErrNotFound when the queue is empty. diff --git a/photo-api/migrations/0002_photo_checksum.pg.sql b/photo-api/migrations/0002_photo_checksum.pg.sql new file mode 100644 index 0000000..c11c4db --- /dev/null +++ b/photo-api/migrations/0002_photo_checksum.pg.sql @@ -0,0 +1,8 @@ +-- Content hash of the stored original, for per-gallery duplicate detection on +-- upload. NULL means "not hashed yet" — rows predating this migration, until +-- `photo-api backfill-checksums` runs. NULLs are distinct under a unique +-- index, so those rows never collide with each other. +-- (No semicolons in these comments: the migration runner splits on them.) +ALTER TABLE photos_photos ADD COLUMN IF NOT EXISTS checksum varchar(64); + +CREATE UNIQUE INDEX IF NOT EXISTS photos_photos_gallery_checksum_idx ON photos_photos (gallery_id, checksum); diff --git a/photo-api/migrations/0002_photo_checksum.sqlite.sql b/photo-api/migrations/0002_photo_checksum.sqlite.sql new file mode 100644 index 0000000..36ee0d1 --- /dev/null +++ b/photo-api/migrations/0002_photo_checksum.sqlite.sql @@ -0,0 +1,8 @@ +-- Content hash of the stored original, for per-gallery duplicate detection on +-- upload. NULL means "not hashed yet" — rows predating this migration, until +-- `photo-api backfill-checksums` runs. NULLs are distinct under a unique +-- index, so those rows never collide with each other. +-- (No semicolons in these comments: the migration runner splits on them.) +ALTER TABLE photos_photos ADD COLUMN checksum text; + +CREATE UNIQUE INDEX IF NOT EXISTS photos_photos_gallery_checksum_idx ON photos_photos (gallery_id, checksum); -- 2.54.0 From fa8686276d129256ef60116e2a0f88b3b826d85e Mon Sep 17 00:00:00 2001 From: Michilis Date: Wed, 5 Aug 2026 22:05:33 +0000 Subject: [PATCH 2/3] Share gallery layout with loading skeletons and add download progress. Keep the public gallery hero/masonry geometry stable across route and client loading, and show busy state while photos download. Co-authored-by: Cursor --- .../(public)/events/[id]/gallery/loading.tsx | 6 + .../(public)/photos/[slug]/GalleryClient.tsx | 160 +++++++++--------- .../app/(public)/photos/[slug]/loading.tsx | 7 + frontend/src/components/Lightbox.tsx | 60 ++++++- .../src/components/gallery/GalleryLayout.tsx | 46 +++++ .../components/gallery/GallerySkeleton.tsx | 59 +++++++ frontend/src/components/gallery/PhotoTile.tsx | 100 +++++++++++ .../src/components/gallery/useDownloads.tsx | 148 ++++++++++++++++ frontend/src/components/ui/Skeleton.tsx | 14 +- frontend/src/components/ui/Spinner.tsx | 18 ++ 10 files changed, 524 insertions(+), 94 deletions(-) create mode 100644 frontend/src/app/(public)/events/[id]/gallery/loading.tsx create mode 100644 frontend/src/app/(public)/photos/[slug]/loading.tsx create mode 100644 frontend/src/components/gallery/GalleryLayout.tsx create mode 100644 frontend/src/components/gallery/GallerySkeleton.tsx create mode 100644 frontend/src/components/gallery/PhotoTile.tsx create mode 100644 frontend/src/components/gallery/useDownloads.tsx create mode 100644 frontend/src/components/ui/Spinner.tsx diff --git a/frontend/src/app/(public)/events/[id]/gallery/loading.tsx b/frontend/src/app/(public)/events/[id]/gallery/loading.tsx new file mode 100644 index 0000000..facfe07 --- /dev/null +++ b/frontend/src/app/(public)/events/[id]/gallery/loading.tsx @@ -0,0 +1,6 @@ +import GallerySkeleton from '@/components/gallery/GallerySkeleton'; + +// Shown while the server component fetches the event gallery. +export default function Loading() { + return ; +} diff --git a/frontend/src/app/(public)/photos/[slug]/GalleryClient.tsx b/frontend/src/app/(public)/photos/[slug]/GalleryClient.tsx index 101025e..ea39a09 100644 --- a/frontend/src/app/(public)/photos/[slug]/GalleryClient.tsx +++ b/frontend/src/app/(public)/photos/[slug]/GalleryClient.tsx @@ -7,11 +7,17 @@ import { useLanguage } from '@/context/LanguageContext'; import { useAuth } from '@/context/AuthContext'; import { photosApi, PhotoGallery, Photo } from '@/lib/api'; import Button from '@/components/ui/Button'; -import { ImageGridSkeleton } from '@/components/ui/Skeleton'; +import GallerySkeleton from '@/components/gallery/GallerySkeleton'; +import PhotoTile from '@/components/gallery/PhotoTile'; +import { + GalleryContainer, + GalleryHeroFrame, + MasonryGrid, +} from '@/components/gallery/GalleryLayout'; +import { useDownloads } from '@/components/gallery/useDownloads'; import Lightbox from '@/components/Lightbox'; import LoginModal from '@/components/LoginModal'; import { - ArrowDownTrayIcon, CalendarIcon, CameraIcon, LinkIcon, @@ -42,6 +48,17 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien const [denied, setDenied] = useState(null); const [lightboxIndex, setLightboxIndex] = useState(null); const [loginOpen, setLoginOpen] = useState(false); + const downloads = useDownloads(es); + const downloadLabels = { + download: es ? 'Descargar' : 'Download', + downloading: es ? 'Descargando…' : 'Downloading…', + }; + const downloadPhoto = (photo: Photo) => + downloads.start({ + id: photo.id, + url: photo.urls.original, + filename: photo.originalFilename, + }); // Server-rendered public galleries need no client fetch. Everything else // (link/ticket/private) is fetched here with the share token and/or the @@ -78,14 +95,10 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien }; }, [slug, eventSlug, shareToken, initial, authLoading, user?.id]); + // Mirrors the hero + masonry layout below, so the real page drops straight + // into the placeholder's geometry instead of replacing it. if (loading || (authLoading && !initial)) { - return ( -
-
- -
-
- ); + return ; } // Gate pages for restricted galleries. After a successful login in the @@ -254,47 +267,48 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien return (
- {/* Hero */} -
- {heroUrl && ( - <> - {/* eslint-disable-next-line @next/next/no-img-element */} - -
- + {/* Hero — same frame the skeleton renders (GalleryLayout). */} + + {/* eslint-disable-next-line @next/next/no-img-element */} + +
+ + ) : null + } + > +

+ {title} +

+ {description && ( +

{description}

)} -
-

- {title} -

- {description && ( -

{description}

+
+ + + {readyPhotos.length} {es ? 'fotos' : 'photos'} + + {gallery.event && ( + + + {eventTitle} + {eventDate && · {eventDate}} + )} -
- - - {readyPhotos.length} {es ? 'fotos' : 'photos'} - - {gallery.event && ( - - - {eventTitle} - {eventDate && · {eventDate}} - - )} -
-
+ {/* Masonry grid */} -
+ {readyPhotos.length === 0 ? (
@@ -303,46 +317,19 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien

) : ( -
+ {readyPhotos.map((photo, i) => ( -
setLightboxIndex(i)} - onKeyDown={(e) => { - if (e.key === 'Enter' || e.key === ' ') { - e.preventDefault(); - setLightboxIndex(i); - } - }} - className="group relative mb-2 md:mb-3 break-inside-avoid overflow-hidden rounded-xl bg-gray-100 cursor-pointer focus:outline-none focus:ring-2 focus:ring-primary-yellow" - style={ - photo.width && photo.height - ? { aspectRatio: `${photo.width} / ${photo.height}` } - : undefined - } - > - {/* eslint-disable-next-line @next/next/no-img-element */} - - + photo={photo} + eager={i < 8} + onOpen={() => setLightboxIndex(i)} + onDownload={() => downloadPhoto(photo)} + downloading={downloads.isPending(photo.id)} + labels={downloadLabels} + /> ))} -
+ )} {lightboxIndex !== null && ( @@ -351,9 +338,16 @@ export default function GalleryClient({ slug, eventSlug, initial }: GalleryClien index={lightboxIndex} onClose={() => setLightboxIndex(null)} onNavigate={setLightboxIndex} + onDownload={(it) => + downloads.start({ id: it.id, url: it.downloadUrl, filename: it.filename }) + } + downloadingId={ + lightboxItems.find((it) => downloads.isPending(it.id))?.id ?? null + } + downloadLabels={downloadLabels} /> )} -
+ {/* Call to action: send attendees to their dashboard, everyone else to the next event. Auth state comes from the same useAuth() the gate diff --git a/frontend/src/app/(public)/photos/[slug]/loading.tsx b/frontend/src/app/(public)/photos/[slug]/loading.tsx new file mode 100644 index 0000000..c5ac403 --- /dev/null +++ b/frontend/src/app/(public)/photos/[slug]/loading.tsx @@ -0,0 +1,7 @@ +import GallerySkeleton from '@/components/gallery/GallerySkeleton'; + +// Shown while the server component fetches the gallery, so the first paint is +// already the gallery's layout rather than an empty page. +export default function Loading() { + return ; +} diff --git a/frontend/src/components/Lightbox.tsx b/frontend/src/components/Lightbox.tsx index d77fe56..d9aa42f 100644 --- a/frontend/src/components/Lightbox.tsx +++ b/frontend/src/components/Lightbox.tsx @@ -7,6 +7,7 @@ import { ChevronRightIcon, ArrowDownTrayIcon, } from '@heroicons/react/24/outline'; +import Spinner from '@/components/ui/Spinner'; export interface LightboxItem { id: string; @@ -24,17 +25,35 @@ interface LightboxProps { onNavigate: (index: number) => void; /** Extra per-item action buttons rendered in the top bar (admin use). */ renderActions?: (item: LightboxItem, index: number) => React.ReactNode; + /** + * Handles the download in JS instead of navigating, so the button can show + * progress. Without it the button stays a plain . + */ + onDownload?: (item: LightboxItem) => void; + /** Id of the item currently downloading (pairs with onDownload). */ + downloadingId?: string | null; + downloadLabels?: { download: string; downloading: string }; } /** * Full-screen photo lightbox with keyboard and swipe navigation, in the * style of the admin gallery preview modal (fixed inset-0 bg-black/90). */ -export default function Lightbox({ items, index, onClose, onNavigate, renderActions }: LightboxProps) { +export default function Lightbox({ + items, + index, + onClose, + onNavigate, + renderActions, + onDownload, + downloadingId, + downloadLabels, +}: LightboxProps) { const touchStart = useRef<{ x: number; y: number } | null>(null); const activeThumbRef = useRef(null); const item = items[index]; const hasMultiple = items.length > 1; + const downloading = !!downloadingId && item?.id === downloadingId; const prev = useCallback(() => { onNavigate(index > 0 ? index - 1 : items.length - 1); @@ -98,14 +117,37 @@ export default function Lightbox({ items, index, onClose, onNavigate, renderActi className="absolute top-4 left-4 z-10 flex items-center gap-4" onClick={(e) => e.stopPropagation()} > - - - + {onDownload ? ( + + ) : ( + + + + )} {renderActions?.(item, index)}
diff --git a/frontend/src/components/gallery/GalleryLayout.tsx b/frontend/src/components/gallery/GalleryLayout.tsx new file mode 100644 index 0000000..37871f9 --- /dev/null +++ b/frontend/src/components/gallery/GalleryLayout.tsx @@ -0,0 +1,46 @@ +// Layout shell shared by the public gallery page (GalleryClient) and its +// loading placeholder (GallerySkeleton). Column count, gaps, radii and +// container padding are defined once here, so the skeleton's geometry can +// never drift from the grid it stands in for. + +export function GalleryHeroFrame({ + children, + backdrop, +}: { + children: React.ReactNode; + /** Cover image + scrim, absolutely positioned behind the text. */ + backdrop?: React.ReactNode; +}) { + return ( +
+ {backdrop} +
{children}
+
+ ); +} + +export function GalleryContainer({ children }: { children: React.ReactNode }) { + return
{children}
; +} + +export function MasonryGrid({ children }: { children: React.ReactNode }) { + return ( +
+ {children} +
+ ); +} + +// Per-tile geometry. Deliberately carries no background so callers can pick +// one (photo tiles: bg-gray-100, skeleton tiles: the skeleton surface) without +// two `bg-*` utilities fighting over CSS order. +export const masonryTileClass = 'mb-2 md:mb-3 break-inside-avoid overflow-hidden rounded-xl'; + +/** + * Reserves the tile's aspect ratio up front so the image can load into a box + * that is already the right size — late arrivals never reflow the columns. + * Returns undefined when the photo has no stored dimensions. + */ +export function aspectStyle(width?: number, height?: number): React.CSSProperties | undefined { + return width && height ? { aspectRatio: `${width} / ${height}` } : undefined; +} diff --git a/frontend/src/components/gallery/GallerySkeleton.tsx b/frontend/src/components/gallery/GallerySkeleton.tsx new file mode 100644 index 0000000..60eccc4 --- /dev/null +++ b/frontend/src/components/gallery/GallerySkeleton.tsx @@ -0,0 +1,59 @@ +import clsx from 'clsx'; +import { Skeleton, SkeletonGroup } from '@/components/ui/Skeleton'; +import { GalleryContainer, GalleryHeroFrame, MasonryGrid, masonryTileClass } from './GalleryLayout'; + +// Loading placeholder for the public gallery page. It renders through the same +// hero frame, container and masonry grid as the real page (see GalleryLayout), +// so replacing it with photos changes only the pixels inside the tiles. + +// Portrait/landscape/square mix standing in for a real event set. Fixed order +// on purpose — a random shuffle would differ between the server and client +// render and blow up hydration. +const FALLBACK_RATIOS = [3 / 4, 4 / 3, 1, 2 / 3, 3 / 2, 4 / 5, 1, 3 / 4, 16 / 9, 4 / 5, 3 / 4, 4 / 3]; + +interface GallerySkeletonProps { + /** Number of tiles to draw; ignored when `ratios` is given. */ + count?: number; + /** + * Real width/height ratios when the caller already knows them (e.g. a + * cached gallery payload). Produces a grid with exactly the right geometry + * instead of the guessed mix above. + */ + ratios?: number[]; +} + +export default function GallerySkeleton({ count = 12, ratios }: GallerySkeletonProps) { + const tiles = + ratios && ratios.length > 0 + ? ratios + : Array.from({ length: count }, (_, i) => FALLBACK_RATIOS[i % FALLBACK_RATIOS.length]); + + return ( + + + {/* Matches the h1 (text-3xl / md:text-5xl) and the pill row below it. */} + +
+ + +
+ + + + + {tiles.map((ratio, i) => ( +