Add photo content-hash dedup and local↔S3 library sync.
Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
This commit is contained in:
@@ -86,8 +86,23 @@ npm run dev:photos # go run ./cmd/photo-api
|
|||||||
npm run build:photos # go build -o bin/photo-api
|
npm run build:photos # go build -o bin/photo-api
|
||||||
npm run migrate:photos # apply photos_* migrations
|
npm run migrate:photos # apply photos_* migrations
|
||||||
npm run test:photos # go test ./...
|
npm run test:photos # go test ./...
|
||||||
|
|
||||||
|
# Move the photo library between storage backends (both must be configured
|
||||||
|
# in photo-api/.env; the source is left untouched, reruns skip what is there)
|
||||||
|
npm run sync:photos:to-s3 # local disk -> S3
|
||||||
|
npm run sync:photos:to-local # S3 -> local disk
|
||||||
|
npm run sync:photos -- to-s3 --dry-run # flags: --dry-run --overwrite
|
||||||
|
# --concurrency=N --gallery=<id>
|
||||||
|
|
||||||
|
# Uploads are deduplicated per gallery by content hash. Photos uploaded before
|
||||||
|
# that existed need hashing once (idempotent, deletes nothing):
|
||||||
|
npm run backfill:photos:checksums
|
||||||
```
|
```
|
||||||
|
|
||||||
|
After a sync, set `STORAGE_BACKEND=s3` (or `local`) in `photo-api/.env` and
|
||||||
|
restart the service to serve from the new backend. See
|
||||||
|
[`photo-api/README.md`](photo-api/README.md#move-the-library-between-backends).
|
||||||
|
|
||||||
You can also run per workspace:
|
You can also run per workspace:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -117,6 +132,7 @@ Key settings (see `photo-api/.env.example`):
|
|||||||
- **DB**: `DB_TYPE` and `DATABASE_URL` — point at the **same** database as the backend
|
- **DB**: `DB_TYPE` and `DATABASE_URL` — point at the **same** database as the backend
|
||||||
- **Auth**: none needed for user auth — the service validates Better Auth session cookies against the shared database. `PHOTO_VIEW_SECRET` signs gallery image view tokens (falls back to `JWT_SECRET` during migration).
|
- **Auth**: none needed for user auth — the service validates Better Auth session cookies against the shared database. `PHOTO_VIEW_SECRET` signs gallery image view tokens (falls back to `JWT_SECRET` during migration).
|
||||||
- **Storage**: `STORAGE_PATH` (local disk) or `S3_ENDPOINT` + `S3_BUCKET` (S3/Garage/MinIO); S3 downloads use short-lived presigned URLs
|
- **Storage**: `STORAGE_PATH` (local disk) or `S3_ENDPOINT` + `S3_BUCKET` (S3/Garage/MinIO); S3 downloads use short-lived presigned URLs
|
||||||
|
- **Storage switch**: `STORAGE_BACKEND=auto|local|s3` (`auto` = S3 when it is configured). Keep both sides configured and flip this one line to move between them; `npm run sync:photos:to-s3` / `:to-local` copies the existing library first
|
||||||
- **Uploads/worker**: `MAX_UPLOAD_MB`, `WORKER_CONCURRENCY` (a worker generates thumb/preview JPEG variants with EXIF stripped)
|
- **Uploads/worker**: `MAX_UPLOAD_MB`, `WORKER_CONCURRENCY` (a worker generates thumb/preview JPEG variants with EXIF stripped)
|
||||||
|
|
||||||
### Frontend (`frontend/.env`)
|
### Frontend (`frontend/.env`)
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
# ============================================================
|
||||||
|
# Spanglish Community - Photo Gallery API
|
||||||
|
# photos.spanglishcommunity.com
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name photos.spanglishcommunity.com;
|
||||||
|
|
||||||
|
location /.well-known/acme-challenge/ {
|
||||||
|
root /var/www/html;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
return 301 https://photos.spanglishcommunity.com$request_uri;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl;
|
||||||
|
http2 on;
|
||||||
|
|
||||||
|
server_name photos.spanglishcommunity.com;
|
||||||
|
|
||||||
|
# Photos can be larger than typical JSON payloads
|
||||||
|
client_max_body_size 25m;
|
||||||
|
|
||||||
|
# SSL
|
||||||
|
ssl_certificate /etc/letsencrypt/live/photos.spanglishcommunity.com/fullchain.pem;
|
||||||
|
ssl_certificate_key /etc/letsencrypt/live/photos.spanglishcommunity.com/privkey.pem;
|
||||||
|
|
||||||
|
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||||
|
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||||
|
|
||||||
|
# Security
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
access_log /var/log/nginx/spanglish_photo_access.log;
|
||||||
|
error_log /var/log/nginx/spanglish_photo_error.log;
|
||||||
|
|
||||||
|
# CORS Configuration
|
||||||
|
set $cors_origin "";
|
||||||
|
if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") {
|
||||||
|
set $cors_origin $http_origin;
|
||||||
|
}
|
||||||
|
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||||
|
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||||
|
|
||||||
|
# Ensure 413 returns JSON + CORS
|
||||||
|
error_page 413 = @payload_too_large;
|
||||||
|
location @payload_too_large {
|
||||||
|
default_type application/json;
|
||||||
|
return 413 '{"error":"Payload too large (413). Please upload a smaller file."}';
|
||||||
|
}
|
||||||
|
|
||||||
|
# Ensure 429 (rate limited) returns JSON + CORS
|
||||||
|
error_page 429 = @rate_limited;
|
||||||
|
location @rate_limited {
|
||||||
|
default_type application/json;
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
return 429 '{"error":"Too many requests. Please slow down."}';
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
limit_req zone=spanglish_photo_limit burst=40 nodelay;
|
||||||
|
|
||||||
|
# Preflight
|
||||||
|
if ($request_method = 'OPTIONS') {
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||||
|
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||||
|
add_header 'Access-Control-Max-Age' 86400 always;
|
||||||
|
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||||
|
add_header 'Content-Length' 0;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass http://spanglish_photo_api;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||||
|
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||||
|
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
proxy_connect_timeout 300s;
|
||||||
|
|
||||||
|
# Buffer large image uploads to disk rather than memory
|
||||||
|
proxy_request_buffering on;
|
||||||
|
proxy_max_temp_file_size 1024m;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
# ============================================================
|
||||||
|
# Spanglish Community - Backend API
|
||||||
|
# api.spanglishcommunity.com
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name api.spanglishcommunity.com;
|
||||||
|
|
||||||
|
# ACME
|
||||||
|
location /.well-known/acme-challenge/ {
|
||||||
|
root /var/www/html;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Force HTTPS
|
||||||
|
location / {
|
||||||
|
return 301 https://api.spanglishcommunity.com$request_uri;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl;
|
||||||
|
http2 on;
|
||||||
|
|
||||||
|
server_name api.spanglishcommunity.com;
|
||||||
|
|
||||||
|
# Upload size limit (avoid nginx 413 on media uploads)
|
||||||
|
# Keep this >= backend MEDIA_MAX_UPLOAD_MB (default 10MB).
|
||||||
|
client_max_body_size 20m;
|
||||||
|
|
||||||
|
# SSL
|
||||||
|
ssl_certificate /etc/letsencrypt/live/spanglishcommunity.com/fullchain.pem;
|
||||||
|
ssl_certificate_key /etc/letsencrypt/live/spanglishcommunity.com/privkey.pem;
|
||||||
|
|
||||||
|
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||||
|
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||||
|
|
||||||
|
# Security (API)
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
access_log /var/log/nginx/spanglish_api_access.log;
|
||||||
|
error_log /var/log/nginx/spanglish_api_error.log;
|
||||||
|
|
||||||
|
# CORS Configuration (set once, used everywhere)
|
||||||
|
set $cors_origin "";
|
||||||
|
if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") {
|
||||||
|
set $cors_origin $http_origin;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Add CORS headers to all responses (including nginx-generated errors)
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||||
|
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||||
|
|
||||||
|
# Ensure 413 returns JSON + CORS (browser otherwise reports "CORS blocked")
|
||||||
|
error_page 413 = @payload_too_large;
|
||||||
|
location @payload_too_large {
|
||||||
|
default_type application/json;
|
||||||
|
return 413 '{"error":"Payload too large (413). Please upload a smaller file."}';
|
||||||
|
}
|
||||||
|
|
||||||
|
# Photo gallery service (photo-api, port 3003). ^~ wins over the "/"
|
||||||
|
# prefix below, so /api/photos/* reaches the Go photo-api instead of the
|
||||||
|
# Node backend (which has no photo routes and would 404). The admin UI
|
||||||
|
# calls this cross-origin via NEXT_PUBLIC_API_URL, so preflight + CORS
|
||||||
|
# must be handled here just like location / below.
|
||||||
|
location ^~ /api/photos/ {
|
||||||
|
limit_req zone=spanglish_api_limit burst=50 nodelay;
|
||||||
|
|
||||||
|
# Handle preflight OPTIONS requests (add_header inside if{} does NOT
|
||||||
|
# inherit server-level headers, so repeat all CORS headers here).
|
||||||
|
if ($request_method = 'OPTIONS') {
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||||
|
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||||
|
add_header 'Access-Control-Max-Age' 86400 always;
|
||||||
|
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||||
|
add_header 'Content-Length' 0;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass http://spanglish_photo_api;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# Strip CORS headers from the service (nginx handles CORS here)
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||||
|
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||||
|
|
||||||
|
# Photo batches can be large; allow bigger bodies + unbuffered upload.
|
||||||
|
client_max_body_size 100m;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
proxy_connect_timeout 300s;
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
limit_req zone=spanglish_api_limit burst=50 nodelay;
|
||||||
|
|
||||||
|
# Handle preflight OPTIONS requests
|
||||||
|
# NOTE: add_header inside if{} does NOT inherit server-level headers,
|
||||||
|
# so we must repeat all CORS headers here.
|
||||||
|
if ($request_method = 'OPTIONS') {
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||||
|
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||||
|
add_header 'Access-Control-Max-Age' 86400 always;
|
||||||
|
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||||
|
add_header 'Content-Length' 0;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass http://spanglish_backend;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# Strip CORS headers from backend (nginx handles CORS at server level)
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||||
|
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||||
|
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
proxy_connect_timeout 300s;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
# ============================================================
|
||||||
|
# Spanglish Community - Frontend
|
||||||
|
# spanglishcommunity.com / www
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name spanglishcommunity.com www.spanglishcommunity.com;
|
||||||
|
|
||||||
|
# ACME
|
||||||
|
location /.well-known/acme-challenge/ {
|
||||||
|
root /var/www/html;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Force HTTPS
|
||||||
|
location / {
|
||||||
|
return 301 https://spanglishcommunity.com$request_uri;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl;
|
||||||
|
http2 on;
|
||||||
|
|
||||||
|
server_name spanglishcommunity.com www.spanglishcommunity.com;
|
||||||
|
|
||||||
|
# Upload size limit (covers same-origin /api uploads via this vhost)
|
||||||
|
client_max_body_size 20m;
|
||||||
|
|
||||||
|
# SSL
|
||||||
|
ssl_certificate /etc/letsencrypt/live/spanglishcommunity.com/fullchain.pem;
|
||||||
|
ssl_certificate_key /etc/letsencrypt/live/spanglishcommunity.com/privkey.pem;
|
||||||
|
|
||||||
|
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||||
|
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||||
|
|
||||||
|
# Security
|
||||||
|
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
access_log /var/log/nginx/spanglish_frontend_access.log;
|
||||||
|
error_log /var/log/nginx/spanglish_frontend_error.log;
|
||||||
|
|
||||||
|
# Proxy /api/photos to the photo-api (Go service, port 3003). ^~ wins over
|
||||||
|
# the /api prefix below so same-origin image/gallery requests reach the
|
||||||
|
# photo-api instead of the Node backend (which has no photo routes -> 404).
|
||||||
|
location ^~ /api/photos/ {
|
||||||
|
proxy_pass http://spanglish_photo_api;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# Photo uploads/batches can be large; allow bigger bodies.
|
||||||
|
client_max_body_size 100m;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
proxy_connect_timeout 300s;
|
||||||
|
|
||||||
|
# Let the photo-api set Cache-Control per image visibility (public vs.
|
||||||
|
# token-gated) rather than forcing a cache policy here.
|
||||||
|
}
|
||||||
|
|
||||||
|
# Proxy /api to backend
|
||||||
|
location /api {
|
||||||
|
proxy_pass http://spanglish_backend;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
proxy_connect_timeout 300s;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Proxy /uploads to backend
|
||||||
|
location /uploads {
|
||||||
|
proxy_pass http://spanglish_backend;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# Cache static files
|
||||||
|
proxy_cache_valid 200 1d;
|
||||||
|
expires 1d;
|
||||||
|
add_header Cache-Control "public, immutable";
|
||||||
|
}
|
||||||
|
|
||||||
|
# Frontend App
|
||||||
|
location / {
|
||||||
|
proxy_pass http://spanglish_frontend;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# WebSocket / HMR
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
|
||||||
|
proxy_read_timeout 60s;
|
||||||
|
proxy_connect_timeout 60s;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# ============================================================
|
||||||
|
# Spanglish Community - Photo Gallery API
|
||||||
|
# photos.spanglishcommunity.com
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name photos.spanglishcommunity.com;
|
||||||
|
|
||||||
|
location /.well-known/acme-challenge/ {
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
return 301 https://photos.spanglishcommunity.com$request_uri;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 443 ssl;
|
||||||
|
http2 on;
|
||||||
|
|
||||||
|
server_name photos.spanglishcommunity.com;
|
||||||
|
|
||||||
|
# Photos can be larger than typical JSON payloads
|
||||||
|
client_max_body_size 25m;
|
||||||
|
|
||||||
|
# SSL
|
||||||
|
ssl_certificate /etc/letsencrypt/live/photos.spanglishcommunity.com/fullchain.pem;
|
||||||
|
ssl_certificate_key /etc/letsencrypt/live/photos.spanglishcommunity.com/privkey.pem;
|
||||||
|
|
||||||
|
include /etc/letsencrypt/options-ssl-nginx.conf;
|
||||||
|
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
||||||
|
|
||||||
|
# Security
|
||||||
|
add_header X-Frame-Options "DENY" always;
|
||||||
|
add_header X-Content-Type-Options "nosniff" always;
|
||||||
|
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
access_log /var/log/nginx/spanglish_photo_access.log;
|
||||||
|
error_log /var/log/nginx/spanglish_photo_error.log;
|
||||||
|
|
||||||
|
# CORS Configuration
|
||||||
|
set $cors_origin "";
|
||||||
|
if ($http_origin ~* "^https://(www\.)?spanglishcommunity\.com$") {
|
||||||
|
set $cors_origin $http_origin;
|
||||||
|
}
|
||||||
|
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||||
|
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||||
|
|
||||||
|
# Ensure 413 returns JSON + CORS
|
||||||
|
error_page 413 = @payload_too_large;
|
||||||
|
location @payload_too_large {
|
||||||
|
default_type application/json;
|
||||||
|
return 413 '{"error":"Payload too large (413). Please upload a smaller file."}';
|
||||||
|
}
|
||||||
|
|
||||||
|
# Ensure 429 (rate limited) returns JSON + CORS
|
||||||
|
error_page 429 = @rate_limited;
|
||||||
|
location @rate_limited {
|
||||||
|
default_type application/json;
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
return 429 '{"error":"Too many requests. Please slow down."}';
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
limit_req zone=spanglish_photo_limit burst=40 nodelay;
|
||||||
|
|
||||||
|
# Preflight
|
||||||
|
if ($request_method = 'OPTIONS') {
|
||||||
|
add_header 'Access-Control-Allow-Origin' $cors_origin always;
|
||||||
|
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH, OPTIONS' always;
|
||||||
|
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
|
||||||
|
add_header 'Access-Control-Allow-Credentials' 'true' always;
|
||||||
|
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
|
||||||
|
add_header 'Access-Control-Max-Age' 86400 always;
|
||||||
|
add_header 'Content-Type' 'text/plain; charset=utf-8';
|
||||||
|
add_header 'Content-Length' 0;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy_pass http://spanglish_photo_api;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Origin';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Methods';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Headers';
|
||||||
|
proxy_hide_header 'Access-Control-Allow-Credentials';
|
||||||
|
proxy_hide_header 'Access-Control-Expose-Headers';
|
||||||
|
|
||||||
|
proxy_read_timeout 300s;
|
||||||
|
proxy_connect_timeout 300s;
|
||||||
|
|
||||||
|
# Buffer large image uploads to disk rather than memory
|
||||||
|
proxy_request_buffering on;
|
||||||
|
proxy_max_temp_file_size 1024m;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
upstream spanglish_frontend {
|
||||||
|
server 127.0.0.1:3019;
|
||||||
|
}
|
||||||
|
upstream spanglish_backend {
|
||||||
|
server 127.0.0.1:3018;
|
||||||
|
}
|
||||||
|
upstream spanglish_photo_api {
|
||||||
|
server 127.0.0.1:3003;
|
||||||
|
}
|
||||||
|
|
||||||
|
limit_req_zone $binary_remote_addr zone=spanglish_photo_limit:10m rate=20r/s;
|
||||||
|
limit_req_zone $binary_remote_addr zone=spanglish_api_limit:10m rate=30r/s;
|
||||||
@@ -18,6 +18,7 @@ import {
|
|||||||
CheckCircleIcon,
|
CheckCircleIcon,
|
||||||
ChevronDownIcon,
|
ChevronDownIcon,
|
||||||
ChevronUpIcon,
|
ChevronUpIcon,
|
||||||
|
DocumentDuplicateIcon,
|
||||||
ExclamationCircleIcon,
|
ExclamationCircleIcon,
|
||||||
ExclamationTriangleIcon,
|
ExclamationTriangleIcon,
|
||||||
LinkIcon,
|
LinkIcon,
|
||||||
@@ -34,7 +35,9 @@ interface UploadItem {
|
|||||||
name: string;
|
name: string;
|
||||||
sizeBytes: number;
|
sizeBytes: number;
|
||||||
progress: number; // 0..1 while uploading
|
progress: number; // 0..1 while uploading
|
||||||
status: 'queued' | 'uploading' | 'processing' | 'error';
|
// 'duplicate' is terminal: the gallery already held these bytes, so nothing
|
||||||
|
// was stored and no new tile appears in the grid.
|
||||||
|
status: 'queued' | 'uploading' | 'processing' | 'error' | 'duplicate';
|
||||||
error?: string;
|
error?: string;
|
||||||
photoId?: string;
|
photoId?: string;
|
||||||
}
|
}
|
||||||
@@ -117,8 +120,19 @@ export default function AdminGalleryDetailPage() {
|
|||||||
const { photos: added } = await photosApi.uploadPhotoWithProgress(id, file, (fraction) =>
|
const { photos: added } = await photosApi.uploadPhotoWithProgress(id, file, (fraction) =>
|
||||||
patchUpload(key, { progress: fraction })
|
patchUpload(key, { progress: fraction })
|
||||||
);
|
);
|
||||||
setPhotos((prev) => [...prev, ...added]);
|
// A duplicate echoes back a photo already in the grid, so merge by id
|
||||||
patchUpload(key, { status: 'processing', progress: 1, photoId: added[0]?.id });
|
// instead of appending (`duplicate` is an upload outcome, not photo
|
||||||
|
// state, so it is dropped here).
|
||||||
|
setPhotos((prev) => {
|
||||||
|
const byId = new Map(prev.map((p) => [p.id, p]));
|
||||||
|
added.forEach(({ duplicate: _duplicate, ...photo }) => byId.set(photo.id, photo));
|
||||||
|
return Array.from(byId.values());
|
||||||
|
});
|
||||||
|
patchUpload(key, {
|
||||||
|
status: added[0]?.duplicate ? 'duplicate' : 'processing',
|
||||||
|
progress: 1,
|
||||||
|
photoId: added[0]?.id,
|
||||||
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
patchUpload(key, {
|
patchUpload(key, {
|
||||||
status: 'error',
|
status: 'error',
|
||||||
@@ -148,7 +162,8 @@ export default function AdminGalleryDetailPage() {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// A row is "done" once its photo finished processing; the panel derives
|
// A row is "done" once its photo finished processing; the panel derives
|
||||||
// this from the photos list instead of tracking it separately.
|
// this from the photos list instead of tracking it separately. 'duplicate'
|
||||||
|
// is terminal and never reconciled — its photo was already there.
|
||||||
const displayStatus = (u: UploadItem): { state: string; error?: string } => {
|
const displayStatus = (u: UploadItem): { state: string; error?: string } => {
|
||||||
if (u.status === 'processing' && u.photoId) {
|
if (u.status === 'processing' && u.photoId) {
|
||||||
const photo = photos.find((p) => p.id === u.photoId);
|
const photo = photos.find((p) => p.id === u.photoId);
|
||||||
@@ -579,6 +594,11 @@ export default function AdminGalleryDetailPage() {
|
|||||||
{ds.state === 'queued' && (es ? 'En cola' : 'Queued')}
|
{ds.state === 'queued' && (es ? 'En cola' : 'Queued')}
|
||||||
{ds.state === 'processing' && (es ? 'Procesando…' : 'Processing…')}
|
{ds.state === 'processing' && (es ? 'Procesando…' : 'Processing…')}
|
||||||
{ds.state === 'done' && formatBytes(u.sizeBytes)}
|
{ds.state === 'done' && formatBytes(u.sizeBytes)}
|
||||||
|
{ds.state === 'duplicate' && (
|
||||||
|
<span className="text-amber-600">
|
||||||
|
{es ? 'Ya está en esta galería' : 'Already in this gallery'}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
{ds.state === 'error' && (
|
{ds.state === 'error' && (
|
||||||
<span className="text-red-600" title={ds.error}>
|
<span className="text-red-600" title={ds.error}>
|
||||||
{ds.error}
|
{ds.error}
|
||||||
@@ -588,6 +608,9 @@ export default function AdminGalleryDetailPage() {
|
|||||||
</div>
|
</div>
|
||||||
<span className="shrink-0">
|
<span className="shrink-0">
|
||||||
{ds.state === 'done' && <CheckCircleIcon className="w-6 h-6 text-green-600" />}
|
{ds.state === 'done' && <CheckCircleIcon className="w-6 h-6 text-green-600" />}
|
||||||
|
{ds.state === 'duplicate' && (
|
||||||
|
<DocumentDuplicateIcon className="w-6 h-6 text-amber-600" />
|
||||||
|
)}
|
||||||
{ds.state === 'error' && <ExclamationCircleIcon className="w-6 h-6 text-red-600" />}
|
{ds.state === 'error' && <ExclamationCircleIcon className="w-6 h-6 text-red-600" />}
|
||||||
{ds.state === 'processing' && (
|
{ds.state === 'processing' && (
|
||||||
<div className="animate-spin w-5 h-5 border-2 border-primary-yellow border-t-transparent rounded-full" />
|
<div className="animate-spin w-5 h-5 border-2 border-primary-yellow border-t-transparent rounded-full" />
|
||||||
|
|||||||
@@ -52,6 +52,9 @@ export interface Photo {
|
|||||||
preview?: string;
|
preview?: string;
|
||||||
original: string;
|
original: string;
|
||||||
};
|
};
|
||||||
|
// Upload responses only: these bytes were already in the gallery, so nothing
|
||||||
|
// was stored and the rest of this object describes the existing photo.
|
||||||
|
duplicate?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CreateGalleryInput {
|
export interface CreateGalleryInput {
|
||||||
|
|||||||
@@ -14,6 +14,10 @@
|
|||||||
"build:photos": "cd photo-api && go build -o bin/photo-api ./cmd/photo-api",
|
"build:photos": "cd photo-api && go build -o bin/photo-api ./cmd/photo-api",
|
||||||
"test:photos": "cd photo-api && go test ./...",
|
"test:photos": "cd photo-api && go test ./...",
|
||||||
"migrate:photos": "cd photo-api && go run ./cmd/photo-api migrate",
|
"migrate:photos": "cd photo-api && go run ./cmd/photo-api migrate",
|
||||||
|
"sync:photos": "cd photo-api && go run ./cmd/photo-api sync",
|
||||||
|
"sync:photos:to-s3": "cd photo-api && go run ./cmd/photo-api sync to-s3",
|
||||||
|
"sync:photos:to-local": "cd photo-api && go run ./cmd/photo-api sync to-local",
|
||||||
|
"backfill:photos:checksums": "cd photo-api && go run ./cmd/photo-api backfill-checksums",
|
||||||
"start": "concurrently \"npm run start:backend\" \"npm run start:frontend\"",
|
"start": "concurrently \"npm run start:backend\" \"npm run start:frontend\"",
|
||||||
"start:backend": "npm run start --workspace=backend",
|
"start:backend": "npm run start --workspace=backend",
|
||||||
"start:frontend": "npm run start --workspace=frontend",
|
"start:frontend": "npm run start --workspace=frontend",
|
||||||
|
|||||||
+12
-3
@@ -29,10 +29,19 @@ JWT_SECRET=
|
|||||||
# Public site origin, used to build share links and allow dev CORS
|
# Public site origin, used to build share links and allow dev CORS
|
||||||
FRONTEND_URL=https://spanglishcommunity.com
|
FRONTEND_URL=https://spanglishcommunity.com
|
||||||
|
|
||||||
# Photo storage. Local disk by default; setting BOTH S3_ENDPOINT and
|
# Photo storage. Which backend serves requests:
|
||||||
# S3_BUCKET switches to S3 (same convention as the backend's media storage).
|
# auto (default) S3 when S3_ENDPOINT + S3_BUCKET are set, local otherwise
|
||||||
# Use a photos-specific bucket — do not reuse the backend's media bucket.
|
# — the backend's convention (backend/src/lib/storage.ts)
|
||||||
|
# local always local disk, even with the S3 settings filled in
|
||||||
|
# s3 always S3
|
||||||
|
# Keep both sides filled in and flip this one line to switch backends. Both
|
||||||
|
# are also required by the library sync (`npm run sync:photos:to-s3` /
|
||||||
|
# `:to-local`), which copies every photo from one backend to the other.
|
||||||
|
STORAGE_BACKEND=auto
|
||||||
|
|
||||||
STORAGE_PATH=./data/photos
|
STORAGE_PATH=./data/photos
|
||||||
|
|
||||||
|
# Use a photos-specific bucket — do not reuse the backend's media bucket.
|
||||||
#S3_ENDPOINT=
|
#S3_ENDPOINT=
|
||||||
#S3_REGION=auto
|
#S3_REGION=auto
|
||||||
#S3_BUCKET=spanglish-photos
|
#S3_BUCKET=spanglish-photos
|
||||||
|
|||||||
+11
-2
@@ -1,11 +1,11 @@
|
|||||||
.PHONY: start build test migrate clean help
|
.PHONY: start build test migrate sync-to-s3 sync-to-local backfill-checksums clean help
|
||||||
|
|
||||||
BINARY := bin/photo-api
|
BINARY := bin/photo-api
|
||||||
CMD := ./cmd/photo-api
|
CMD := ./cmd/photo-api
|
||||||
|
|
||||||
help: ## Show available targets
|
help: ## Show available targets
|
||||||
@grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \
|
@grep -E '^[a-zA-Z_-]+:.*?##' $(MAKEFILE_LIST) | \
|
||||||
awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}'
|
awk 'BEGIN {FS = ":.*?## "}; {printf " %-19s %s\n", $$1, $$2}'
|
||||||
|
|
||||||
start: ## Run the photo-api server (go run)
|
start: ## Run the photo-api server (go run)
|
||||||
go run $(CMD)
|
go run $(CMD)
|
||||||
@@ -19,5 +19,14 @@ test: ## Run all Go tests
|
|||||||
migrate: ## Apply pending photos_* migrations
|
migrate: ## Apply pending photos_* migrations
|
||||||
go run $(CMD) migrate
|
go run $(CMD) migrate
|
||||||
|
|
||||||
|
sync-to-s3: ## Copy the photo library from local disk to S3 (needs both in .env)
|
||||||
|
go run $(CMD) sync to-s3
|
||||||
|
|
||||||
|
sync-to-local: ## Copy the photo library from S3 back to local disk
|
||||||
|
go run $(CMD) sync to-local
|
||||||
|
|
||||||
|
backfill-checksums: ## Hash photos uploaded before duplicate detection existed
|
||||||
|
go run $(CMD) backfill-checksums
|
||||||
|
|
||||||
clean: ## Remove built binary
|
clean: ## Remove built binary
|
||||||
rm -rf bin
|
rm -rf bin
|
||||||
|
|||||||
+1
-1
@@ -131,7 +131,7 @@ All routes under `/api/photos`. Auth = `Authorization: Bearer <existing JWT>`. "
|
|||||||
| `GET /api/photos/galleries/:id` | Full gallery detail incl. photos in position order with per-variant URLs and processing status. | `{ gallery, photos: [...] }` |
|
| `GET /api/photos/galleries/:id` | Full gallery detail incl. photos in position order with per-variant URLs and processing status. | `{ gallery, photos: [...] }` |
|
||||||
| `PATCH /api/photos/galleries/:id` | Update title/description/eventId/visibility/coverPhotoId. | `{ gallery }` |
|
| `PATCH /api/photos/galleries/:id` | Update title/description/eventId/visibility/coverPhotoId. | `{ gallery }` |
|
||||||
| `DELETE /api/photos/galleries/:id` | Delete gallery + photos + stored objects. | `{ message }` |
|
| `DELETE /api/photos/galleries/:id` | Delete gallery + photos + stored objects. | `{ message }` |
|
||||||
| `POST /api/photos/galleries/:id/photos` | Multipart upload, field `files` (repeatable). Sniffs magic bytes (JPEG/PNG/WebP/GIF/AVIF; explicit `415 { error: "HEIC is not supported, please upload JPEG" }` for HEIC). Stores original, inserts row `status='queued'`, appends at end position. | `201 { photos: [...] }` |
|
| `POST /api/photos/galleries/:id/photos` | Multipart upload, field `files` (repeatable). Sniffs magic bytes (JPEG/PNG/WebP/GIF/AVIF; explicit `415 { error: "HEIC is not supported, please upload JPEG" }` for HEIC). Stores original, inserts row `status='queued'`, appends at end position. Hashes the bytes into `checksum`: if the gallery already holds them, nothing is stored and the existing photo is echoed with `duplicate: true` (the batch continues, no position consumed). | `201 { photos: [...] }` |
|
||||||
| `PATCH /api/photos/galleries/:id/order` | Body `{ photoIds: [uuid,...] }` — full ordering; positions rewritten in one transaction. | `{ message }` |
|
| `PATCH /api/photos/galleries/:id/order` | Body `{ photoIds: [uuid,...] }` — full ordering; positions rewritten in one transaction. | `{ message }` |
|
||||||
| `DELETE /api/photos/photos/:photoId` | Delete one photo + its objects; compacts positions. | `{ message }` |
|
| `DELETE /api/photos/photos/:photoId` | Delete one photo + its objects; compacts positions. | `{ message }` |
|
||||||
| `POST /api/photos/galleries/:id/share-token` | Rotate share token (invalidate old links). | `{ gallery }` |
|
| `POST /api/photos/galleries/:id/share-token` | Rotate share token (invalidate old links). | `{ gallery }` |
|
||||||
|
|||||||
+81
-4
@@ -14,9 +14,13 @@ Design/decisions: [PLAN.md](./PLAN.md).
|
|||||||
- Originals are stored byte-identical for download; a worker generates JPEG
|
- Originals are stored byte-identical for download; a worker generates JPEG
|
||||||
variants (thumb 512px q78, preview 2048px q85, EXIF stripped/orientation
|
variants (thumb 512px q78, preview 2048px q85, EXIF stripped/orientation
|
||||||
applied) queued in the DB with retries.
|
applied) queued in the DB with retries.
|
||||||
|
- Uploads are deduplicated per gallery: see
|
||||||
|
[Duplicate detection](#duplicate-detection).
|
||||||
- Storage: local disk (`STORAGE_PATH`) or S3/Garage/MinIO (set `S3_ENDPOINT`
|
- Storage: local disk (`STORAGE_PATH`) or S3/Garage/MinIO (set `S3_ENDPOINT`
|
||||||
+ `S3_BUCKET`), same selection convention as the backend. S3 downloads use
|
+ `S3_BUCKET`), same selection convention as the backend, overridable with
|
||||||
short-lived presigned URLs; nothing in the bucket is public.
|
`STORAGE_BACKEND`. S3 downloads use short-lived presigned URLs; nothing in
|
||||||
|
the bucket is public. Switching backends later: see
|
||||||
|
[Move the library between backends](#move-the-library-between-backends).
|
||||||
- Auth: validates the backend's HS256 JWTs with the shared `JWT_SECRET`
|
- Auth: validates the backend's HS256 JWTs with the shared `JWT_SECRET`
|
||||||
(issuer `spanglish`, audience `spanglish-app`) including the DB-backed
|
(issuer `spanglish`, audience `spanglish-app`) including the DB-backed
|
||||||
tokenVersion/account-status revocation check. Admin surface is
|
tokenVersion/account-status revocation check. Admin surface is
|
||||||
@@ -37,7 +41,8 @@ go test ./... # SQLite; add PHOTO_TEST_PG=<url> to also run on Post
|
|||||||
```
|
```
|
||||||
|
|
||||||
From the repo root: `npm run dev:photos`, `npm run build:photos`,
|
From the repo root: `npm run dev:photos`, `npm run build:photos`,
|
||||||
`npm run test:photos`. The Next dev server rewrites `/api/photos/*` to
|
`npm run test:photos`, `npm run migrate:photos`, `npm run sync:photos`,
|
||||||
|
`npm run backfill:photos:checksums`. The Next dev server rewrites `/api/photos/*` to
|
||||||
`PHOTO_API_URL` (default `http://localhost:3003`), so the frontend needs no
|
`PHOTO_API_URL` (default `http://localhost:3003`), so the frontend needs no
|
||||||
extra config in dev.
|
extra config in dev.
|
||||||
|
|
||||||
@@ -45,6 +50,78 @@ HEIC uploads require a converter CLI on the host: `apt install libvips-tools`
|
|||||||
(or `libheif-examples`). Without one, HEIC uploads are rejected with a clear
|
(or `libheif-examples`). Without one, HEIC uploads are rejected with a clear
|
||||||
message and a startup warning is logged.
|
message and a startup warning is logged.
|
||||||
|
|
||||||
|
## Move the library between backends
|
||||||
|
|
||||||
|
`photo-api sync` copies the whole photo library one way between local disk and
|
||||||
|
S3, so local↔S3 is a config switch rather than a migration project. **Both
|
||||||
|
backends must be configured in `photo-api/.env`** (`STORAGE_PATH` *and* the
|
||||||
|
`S3_*` values); `STORAGE_BACKEND` decides which one actually serves requests,
|
||||||
|
so filling in S3 does not switch anything by itself.
|
||||||
|
|
||||||
|
Local disk → S3:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run sync:photos -- to-s3 --dry-run # see what would be copied
|
||||||
|
npm run sync:photos:to-s3 # copy it
|
||||||
|
# then set STORAGE_BACKEND=s3 in photo-api/.env and restart the service
|
||||||
|
```
|
||||||
|
|
||||||
|
S3 → local disk is the same with `to-local` / `STORAGE_BACKEND=local`
|
||||||
|
(`npm run sync:photos:to-local`). From `photo-api/`: `make sync-to-s3`,
|
||||||
|
`make sync-to-local`.
|
||||||
|
|
||||||
|
Flags (`npm run sync:photos -- to-s3 --overwrite`, or after the direction on
|
||||||
|
the direct scripts): `--dry-run`, `--overwrite` (re-copy objects already
|
||||||
|
present with the same size), `--concurrency=N` (default 4), `--gallery=<id>`.
|
||||||
|
|
||||||
|
How it behaves:
|
||||||
|
|
||||||
|
- The `photos_photos` rows are the inventory — for each photo the original
|
||||||
|
plus, once processed, the thumb and preview. Objects with no row (worker
|
||||||
|
scratch files, leftovers of deleted galleries) are not copied.
|
||||||
|
- Only the destination is written. The source stays as a fallback; delete it
|
||||||
|
by hand once the switch is verified.
|
||||||
|
- Reruns are cheap and safe: objects already on the destination with the same
|
||||||
|
size are skipped, so an interrupted or partly failed sync just needs
|
||||||
|
rerunning. A failed object is logged and the exit code is non-zero.
|
||||||
|
- Keys are identical on both backends, so nothing in the database changes and
|
||||||
|
no re-processing is triggered.
|
||||||
|
- Photos uploaded *after* the copy but *before* the restart land on the old
|
||||||
|
backend. For a clean cutover, stop the service, sync, flip
|
||||||
|
`STORAGE_BACKEND`, start again — or sync a second time after the switch to
|
||||||
|
pick up stragglers.
|
||||||
|
|
||||||
|
## Duplicate detection
|
||||||
|
|
||||||
|
Every upload is hashed (sha256 of the original bytes) into
|
||||||
|
`photos_photos.checksum`, unique per `(gallery_id, checksum)`. If a gallery
|
||||||
|
already holds those exact bytes, the incoming copy is **discarded**: no object
|
||||||
|
is stored, no row is inserted, and the upload response echoes the existing
|
||||||
|
photo with `"duplicate": true`. The rest of the batch continues normally — a
|
||||||
|
duplicate is not an error and does not consume a position.
|
||||||
|
|
||||||
|
Scope is one gallery. The same image can still live in several galleries, each
|
||||||
|
with its own row and its own stored object, so deleting a gallery never orphans
|
||||||
|
another one's photos.
|
||||||
|
|
||||||
|
The admin uploader panel shows those rows as *"Already in this gallery"*; no
|
||||||
|
second tile appears in the grid.
|
||||||
|
|
||||||
|
Photos uploaded before this existed have no checksum, so they are not matched
|
||||||
|
until hashed once:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run backfill:photos:checksums -- --dry-run # what would be hashed
|
||||||
|
npm run backfill:photos:checksums # hash it
|
||||||
|
```
|
||||||
|
|
||||||
|
From `photo-api/`: `make backfill-checksums`. Flags: `--dry-run`,
|
||||||
|
`--concurrency=N` (default 4), `--gallery=<id>`. It reads from the active
|
||||||
|
`STORAGE_BACKEND`, only ever writes the checksum column, and is idempotent —
|
||||||
|
rerun it after a sync or a restore. Photos whose content already matches an
|
||||||
|
earlier one in the same gallery are **reported and left unhashed**; the command
|
||||||
|
never deletes anything, so removing the extras is an admin's call.
|
||||||
|
|
||||||
## API
|
## API
|
||||||
|
|
||||||
Everything under `/api/photos`. Errors are `{"error": string}`.
|
Everything under `/api/photos`. Errors are `{"error": string}`.
|
||||||
@@ -58,7 +135,7 @@ Admin (Bearer token, role admin/organizer):
|
|||||||
| GET | `/api/photos/galleries/:id` | gallery + photos (all statuses) |
|
| GET | `/api/photos/galleries/:id` | gallery + photos (all statuses) |
|
||||||
| PATCH | `/api/photos/galleries/:id` | update title/visibility/event/cover |
|
| PATCH | `/api/photos/galleries/:id` | update title/visibility/event/cover |
|
||||||
| DELETE | `/api/photos/galleries/:id` | delete gallery + objects |
|
| DELETE | `/api/photos/galleries/:id` | delete gallery + objects |
|
||||||
| POST | `/api/photos/galleries/:id/photos` | multipart upload (`files`) |
|
| POST | `/api/photos/galleries/:id/photos` | multipart upload (`files`), deduplicated |
|
||||||
| PATCH | `/api/photos/galleries/:id/order` | reorder (`{photoIds}`) |
|
| PATCH | `/api/photos/galleries/:id/order` | reorder (`{photoIds}`) |
|
||||||
| POST | `/api/photos/galleries/:id/share-token` | rotate share token |
|
| POST | `/api/photos/galleries/:id/share-token` | rotate share token |
|
||||||
| DELETE | `/api/photos/photos/:photoId` | delete photo |
|
| DELETE | `/api/photos/photos/:photoId` | delete photo |
|
||||||
|
|||||||
+135
-15
@@ -1,12 +1,15 @@
|
|||||||
// photo-api serves event photo galleries for the Spanglish platform.
|
// photo-api serves event photo galleries for the Spanglish platform.
|
||||||
//
|
//
|
||||||
// photo-api start the HTTP server
|
// photo-api start the HTTP server
|
||||||
// photo-api migrate apply pending photos_* migrations and exit
|
// photo-api migrate apply pending photos_* migrations and exit
|
||||||
|
// photo-api sync to-s3|to-local copy the photo library between backends
|
||||||
|
// photo-api backfill-checksums hash photos uploaded before duplicate detection
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -16,9 +19,11 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/httpapi"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/httpapi"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/photosync"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/worker"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/worker"
|
||||||
@@ -38,15 +43,21 @@ func main() {
|
|||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
if len(os.Args) > 1 && os.Args[1] == "migrate" {
|
|
||||||
if err := db.Migrate(context.Background()); err != nil {
|
|
||||||
log.Fatalf("migrate: %v", err)
|
|
||||||
}
|
|
||||||
log.Println("migrations up to date")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(os.Args) > 1 {
|
if len(os.Args) > 1 {
|
||||||
log.Fatalf("unknown subcommand %q (expected: migrate)", os.Args[1])
|
switch os.Args[1] {
|
||||||
|
case "migrate":
|
||||||
|
if err := db.Migrate(context.Background()); err != nil {
|
||||||
|
log.Fatalf("migrate: %v", err)
|
||||||
|
}
|
||||||
|
log.Println("migrations up to date")
|
||||||
|
case "sync":
|
||||||
|
runSync(cfg, db, os.Args[2:])
|
||||||
|
case "backfill-checksums":
|
||||||
|
runBackfillChecksums(cfg, db, os.Args[2:])
|
||||||
|
default:
|
||||||
|
log.Fatalf("unknown subcommand %q (expected: migrate, sync, backfill-checksums)", os.Args[1])
|
||||||
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
@@ -64,11 +75,7 @@ func main() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("storage: %v", err)
|
log.Fatalf("storage: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.S3Enabled() {
|
log.Printf("storage: %s (STORAGE_BACKEND=%s)", storage.Name(cfg, st), cfg.StorageBackend)
|
||||||
log.Printf("storage: S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint)
|
|
||||||
} else {
|
|
||||||
log.Printf("storage: local disk at %s", cfg.StoragePath)
|
|
||||||
}
|
|
||||||
|
|
||||||
heic := imaging.DetectHeicConverter(cfg.HeicConverter)
|
heic := imaging.DetectHeicConverter(cfg.HeicConverter)
|
||||||
if heic == nil {
|
if heic == nil {
|
||||||
@@ -104,3 +111,116 @@ func main() {
|
|||||||
log.Printf("shutdown: %v", err)
|
log.Printf("shutdown: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const syncUsage = `usage: photo-api sync to-s3|to-local [flags]
|
||||||
|
|
||||||
|
Copies every object of every photo (original, thumb, preview) from one storage
|
||||||
|
backend to the other. Both must be configured in photo-api/.env; the source is
|
||||||
|
never modified, and reruns skip objects already present on the destination.
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
`
|
||||||
|
|
||||||
|
// runSync handles `photo-api sync <direction> [flags]`, the storage migration
|
||||||
|
// used when moving the library between local disk and S3.
|
||||||
|
func runSync(cfg config.Config, db *store.DB, args []string) {
|
||||||
|
fs := flag.NewFlagSet("sync", flag.ExitOnError)
|
||||||
|
fs.Usage = func() {
|
||||||
|
fmt.Fprint(fs.Output(), syncUsage)
|
||||||
|
fs.PrintDefaults()
|
||||||
|
}
|
||||||
|
var (
|
||||||
|
dryRun = fs.Bool("dry-run", false, "report what would be copied without writing")
|
||||||
|
overwrite = fs.Bool("overwrite", false, "re-copy objects already present with the same size")
|
||||||
|
workers = fs.Int("concurrency", 4, "objects copied in parallel")
|
||||||
|
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
||||||
|
)
|
||||||
|
if len(args) == 0 {
|
||||||
|
fs.Usage()
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
direction, err := photosync.ParseDirection(args[0])
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%v\n\n", err)
|
||||||
|
fs.Usage()
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if err := fs.Parse(args[1:]); err != nil {
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
if _, err := photosync.Run(ctx, cfg, db, photosync.Options{
|
||||||
|
Direction: direction,
|
||||||
|
GalleryID: *gallery,
|
||||||
|
Concurrency: *workers,
|
||||||
|
DryRun: *dryRun,
|
||||||
|
Overwrite: *overwrite,
|
||||||
|
}); err != nil {
|
||||||
|
log.Fatalf("sync: %v", err)
|
||||||
|
}
|
||||||
|
if *dryRun {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
target := "s3"
|
||||||
|
if direction == photosync.ToLocal {
|
||||||
|
target = "local"
|
||||||
|
}
|
||||||
|
log.Printf("sync: set STORAGE_BACKEND=%s in photo-api/.env and restart the service to serve from it "+
|
||||||
|
"(the source copy is left in place — delete it once the switch is verified)", target)
|
||||||
|
}
|
||||||
|
|
||||||
|
const backfillUsage = `usage: photo-api backfill-checksums [flags]
|
||||||
|
|
||||||
|
Hashes the stored original of every photo that has no checksum yet, so uploads
|
||||||
|
of a photo already in a gallery are recognised as duplicates. Runs against the
|
||||||
|
active storage backend (STORAGE_BACKEND) and only ever writes the checksum
|
||||||
|
column — no photo is deleted. Photos whose content already matches an earlier
|
||||||
|
one in the same gallery are reported and left unhashed.
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
`
|
||||||
|
|
||||||
|
// runBackfillChecksums handles `photo-api backfill-checksums [flags]`, the
|
||||||
|
// one-off pass needed after the checksum column is added to an existing
|
||||||
|
// library.
|
||||||
|
func runBackfillChecksums(cfg config.Config, db *store.DB, args []string) {
|
||||||
|
fs := flag.NewFlagSet("backfill-checksums", flag.ExitOnError)
|
||||||
|
fs.Usage = func() {
|
||||||
|
fmt.Fprint(fs.Output(), backfillUsage)
|
||||||
|
fs.PrintDefaults()
|
||||||
|
}
|
||||||
|
var (
|
||||||
|
dryRun = fs.Bool("dry-run", false, "report what would be hashed without writing")
|
||||||
|
workers = fs.Int("concurrency", 4, "originals hashed in parallel")
|
||||||
|
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
||||||
|
)
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := storage.New(cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("storage: %v", err)
|
||||||
|
}
|
||||||
|
log.Printf("backfill-checksums: reading from %s", storage.Name(cfg, st))
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
res, err := checksum.Backfill(ctx, db, st, checksum.Options{
|
||||||
|
GalleryID: *gallery,
|
||||||
|
Concurrency: *workers,
|
||||||
|
DryRun: *dryRun,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("backfill-checksums: %v", err)
|
||||||
|
}
|
||||||
|
if res.Duplicates > 0 {
|
||||||
|
log.Printf("backfill-checksums: %d existing photos duplicate an earlier one in their gallery "+
|
||||||
|
"(listed above); they still show in the gallery — delete the unwanted ones from the admin page",
|
||||||
|
res.Duplicates)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package checksum
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Options configures a backfill run.
|
||||||
|
type Options struct {
|
||||||
|
// GalleryID limits the run to one gallery; empty means the whole library.
|
||||||
|
GalleryID string
|
||||||
|
// Concurrency is how many originals are hashed at a time.
|
||||||
|
Concurrency int
|
||||||
|
// DryRun reports what would be hashed without writing to the database.
|
||||||
|
DryRun bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result counts what a run did. Duplicates are rows whose bytes match an
|
||||||
|
// earlier photo in the same gallery; they keep a NULL checksum and are listed
|
||||||
|
// so an admin can decide what to do with them.
|
||||||
|
type Result struct {
|
||||||
|
Total int // rows without a checksum
|
||||||
|
Hashed int // hashed and recorded (or, with DryRun, would be)
|
||||||
|
Duplicates int
|
||||||
|
Missing int // original object absent from storage
|
||||||
|
Failed int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backfill hashes the stored original of every photo that has no checksum yet,
|
||||||
|
// so duplicate detection also catches re-uploads of photos from before the
|
||||||
|
// checksum column existed. It only ever writes the checksum column; no photo,
|
||||||
|
// row or object is deleted.
|
||||||
|
//
|
||||||
|
// It is idempotent: rerunning it finds only what the previous run left behind.
|
||||||
|
func Backfill(ctx context.Context, db *store.DB, st storage.Storage, opts Options) (Result, error) {
|
||||||
|
if opts.Concurrency < 1 {
|
||||||
|
opts.Concurrency = 4
|
||||||
|
}
|
||||||
|
photos, err := db.PhotosMissingChecksum(ctx, opts.GalleryID)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, fmt.Errorf("list photos without a checksum: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prefix := ""
|
||||||
|
if opts.DryRun {
|
||||||
|
prefix = "[dry-run] "
|
||||||
|
}
|
||||||
|
log.Printf("backfill-checksums: %s%d photos to hash, concurrency %d", prefix, len(photos), opts.Concurrency)
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.Mutex
|
||||||
|
res = Result{Total: len(photos)}
|
||||||
|
done int64
|
||||||
|
jobs = make(chan store.Photo)
|
||||||
|
wg sync.WaitGroup
|
||||||
|
)
|
||||||
|
// Sequential hashing per worker, but the database update is what can
|
||||||
|
// collide: two photos of the same gallery with identical bytes race, and
|
||||||
|
// the unique index decides which one keeps the checksum.
|
||||||
|
for i := 0; i < opts.Concurrency; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for p := range jobs {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sum, err := hashOriginal(ctx, st, p)
|
||||||
|
if err == nil && !opts.DryRun {
|
||||||
|
err = db.SetPhotoChecksum(ctx, p.ID, sum)
|
||||||
|
}
|
||||||
|
n := atomic.AddInt64(&done, 1)
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, storage.ErrNotExist):
|
||||||
|
res.Missing++
|
||||||
|
log.Printf("backfill-checksums: [%d/%d] MISSING object, skipped: %s (photo %s)",
|
||||||
|
n, len(photos), p.OriginalKey, p.ID)
|
||||||
|
case store.IsUniqueViolation(err):
|
||||||
|
res.Duplicates++
|
||||||
|
other := "an earlier photo"
|
||||||
|
if existing, findErr := db.FindPhotoByChecksum(ctx, p.GalleryID, sum); findErr == nil {
|
||||||
|
other = "photo " + existing.ID
|
||||||
|
}
|
||||||
|
log.Printf("backfill-checksums: [%d/%d] DUPLICATE: photo %s (%s) has the same content as %s "+
|
||||||
|
"in gallery %s — left without a checksum, delete it by hand if unwanted",
|
||||||
|
n, len(photos), p.ID, p.OriginalFilename, other, p.GalleryID)
|
||||||
|
case err != nil:
|
||||||
|
res.Failed++
|
||||||
|
log.Printf("backfill-checksums: [%d/%d] FAILED %s (photo %s): %v",
|
||||||
|
n, len(photos), p.OriginalKey, p.ID, err)
|
||||||
|
default:
|
||||||
|
res.Hashed++
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
for _, p := range photos {
|
||||||
|
select {
|
||||||
|
case jobs <- p:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
close(jobs)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
log.Printf("backfill-checksums: %sdone — %d hashed, %d duplicates left unhashed, %d objects missing, %d failed",
|
||||||
|
prefix, res.Hashed, res.Duplicates, res.Missing, res.Failed)
|
||||||
|
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return res, fmt.Errorf("interrupted after %d/%d photos: %w", res.Hashed, res.Total, err)
|
||||||
|
}
|
||||||
|
if res.Failed > 0 {
|
||||||
|
return res, fmt.Errorf("%d of %d photos failed to hash (rerun to retry; already-hashed photos are skipped)",
|
||||||
|
res.Failed, res.Total)
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashOriginal reads a photo's stored original. Stat runs first because it is
|
||||||
|
// the only call that reports a missing object as storage.ErrNotExist on both
|
||||||
|
// backends — Open surfaces the driver's own error.
|
||||||
|
func hashOriginal(ctx context.Context, st storage.Storage, p store.Photo) (string, error) {
|
||||||
|
if _, err := st.Stat(ctx, p.OriginalKey); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
r, _, err := st.Open(ctx, p.OriginalKey)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
return Sum(r)
|
||||||
|
}
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
package checksum
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const galleryID = "11111111-1111-1111-1111-111111111111"
|
||||||
|
|
||||||
|
// env builds a migrated scratch SQLite database with one gallery, plus a local
|
||||||
|
// storage backend rooted next to it.
|
||||||
|
func env(t *testing.T) (*store.DB, storage.Storage) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
db, err := store.Open(config.Config{DBType: "sqlite", DatabaseURL: filepath.Join(dir, "test.db")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { db.Close() })
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := db.Migrate(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
if err := db.CreateGallery(ctx, store.Gallery{
|
||||||
|
ID: galleryID, Slug: "g", Title: "G", Visibility: "private", ShareToken: "tok",
|
||||||
|
CreatedAt: now, UpdatedAt: now,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
st, err := storage.NewLocal(filepath.Join(dir, "photos"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return db, st
|
||||||
|
}
|
||||||
|
|
||||||
|
// seed inserts an unhashed photo whose original holds body, mimicking a row
|
||||||
|
// uploaded before the checksum column existed.
|
||||||
|
func seed(t *testing.T, db *store.DB, st storage.Storage, id, body string) store.Photo {
|
||||||
|
t.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
key := "galleries/" + galleryID + "/orig/" + id + ".jpg"
|
||||||
|
if body != "" {
|
||||||
|
if err := st.Put(ctx, key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
p := store.Photo{
|
||||||
|
ID: id, GalleryID: galleryID, OriginalKey: key, OriginalFilename: id + ".jpg",
|
||||||
|
ContentType: "image/jpeg", SizeBytes: int64(len(body)), Status: "ready",
|
||||||
|
NextAttemptAt: now, CreatedAt: now, UpdatedAt: now,
|
||||||
|
}
|
||||||
|
if err := db.InsertPhoto(ctx, p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func checksumOf(t *testing.T, db *store.DB, id string) string {
|
||||||
|
t.Helper()
|
||||||
|
p, err := db.GetPhoto(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p.Checksum
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackfill(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("hashes unhashed photos and is idempotent", func(t *testing.T) {
|
||||||
|
db, st := env(t)
|
||||||
|
seed(t, db, st, "p1", "alpha")
|
||||||
|
seed(t, db, st, "p2", "beta")
|
||||||
|
|
||||||
|
res, err := Backfill(ctx, db, st, Options{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Total != 2 || res.Hashed != 2 || res.Duplicates != 0 || res.Failed != 0 {
|
||||||
|
t.Fatalf("first run: %+v", res)
|
||||||
|
}
|
||||||
|
want, err := Sum(strings.NewReader("alpha"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := checksumOf(t, db, "p1"); got != want {
|
||||||
|
t.Fatalf("p1 checksum = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err = Backfill(ctx, db, st, Options{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Total != 0 || res.Hashed != 0 {
|
||||||
|
t.Fatalf("rerun should find nothing to do: %+v", res)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("reports pre-existing duplicates and leaves them unhashed", func(t *testing.T) {
|
||||||
|
db, st := env(t)
|
||||||
|
seed(t, db, st, "p1", "same")
|
||||||
|
seed(t, db, st, "p2", "same")
|
||||||
|
|
||||||
|
res, err := Backfill(ctx, db, st, Options{Concurrency: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Hashed != 1 || res.Duplicates != 1 || res.Failed != 0 {
|
||||||
|
t.Fatalf("got %+v, want 1 hashed and 1 duplicate", res)
|
||||||
|
}
|
||||||
|
// The earlier row keeps the checksum, the later one is left alone —
|
||||||
|
// nothing is deleted either way.
|
||||||
|
if checksumOf(t, db, "p1") == "" {
|
||||||
|
t.Fatal("p1 should have been hashed")
|
||||||
|
}
|
||||||
|
if got := checksumOf(t, db, "p2"); got != "" {
|
||||||
|
t.Fatalf("p2 checksum = %q, want empty", got)
|
||||||
|
}
|
||||||
|
if _, err := db.GetPhoto(ctx, "p2"); err != nil {
|
||||||
|
t.Fatalf("duplicate row must survive: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("skips photos whose object is gone", func(t *testing.T) {
|
||||||
|
db, st := env(t)
|
||||||
|
seed(t, db, st, "p1", "") // row without a stored original
|
||||||
|
|
||||||
|
res, err := Backfill(ctx, db, st, Options{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Missing != 1 || res.Hashed != 0 || res.Failed != 0 {
|
||||||
|
t.Fatalf("got %+v, want 1 missing", res)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("dry run writes nothing", func(t *testing.T) {
|
||||||
|
db, st := env(t)
|
||||||
|
seed(t, db, st, "p1", "alpha")
|
||||||
|
|
||||||
|
res, err := Backfill(ctx, db, st, Options{DryRun: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Hashed != 1 {
|
||||||
|
t.Fatalf("got %+v, want 1 hashed", res)
|
||||||
|
}
|
||||||
|
if got := checksumOf(t, db, "p1"); got != "" {
|
||||||
|
t.Fatalf("dry run recorded %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("limits to one gallery", func(t *testing.T) {
|
||||||
|
db, st := env(t)
|
||||||
|
seed(t, db, st, "p1", "alpha")
|
||||||
|
|
||||||
|
res, err := Backfill(ctx, db, st, Options{GalleryID: "22222222-2222-2222-2222-222222222222"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Total != 0 {
|
||||||
|
t.Fatalf("other gallery should have nothing to do: %+v", res)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
// Package checksum defines the content hash used to spot duplicate photos and
|
||||||
|
// the backfill that fills it in for photos uploaded before it existed.
|
||||||
|
//
|
||||||
|
// The hash is sha256 over the untouched original bytes, hex encoded, stored in
|
||||||
|
// photos_photos.checksum. Duplicate scope is one gallery — the unique index is
|
||||||
|
// on (gallery_id, checksum) — so the same image may still live in several
|
||||||
|
// galleries, each with its own row and its own stored object.
|
||||||
|
package checksum
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"hash"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
// New returns a fresh hasher. Upload hashes as it spools the body, so it needs
|
||||||
|
// the writer rather than a finished reader.
|
||||||
|
func New() hash.Hash { return sha256.New() }
|
||||||
|
|
||||||
|
// Format renders a hasher's digest the way it is stored.
|
||||||
|
func Format(h hash.Hash) string { return hex.EncodeToString(h.Sum(nil)) }
|
||||||
|
|
||||||
|
// Sum reads r to EOF and returns its digest.
|
||||||
|
func Sum(r io.Reader) (string, error) {
|
||||||
|
h := New()
|
||||||
|
if _, err := io.Copy(h, r); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return Format(h), nil
|
||||||
|
}
|
||||||
@@ -20,6 +20,11 @@ type Config struct {
|
|||||||
// JWT_SECRET during the Better Auth migration.
|
// JWT_SECRET during the Better Auth migration.
|
||||||
ViewTokenSecret string
|
ViewTokenSecret string
|
||||||
|
|
||||||
|
// StorageBackend selects the active backend: "auto" (S3 when it is
|
||||||
|
// configured, local otherwise), "local" or "s3". Explicit values let both
|
||||||
|
// backends stay configured — required to run `photo-api sync`, and the
|
||||||
|
// one-line switch after a sync.
|
||||||
|
StorageBackend string
|
||||||
StoragePath string
|
StoragePath string
|
||||||
S3Endpoint string
|
S3Endpoint string
|
||||||
S3Region string
|
S3Region string
|
||||||
@@ -34,12 +39,26 @@ type Config struct {
|
|||||||
HeicConverter string // optional explicit converter command; autodetected when empty
|
HeicConverter string // optional explicit converter command; autodetected when empty
|
||||||
}
|
}
|
||||||
|
|
||||||
// S3Enabled mirrors backend/src/lib/storage.ts: S3 is active when both
|
// S3Configured reports whether the S3 credentials are present at all,
|
||||||
// S3_ENDPOINT and S3_BUCKET are set.
|
// mirroring backend/src/lib/storage.ts: both S3_ENDPOINT and S3_BUCKET set.
|
||||||
func (c Config) S3Enabled() bool {
|
// Kept separate from S3Enabled so a configured-but-inactive S3 backend can
|
||||||
|
// still be reached by `photo-api sync`.
|
||||||
|
func (c Config) S3Configured() bool {
|
||||||
return c.S3Endpoint != "" && c.S3Bucket != ""
|
return c.S3Endpoint != "" && c.S3Bucket != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// S3Enabled reports whether S3 is the backend serving requests.
|
||||||
|
func (c Config) S3Enabled() bool {
|
||||||
|
switch c.StorageBackend {
|
||||||
|
case "s3":
|
||||||
|
return true
|
||||||
|
case "local":
|
||||||
|
return false
|
||||||
|
default: // auto
|
||||||
|
return c.S3Configured()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func Load() (Config, error) {
|
func Load() (Config, error) {
|
||||||
loadDotenv(".env")
|
loadDotenv(".env")
|
||||||
|
|
||||||
@@ -48,6 +67,7 @@ func Load() (Config, error) {
|
|||||||
DBType: strings.ToLower(env("DB_TYPE", "sqlite")),
|
DBType: strings.ToLower(env("DB_TYPE", "sqlite")),
|
||||||
DatabaseURL: env("DATABASE_URL", ""),
|
DatabaseURL: env("DATABASE_URL", ""),
|
||||||
ViewTokenSecret: env("PHOTO_VIEW_SECRET", env("JWT_SECRET", "")),
|
ViewTokenSecret: env("PHOTO_VIEW_SECRET", env("JWT_SECRET", "")),
|
||||||
|
StorageBackend: strings.ToLower(env("STORAGE_BACKEND", "auto")),
|
||||||
StoragePath: env("STORAGE_PATH", "./data/photos"),
|
StoragePath: env("STORAGE_PATH", "./data/photos"),
|
||||||
S3Endpoint: env("S3_ENDPOINT", ""),
|
S3Endpoint: env("S3_ENDPOINT", ""),
|
||||||
S3Region: env("S3_REGION", "auto"),
|
S3Region: env("S3_REGION", "auto"),
|
||||||
@@ -70,6 +90,14 @@ func Load() (Config, error) {
|
|||||||
if cfg.ViewTokenSecret == "" {
|
if cfg.ViewTokenSecret == "" {
|
||||||
return cfg, fmt.Errorf("PHOTO_VIEW_SECRET is required (or legacy JWT_SECRET as fallback)")
|
return cfg, fmt.Errorf("PHOTO_VIEW_SECRET is required (or legacy JWT_SECRET as fallback)")
|
||||||
}
|
}
|
||||||
|
switch cfg.StorageBackend {
|
||||||
|
case "auto", "local", "s3":
|
||||||
|
default:
|
||||||
|
return cfg, fmt.Errorf("STORAGE_BACKEND must be auto, local or s3, got %q", cfg.StorageBackend)
|
||||||
|
}
|
||||||
|
if cfg.StorageBackend == "s3" && !cfg.S3Configured() {
|
||||||
|
return cfg, fmt.Errorf("STORAGE_BACKEND=s3 requires S3_ENDPOINT and S3_BUCKET")
|
||||||
|
}
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
"image"
|
"image"
|
||||||
"image/color"
|
"image/color"
|
||||||
@@ -38,10 +39,11 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type testEnv struct {
|
type testEnv struct {
|
||||||
handler http.Handler
|
handler http.Handler
|
||||||
db *store.DB
|
db *store.DB
|
||||||
worker *worker.Worker
|
worker *worker.Worker
|
||||||
pg bool
|
storagePath string
|
||||||
|
pg bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// setup migrates a scratch DB (SQLite by default; Postgres when
|
// setup migrates a scratch DB (SQLite by default; Postgres when
|
||||||
@@ -123,7 +125,13 @@ func setup(t *testing.T) *testEnv {
|
|||||||
}
|
}
|
||||||
wrk := worker.New(db, st, nil, cfg.StoragePath, 1)
|
wrk := worker.New(db, st, nil, cfg.StoragePath, 1)
|
||||||
srv := New(cfg, db, st, auth.NewVerifier(db), wrk)
|
srv := New(cfg, db, st, auth.NewVerifier(db), wrk)
|
||||||
return &testEnv{handler: srv.Handler(), db: db, worker: wrk, pg: cfg.DBType == "postgres"}
|
return &testEnv{
|
||||||
|
handler: srv.Handler(),
|
||||||
|
db: db,
|
||||||
|
worker: wrk,
|
||||||
|
storagePath: cfg.StoragePath,
|
||||||
|
pg: cfg.DBType == "postgres",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// makeToken inserts a Better Auth session row for the user and returns a
|
// makeToken inserts a Better Auth session row for the user and returns a
|
||||||
@@ -175,12 +183,16 @@ func decode[T any](t *testing.T, w *httptest.ResponseRecorder) T {
|
|||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|
||||||
func testJPEG(t *testing.T) []byte {
|
func testJPEG(t *testing.T) []byte { return testJPEGTinted(t, 128) }
|
||||||
|
|
||||||
|
// testJPEGTinted varies the blue channel so tests that need two *different*
|
||||||
|
// images (duplicate detection) can get them without a fixture file.
|
||||||
|
func testJPEGTinted(t *testing.T, blue uint8) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
img := image.NewRGBA(image.Rect(0, 0, 800, 600))
|
img := image.NewRGBA(image.Rect(0, 0, 800, 600))
|
||||||
for x := 0; x < 800; x += 10 {
|
for x := 0; x < 800; x += 10 {
|
||||||
for y := 0; y < 600; y++ {
|
for y := 0; y < 600; y++ {
|
||||||
img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: 128, A: 255})
|
img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: blue, A: 255})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
@@ -296,6 +308,130 @@ func uploadPhoto(t *testing.T, e *testEnv, admin, galleryID string, file []byte)
|
|||||||
return photos[0]
|
return photos[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// uploadFiles posts several parts in one request, the way the API allows even
|
||||||
|
// though the admin UI sends one file per request.
|
||||||
|
func uploadFiles(t *testing.T, e *testEnv, admin, galleryID string, files ...[]byte) []photoJSON {
|
||||||
|
t.Helper()
|
||||||
|
var buf bytes.Buffer
|
||||||
|
mw := multipart.NewWriter(&buf)
|
||||||
|
for i, f := range files {
|
||||||
|
fw, _ := mw.CreateFormFile("files", fmt.Sprintf("photo-%d.jpg", i))
|
||||||
|
fw.Write(f)
|
||||||
|
}
|
||||||
|
mw.Close()
|
||||||
|
req := httptest.NewRequest("POST", "/api/photos/galleries/"+galleryID+"/photos", &buf)
|
||||||
|
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||||
|
req.Header.Set("Authorization", "Bearer "+admin)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
e.handler.ServeHTTP(w, req)
|
||||||
|
if w.Code != 201 {
|
||||||
|
t.Fatalf("upload: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
return decode[struct {
|
||||||
|
Photos []photoJSON `json:"photos"`
|
||||||
|
}](t, w).Photos
|
||||||
|
}
|
||||||
|
|
||||||
|
// countOriginals is how many objects actually landed on disk for a gallery —
|
||||||
|
// a duplicate must not add one.
|
||||||
|
func countOriginals(t *testing.T, e *testEnv, galleryID string) int {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir(filepath.Join(e.storagePath, "galleries", galleryID, "orig"))
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return len(entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadSkipsDuplicates(t *testing.T) {
|
||||||
|
e := setup(t)
|
||||||
|
admin := e.makeToken(t, uAdmin)
|
||||||
|
g := createGallery(t, e, admin, map[string]any{"title": "Dupes"})
|
||||||
|
img := testJPEG(t)
|
||||||
|
|
||||||
|
first := uploadPhoto(t, e, admin, g.ID, img)
|
||||||
|
if first.Duplicate {
|
||||||
|
t.Fatalf("first upload flagged as duplicate: %+v", first)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same bytes again: echoed back as the existing photo, nothing stored.
|
||||||
|
second := uploadPhoto(t, e, admin, g.ID, img)
|
||||||
|
if !second.Duplicate {
|
||||||
|
t.Fatalf("second upload not flagged as duplicate: %+v", second)
|
||||||
|
}
|
||||||
|
if second.ID != first.ID {
|
||||||
|
t.Fatalf("duplicate should echo the existing photo: got %s, want %s", second.ID, first.ID)
|
||||||
|
}
|
||||||
|
photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(photos) != 1 {
|
||||||
|
t.Fatalf("want 1 row after re-upload, got %d", len(photos))
|
||||||
|
}
|
||||||
|
if n := countOriginals(t, e, g.ID); n != 1 {
|
||||||
|
t.Fatalf("want 1 stored original after re-upload, got %d", n)
|
||||||
|
}
|
||||||
|
if photos[0].Checksum == "" {
|
||||||
|
t.Fatal("checksum was not recorded")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A different image is unaffected and takes the next position.
|
||||||
|
other := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))
|
||||||
|
if other.Duplicate || other.ID == first.ID {
|
||||||
|
t.Fatalf("distinct image treated as duplicate: %+v", other)
|
||||||
|
}
|
||||||
|
if other.Position != 1 {
|
||||||
|
t.Fatalf("want position 1 for the second distinct photo, got %d", other.Position)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Duplicate scope is one gallery: the same bytes elsewhere upload normally.
|
||||||
|
g2 := createGallery(t, e, admin, map[string]any{"title": "Other gallery"})
|
||||||
|
elsewhere := uploadPhoto(t, e, admin, g2.ID, img)
|
||||||
|
if elsewhere.Duplicate {
|
||||||
|
t.Fatalf("same image in another gallery must not be a duplicate: %+v", elsewhere)
|
||||||
|
}
|
||||||
|
if elsewhere.ID == first.ID {
|
||||||
|
t.Fatal("second gallery should get its own photo row")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadBatchContinuesPastDuplicate(t *testing.T) {
|
||||||
|
e := setup(t)
|
||||||
|
admin := e.makeToken(t, uAdmin)
|
||||||
|
g := createGallery(t, e, admin, map[string]any{"title": "Batch"})
|
||||||
|
a, b, c := testJPEG(t), testJPEGTinted(t, 32), testJPEGTinted(t, 200)
|
||||||
|
|
||||||
|
if got := uploadFiles(t, e, admin, g.ID, a); len(got) != 1 {
|
||||||
|
t.Fatalf("seed upload: %d photos", len(got))
|
||||||
|
}
|
||||||
|
// b duplicates nothing, a is already there, c is new: the batch must not
|
||||||
|
// abort, and the duplicate must not consume a position.
|
||||||
|
batch := uploadFiles(t, e, admin, g.ID, b, a, c)
|
||||||
|
if len(batch) != 3 {
|
||||||
|
t.Fatalf("want 3 results, got %d", len(batch))
|
||||||
|
}
|
||||||
|
if batch[0].Duplicate || !batch[1].Duplicate || batch[2].Duplicate {
|
||||||
|
t.Fatalf("duplicate flags: %v %v %v", batch[0].Duplicate, batch[1].Duplicate, batch[2].Duplicate)
|
||||||
|
}
|
||||||
|
if batch[0].Position != 1 || batch[2].Position != 2 {
|
||||||
|
t.Fatalf("positions should stay dense: %d, %d", batch[0].Position, batch[2].Position)
|
||||||
|
}
|
||||||
|
photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(photos) != 3 {
|
||||||
|
t.Fatalf("want 3 rows, got %d", len(photos))
|
||||||
|
}
|
||||||
|
if n := countOriginals(t, e, g.ID); n != 3 {
|
||||||
|
t.Fatalf("want 3 stored originals, got %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// processQueue runs the worker until the photo is ready or failed.
|
// processQueue runs the worker until the photo is ready or failed.
|
||||||
func processQueue(t *testing.T, e *testEnv, photoID string) store.Photo {
|
func processQueue(t *testing.T, e *testEnv, photoID string) store.Photo {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -465,9 +601,9 @@ func TestReorderAndVisibilityUpdate(t *testing.T) {
|
|||||||
e := setup(t)
|
e := setup(t)
|
||||||
admin := e.makeToken(t, uAdmin)
|
admin := e.makeToken(t, uAdmin)
|
||||||
g := createGallery(t, e, admin, map[string]any{"title": "Orden", "visibility": "public"})
|
g := createGallery(t, e, admin, map[string]any{"title": "Orden", "visibility": "public"})
|
||||||
jpg := testJPEG(t)
|
// Two distinct images: the same bytes twice would be deduplicated.
|
||||||
p1 := uploadPhoto(t, e, admin, g.ID, jpg)
|
p1 := uploadPhoto(t, e, admin, g.ID, testJPEG(t))
|
||||||
p2 := uploadPhoto(t, e, admin, g.ID, jpg)
|
p2 := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))
|
||||||
|
|
||||||
if w := e.request(t, "PATCH", "/api/photos/galleries/"+g.ID+"/order", admin,
|
if w := e.request(t, "PATCH", "/api/photos/galleries/"+g.ID+"/order", admin,
|
||||||
map[string]any{"photoIds": []string{p2.ID, p1.ID}}); w.Code != 200 {
|
map[string]any{"photoIds": []string{p2.ID, p1.ID}}); w.Code != 200 {
|
||||||
|
|||||||
@@ -58,6 +58,10 @@ type photoJSON struct {
|
|||||||
LastError string `json:"lastError,omitempty"` // admin only
|
LastError string `json:"lastError,omitempty"` // admin only
|
||||||
CreatedAt string `json:"createdAt"`
|
CreatedAt string `json:"createdAt"`
|
||||||
URLs photoURLs `json:"urls"`
|
URLs photoURLs `json:"urls"`
|
||||||
|
// Duplicate marks an upload that was skipped because the gallery already
|
||||||
|
// held these bytes; the rest of the object describes the existing photo.
|
||||||
|
// Set by the upload handler only, never persisted.
|
||||||
|
Duplicate bool `json:"duplicate,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// viewTokenFor returns the token to embed in a gallery's file URLs: none
|
// viewTokenFor returns the token to embed in a gallery's file URLs: none
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||||
)
|
)
|
||||||
@@ -20,6 +21,10 @@ import (
|
|||||||
// Each file is sniffed by magic bytes (client filename/Content-Type are
|
// Each file is sniffed by magic bytes (client filename/Content-Type are
|
||||||
// untrusted, same policy as /api/media/upload), stored as the untouched
|
// untrusted, same policy as /api/media/upload), stored as the untouched
|
||||||
// original, and queued for variant processing.
|
// original, and queued for variant processing.
|
||||||
|
//
|
||||||
|
// A file whose bytes are already in this gallery is not stored a second time:
|
||||||
|
// the incoming copy is discarded, the existing photo is echoed back with
|
||||||
|
// "duplicate": true, and the rest of the batch carries on.
|
||||||
func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.User) {
|
func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.User) {
|
||||||
galleryID := r.PathValue("id")
|
galleryID := r.PathValue("id")
|
||||||
g, err := s.db.GetGallery(r.Context(), galleryID)
|
g, err := s.db.GetGallery(r.Context(), galleryID)
|
||||||
@@ -57,7 +62,7 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
|
|||||||
part.Close()
|
part.Close()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
photo, uploadErr := s.saveUpload(r, g, part, position, maxFile)
|
photo, duplicate, uploadErr := s.saveUpload(r, g, part, position, maxFile)
|
||||||
part.Close()
|
part.Close()
|
||||||
if uploadErr != nil {
|
if uploadErr != nil {
|
||||||
// One bad file fails the request explicitly rather than silently
|
// One bad file fails the request explicitly rather than silently
|
||||||
@@ -65,7 +70,12 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
|
|||||||
writeError(w, uploadErr.status, uploadErr.msg)
|
writeError(w, uploadErr.status, uploadErr.msg)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
created = append(created, s.photoToJSON(photo, s.viewTokenFor(g), true))
|
out := s.photoToJSON(photo, s.viewTokenFor(g), true)
|
||||||
|
out.Duplicate = duplicate
|
||||||
|
created = append(created, out)
|
||||||
|
if duplicate {
|
||||||
|
continue // nothing was stored, so the next file keeps this position
|
||||||
|
}
|
||||||
position++
|
position++
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -82,51 +92,67 @@ type uploadError struct {
|
|||||||
msg string
|
msg string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, *uploadError) {
|
// saveUpload stores one part. The bool it returns reports a duplicate: the
|
||||||
|
// gallery already holds these bytes, so nothing was written and the photo
|
||||||
|
// returned is the existing one.
|
||||||
|
func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, bool, *uploadError) {
|
||||||
head := make([]byte, 16)
|
head := make([]byte, 16)
|
||||||
n, err := io.ReadFull(part, head)
|
n, err := io.ReadFull(part, head)
|
||||||
if err != nil && err != io.ErrUnexpectedEOF {
|
if err != nil && err != io.ErrUnexpectedEOF {
|
||||||
return store.Photo{}, &uploadError{http.StatusBadRequest, "Could not read file"}
|
return store.Photo{}, false, &uploadError{http.StatusBadRequest, "Could not read file"}
|
||||||
}
|
}
|
||||||
head = head[:n]
|
head = head[:n]
|
||||||
contentType, ext, ok, reason := imaging.Sniff(head)
|
contentType, ext, ok, reason := imaging.Sniff(head)
|
||||||
if !ok {
|
if !ok {
|
||||||
return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType, reason}
|
return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType, reason}
|
||||||
}
|
}
|
||||||
if contentType == "image/heic" && imaging.DetectHeicConverter(s.cfg.HeicConverter) == nil {
|
if contentType == "image/heic" && imaging.DetectHeicConverter(s.cfg.HeicConverter) == nil {
|
||||||
return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType,
|
return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType,
|
||||||
"HEIC uploads need an image converter on the server (install libvips-tools); please upload JPEG instead"}
|
"HEIC uploads need an image converter on the server (install libvips-tools); please upload JPEG instead"}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Spool to a temp file to learn the size before handing to storage
|
// Spool to a temp file to learn the size before handing to storage
|
||||||
// (S3 wants a length; local rename wants a file anyway).
|
// (S3 wants a length; local rename wants a file anyway). The same pass
|
||||||
|
// hashes the bytes for the duplicate check below.
|
||||||
tmp, err := os.CreateTemp(s.cfg.StoragePath, ".incoming-*")
|
tmp, err := os.CreateTemp(s.cfg.StoragePath, ".incoming-*")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Error: %v", err)
|
log.Printf("Error: %v", err)
|
||||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||||
}
|
}
|
||||||
defer os.Remove(tmp.Name())
|
defer os.Remove(tmp.Name())
|
||||||
defer tmp.Close()
|
defer tmp.Close()
|
||||||
|
|
||||||
size, err := io.Copy(tmp, io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
|
hasher := checksum.New()
|
||||||
|
size, err := io.Copy(io.MultiWriter(tmp, hasher),
|
||||||
|
io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Error: %v", err)
|
log.Printf("Error: %v", err)
|
||||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||||
}
|
}
|
||||||
if size > maxFile {
|
if size > maxFile {
|
||||||
return store.Photo{}, &uploadError{http.StatusRequestEntityTooLarge,
|
return store.Photo{}, false, &uploadError{http.StatusRequestEntityTooLarge,
|
||||||
fmt.Sprintf("File exceeds the %d MB limit", s.cfg.MaxUploadMB)}
|
fmt.Sprintf("File exceeds the %d MB limit", s.cfg.MaxUploadMB)}
|
||||||
}
|
}
|
||||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||||
log.Printf("Error: %v", err)
|
log.Printf("Error: %v", err)
|
||||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||||
|
}
|
||||||
|
sum := checksum.Format(hasher)
|
||||||
|
|
||||||
|
// Duplicate check before anything is written: the temp copy is dropped by
|
||||||
|
// the deferred Remove, no object is stored and no row is inserted.
|
||||||
|
if existing, err := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); err == nil {
|
||||||
|
return existing, true, nil
|
||||||
|
} else if err != store.ErrNotFound {
|
||||||
|
log.Printf("Error: %v", err)
|
||||||
|
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||||
}
|
}
|
||||||
|
|
||||||
photoID := newID()
|
photoID := newID()
|
||||||
key := fmt.Sprintf("galleries/%s/orig/%s%s", g.ID, photoID, ext)
|
key := fmt.Sprintf("galleries/%s/orig/%s%s", g.ID, photoID, ext)
|
||||||
if err := s.storage.Put(r.Context(), key, tmp, size, contentType); err != nil {
|
if err := s.storage.Put(r.Context(), key, tmp, size, contentType); err != nil {
|
||||||
log.Printf("Error: %v", err)
|
log.Printf("Error: %v", err)
|
||||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
@@ -142,13 +168,22 @@ func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Pa
|
|||||||
NextAttemptAt: now,
|
NextAttemptAt: now,
|
||||||
CreatedAt: now,
|
CreatedAt: now,
|
||||||
UpdatedAt: now,
|
UpdatedAt: now,
|
||||||
|
Checksum: sum,
|
||||||
}
|
}
|
||||||
if err := s.db.InsertPhoto(r.Context(), photo); err != nil {
|
if err := s.db.InsertPhoto(r.Context(), photo); err != nil {
|
||||||
s.storage.Delete(r.Context(), key)
|
s.storage.Delete(r.Context(), key)
|
||||||
|
// A concurrent upload of the same bytes won the race between the
|
||||||
|
// check above and this insert; the unique index caught it, so report
|
||||||
|
// the winner as the duplicate instead of failing.
|
||||||
|
if store.IsUniqueViolation(err) {
|
||||||
|
if existing, findErr := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); findErr == nil {
|
||||||
|
return existing, true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
log.Printf("Error: %v", err)
|
log.Printf("Error: %v", err)
|
||||||
return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
|
||||||
}
|
}
|
||||||
return photo, nil
|
return photo, false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type reorderBody struct {
|
type reorderBody struct {
|
||||||
|
|||||||
@@ -0,0 +1,229 @@
|
|||||||
|
// Package photosync copies the photo library between the local-disk and the
|
||||||
|
// S3 backend so the active one (STORAGE_BACKEND) can be switched without
|
||||||
|
// losing photos. Both backends must be configured in photo-api/.env for a
|
||||||
|
// sync to run — the direction picks which is the source.
|
||||||
|
//
|
||||||
|
// The photos_photos rows are the inventory: every row contributes its
|
||||||
|
// original key and, once the worker has processed it, its thumb and preview
|
||||||
|
// key. Objects on disk or in the bucket with no row (worker scratch dirs,
|
||||||
|
// leftovers of deleted galleries) are deliberately not copied.
|
||||||
|
//
|
||||||
|
// Sync only ever writes to the destination. The source is left untouched, so
|
||||||
|
// a sync is repeatable, safe to interrupt, and leaves the old backend as a
|
||||||
|
// fallback until it is cleaned up by hand.
|
||||||
|
package photosync
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Direction is which way objects move.
|
||||||
|
type Direction string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ToS3 Direction = "to-s3"
|
||||||
|
ToLocal Direction = "to-local"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParseDirection accepts the short and the spelled-out form.
|
||||||
|
func ParseDirection(s string) (Direction, error) {
|
||||||
|
switch s {
|
||||||
|
case "to-s3", "local-to-s3":
|
||||||
|
return ToS3, nil
|
||||||
|
case "to-local", "s3-to-local":
|
||||||
|
return ToLocal, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unknown direction %q (expected to-s3 or to-local)", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type Options struct {
|
||||||
|
Direction Direction
|
||||||
|
// GalleryID limits the sync to one gallery; empty means the whole library.
|
||||||
|
GalleryID string
|
||||||
|
// Concurrency is how many objects are copied at a time.
|
||||||
|
Concurrency int
|
||||||
|
// DryRun reports what would be copied without writing anything.
|
||||||
|
DryRun bool
|
||||||
|
// Overwrite re-copies objects that already exist on the destination with
|
||||||
|
// the same size (default: those are skipped, which makes reruns cheap).
|
||||||
|
Overwrite bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type Result struct {
|
||||||
|
Total int // objects in the inventory
|
||||||
|
Copied int // copied (or, with DryRun, would be copied)
|
||||||
|
Skipped int // already on the destination
|
||||||
|
Missing int // absent from the source — nothing to copy
|
||||||
|
Failed int // copy attempted and errored
|
||||||
|
Bytes int64 // bytes copied
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run copies the library in the requested direction. It returns a Result even
|
||||||
|
// on error, and an error if any object failed.
|
||||||
|
func Run(ctx context.Context, cfg config.Config, db *store.DB, opts Options) (Result, error) {
|
||||||
|
if !cfg.S3Configured() {
|
||||||
|
return Result{}, errors.New("sync needs both backends configured: set S3_ENDPOINT, S3_BUCKET, " +
|
||||||
|
"S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY (next to STORAGE_PATH) in photo-api/.env")
|
||||||
|
}
|
||||||
|
if opts.Concurrency < 1 {
|
||||||
|
opts.Concurrency = 4
|
||||||
|
}
|
||||||
|
|
||||||
|
local, err := storage.NewLocal(cfg.StoragePath)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, fmt.Errorf("local storage: %w", err)
|
||||||
|
}
|
||||||
|
s3, err := storage.NewS3(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, fmt.Errorf("s3 storage: %w", err)
|
||||||
|
}
|
||||||
|
src, dst := local, s3
|
||||||
|
srcName, dstName := "local disk "+cfg.StoragePath, "s3 bucket "+cfg.S3Bucket
|
||||||
|
if opts.Direction == ToLocal {
|
||||||
|
src, dst = s3, local
|
||||||
|
srcName, dstName = dstName, srcName
|
||||||
|
}
|
||||||
|
|
||||||
|
objects, err := db.AllPhotoObjects(ctx, opts.GalleryID)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, fmt.Errorf("list photo objects: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prefix := ""
|
||||||
|
if opts.DryRun {
|
||||||
|
prefix = "[dry-run] "
|
||||||
|
}
|
||||||
|
log.Printf("sync: %s%s → %s: %d objects, concurrency %d",
|
||||||
|
prefix, srcName, dstName, len(objects), opts.Concurrency)
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.Mutex
|
||||||
|
res = Result{Total: len(objects)}
|
||||||
|
done int64
|
||||||
|
jobs = make(chan store.PhotoObject)
|
||||||
|
wg sync.WaitGroup
|
||||||
|
)
|
||||||
|
for i := 0; i < opts.Concurrency; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for o := range jobs {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
act, size, err := copyObject(ctx, src, dst, o, opts)
|
||||||
|
n := atomic.AddInt64(&done, 1)
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
res.Failed++
|
||||||
|
log.Printf("sync: [%d/%d] FAILED %s (photo %s %s): %v",
|
||||||
|
n, len(objects), o.Key, o.PhotoID, o.Variant, err)
|
||||||
|
case act == actionCopied:
|
||||||
|
res.Copied++
|
||||||
|
res.Bytes += size
|
||||||
|
log.Printf("sync: [%d/%d] %scopied %s (%s)", n, len(objects), prefix, o.Key, humanBytes(size))
|
||||||
|
case act == actionMissing:
|
||||||
|
res.Missing++
|
||||||
|
log.Printf("sync: [%d/%d] MISSING on source, skipped: %s (photo %s %s)",
|
||||||
|
n, len(objects), o.Key, o.PhotoID, o.Variant)
|
||||||
|
default:
|
||||||
|
res.Skipped++
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
for _, o := range objects {
|
||||||
|
select {
|
||||||
|
case jobs <- o:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
close(jobs)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
log.Printf("sync: %sdone — %d copied (%s), %d already present, %d missing on source, %d failed",
|
||||||
|
prefix, res.Copied, humanBytes(res.Bytes), res.Skipped, res.Missing, res.Failed)
|
||||||
|
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return res, fmt.Errorf("interrupted after %d/%d objects: %w", res.Copied+res.Skipped, res.Total, err)
|
||||||
|
}
|
||||||
|
if res.Failed > 0 {
|
||||||
|
return res, fmt.Errorf("%d of %d objects failed to copy (rerun to retry; already-copied objects are skipped)",
|
||||||
|
res.Failed, res.Total)
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type action int
|
||||||
|
|
||||||
|
const (
|
||||||
|
actionCopied action = iota
|
||||||
|
actionSkipped
|
||||||
|
actionMissing
|
||||||
|
)
|
||||||
|
|
||||||
|
func copyObject(ctx context.Context, src, dst storage.Storage, o store.PhotoObject, opts Options) (action, int64, error) {
|
||||||
|
srcSize, err := src.Stat(ctx, o.Key)
|
||||||
|
if errors.Is(err, storage.ErrNotExist) {
|
||||||
|
return actionMissing, 0, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return actionSkipped, 0, fmt.Errorf("stat source: %w", err)
|
||||||
|
}
|
||||||
|
if !opts.Overwrite {
|
||||||
|
if dstSize, err := dst.Stat(ctx, o.Key); err == nil && dstSize == srcSize {
|
||||||
|
return actionSkipped, 0, nil
|
||||||
|
} else if err != nil && !errors.Is(err, storage.ErrNotExist) {
|
||||||
|
return actionSkipped, 0, fmt.Errorf("stat destination: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if opts.DryRun {
|
||||||
|
return actionCopied, srcSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
r, size, err := src.Open(ctx, o.Key)
|
||||||
|
if err != nil {
|
||||||
|
return actionSkipped, 0, fmt.Errorf("read source: %w", err)
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
if size <= 0 {
|
||||||
|
size = srcSize // local Open reports the stat size; be defensive anyway
|
||||||
|
}
|
||||||
|
contentType := o.ContentType
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
if err := dst.Put(ctx, o.Key, r, size, contentType); err != nil {
|
||||||
|
return actionSkipped, 0, fmt.Errorf("write destination: %w", err)
|
||||||
|
}
|
||||||
|
return actionCopied, size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func humanBytes(n int64) string {
|
||||||
|
const unit = 1024
|
||||||
|
if n < unit {
|
||||||
|
return fmt.Sprintf("%d B", n)
|
||||||
|
}
|
||||||
|
v, exp := float64(n), 0
|
||||||
|
for v >= unit && exp < 4 {
|
||||||
|
v /= unit
|
||||||
|
exp++
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.1f %ciB", v, "KMGT"[exp-1])
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
package photosync
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Two local backends stand in for the real pair: copyObject only talks to the
|
||||||
|
// storage.Storage interface, so the S3 side needs no fake here.
|
||||||
|
func backends(t *testing.T) (src, dst storage.Storage, srcRoot string) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
srcRoot = filepath.Join(dir, "src")
|
||||||
|
src, err := storage.NewLocal(srcRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dst, err = storage.NewLocal(filepath.Join(dir, "dst"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return src, dst, srcRoot
|
||||||
|
}
|
||||||
|
|
||||||
|
func put(t *testing.T, s storage.Storage, key, body string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := s.Put(context.Background(), key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const testKey = "galleries/g1/original/p1.jpg"
|
||||||
|
|
||||||
|
func testObject() store.PhotoObject {
|
||||||
|
return store.PhotoObject{PhotoID: "p1", GalleryID: "g1", Variant: "original", Key: testKey, ContentType: "image/jpeg"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyObject(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("copies to destination", func(t *testing.T) {
|
||||||
|
src, dst, _ := backends(t)
|
||||||
|
put(t, src, testKey, "hello-photo")
|
||||||
|
|
||||||
|
act, size, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||||
|
if err != nil || act != actionCopied || size != 11 {
|
||||||
|
t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err)
|
||||||
|
}
|
||||||
|
if got, err := dst.Stat(ctx, testKey); err != nil || got != 11 {
|
||||||
|
t.Fatalf("destination stat: (%d, %v)", got, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("skips same-size object already present", func(t *testing.T) {
|
||||||
|
src, dst, _ := backends(t)
|
||||||
|
put(t, src, testKey, "hello-photo")
|
||||||
|
put(t, dst, testKey, "hello-photo")
|
||||||
|
|
||||||
|
act, _, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||||
|
if err != nil || act != actionSkipped {
|
||||||
|
t.Fatalf("got (%v, %v), want (skipped, nil)", act, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("overwrite re-copies", func(t *testing.T) {
|
||||||
|
src, dst, _ := backends(t)
|
||||||
|
put(t, src, testKey, "new-content")
|
||||||
|
put(t, dst, testKey, "old-content")
|
||||||
|
|
||||||
|
act, _, err := copyObject(ctx, src, dst, testObject(), Options{Overwrite: true})
|
||||||
|
if err != nil || act != actionCopied {
|
||||||
|
t.Fatalf("got (%v, %v), want (copied, nil)", act, err)
|
||||||
|
}
|
||||||
|
r, _, err := dst.Open(ctx, testKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
body, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(body) != "new-content" {
|
||||||
|
t.Fatalf("destination body = %q, want new-content", body)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("reports objects missing on the source", func(t *testing.T) {
|
||||||
|
src, dst, _ := backends(t)
|
||||||
|
|
||||||
|
act, _, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||||
|
if err != nil || act != actionMissing {
|
||||||
|
t.Fatalf("got (%v, %v), want (missing, nil)", act, err)
|
||||||
|
}
|
||||||
|
if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist {
|
||||||
|
t.Fatalf("destination stat err = %v, want ErrNotExist", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("dry run writes nothing", func(t *testing.T) {
|
||||||
|
src, dst, _ := backends(t)
|
||||||
|
put(t, src, testKey, "hello-photo")
|
||||||
|
|
||||||
|
act, size, err := copyObject(ctx, src, dst, testObject(), Options{DryRun: true})
|
||||||
|
if err != nil || act != actionCopied || size != 11 {
|
||||||
|
t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err)
|
||||||
|
}
|
||||||
|
if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist {
|
||||||
|
t.Fatalf("destination stat err = %v, want ErrNotExist", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDirection(t *testing.T) {
|
||||||
|
for in, want := range map[string]Direction{
|
||||||
|
"to-s3": ToS3,
|
||||||
|
"local-to-s3": ToS3,
|
||||||
|
"to-local": ToLocal,
|
||||||
|
"s3-to-local": ToLocal,
|
||||||
|
} {
|
||||||
|
got, err := ParseDirection(in)
|
||||||
|
if err != nil || got != want {
|
||||||
|
t.Errorf("ParseDirection(%q) = (%v, %v), want %v", in, got, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := ParseDirection("sideways"); err == nil {
|
||||||
|
t.Error("ParseDirection(\"sideways\") should fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,262 @@
|
|||||||
|
package photosync
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||||
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stubS3 is a minimal path-style S3 (PUT/HEAD/GET on /bucket/key) so the sync
|
||||||
|
// can be driven end to end over the real aws-sdk client.
|
||||||
|
type stubS3 struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
objects map[string][]byte
|
||||||
|
puts int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStubS3(t *testing.T) (*stubS3, string) {
|
||||||
|
t.Helper()
|
||||||
|
s := &stubS3{objects: map[string][]byte{}}
|
||||||
|
srv := httptest.NewServer(s)
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return s, srv.URL
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubS3) get(key string) ([]byte, bool) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
b, ok := s.objects[key]
|
||||||
|
return b, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubS3) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
key := strings.TrimPrefix(r.URL.Path, "/test-bucket/")
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodPut:
|
||||||
|
body, err := readS3Body(r)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
s.objects[key] = body
|
||||||
|
s.puts++
|
||||||
|
s.mu.Unlock()
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case http.MethodHead:
|
||||||
|
body, ok := s.get(key)
|
||||||
|
if !ok {
|
||||||
|
w.WriteHeader(http.StatusNotFound) // a HEAD carries no error body
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case http.MethodGet:
|
||||||
|
body, ok := s.get(key)
|
||||||
|
if !ok {
|
||||||
|
w.Header().Set("Content-Type", "application/xml")
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
fmt.Fprint(w, `<Error><Code>NoSuchKey</Code></Error>`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||||
|
w.Write(body)
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readS3Body undoes the SDK's aws-chunked framing when it streams with a
|
||||||
|
// trailing checksum (what it does for the non-seekable S3-to-S3 style reader).
|
||||||
|
func readS3Body(r *http.Request) ([]byte, error) {
|
||||||
|
raw, err := io.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") {
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
var out []byte
|
||||||
|
rest := raw
|
||||||
|
for {
|
||||||
|
nl := strings.Index(string(rest), "\r\n")
|
||||||
|
if nl < 0 {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
header := string(rest[:nl])
|
||||||
|
rest = rest[nl+2:]
|
||||||
|
size, err := strconv.ParseInt(strings.SplitN(header, ";", 2)[0], 16, 64)
|
||||||
|
if err != nil || size == 0 {
|
||||||
|
return out, nil // trailer section or malformed: body is complete
|
||||||
|
}
|
||||||
|
if int64(len(rest)) < size {
|
||||||
|
return nil, fmt.Errorf("truncated aws-chunked body")
|
||||||
|
}
|
||||||
|
out = append(out, rest[:size]...)
|
||||||
|
rest = rest[size:]
|
||||||
|
if len(rest) >= 2 {
|
||||||
|
rest = rest[2:] // chunk CRLF
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncEnv seeds a scratch SQLite DB with one gallery holding one ready photo
|
||||||
|
// (original + thumb + preview) and returns a config wired to the stub bucket.
|
||||||
|
func syncEnv(t *testing.T) (config.Config, *store.DB, *stubS3, []string) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
stub, endpoint := newStubS3(t)
|
||||||
|
cfg := config.Config{
|
||||||
|
DBType: "sqlite",
|
||||||
|
DatabaseURL: filepath.Join(dir, "test.db"),
|
||||||
|
ViewTokenSecret: "test-secret",
|
||||||
|
StoragePath: filepath.Join(dir, "photos"),
|
||||||
|
S3Endpoint: endpoint,
|
||||||
|
S3Region: "auto",
|
||||||
|
S3Bucket: "test-bucket",
|
||||||
|
S3AccessKeyID: "key",
|
||||||
|
S3SecretKey: "secret",
|
||||||
|
S3ForcePathStyle: true,
|
||||||
|
}
|
||||||
|
db, err := store.Open(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { db.Close() })
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := db.Migrate(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
galleryID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
|
||||||
|
photoID = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
|
||||||
|
)
|
||||||
|
now := time.Now()
|
||||||
|
if err := db.CreateGallery(ctx, store.Gallery{
|
||||||
|
ID: galleryID, Slug: "trip", Title: "Trip", Visibility: store.VisibilityPublic,
|
||||||
|
ShareToken: "tok", CreatedAt: now, UpdatedAt: now,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
origKey := "galleries/" + galleryID + "/original/" + photoID + ".jpg"
|
||||||
|
thumbKey := "galleries/" + galleryID + "/thumb/" + photoID + ".jpg"
|
||||||
|
previewKey := "galleries/" + galleryID + "/preview/" + photoID + ".jpg"
|
||||||
|
if err := db.InsertPhoto(ctx, store.Photo{
|
||||||
|
ID: photoID, GalleryID: galleryID, OriginalKey: origKey, ContentType: "image/jpeg",
|
||||||
|
SizeBytes: 11, NextAttemptAt: now, CreatedAt: now, UpdatedAt: now,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := db.MarkPhotoReady(ctx, photoID, thumbKey, previewKey, 100, 80, now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return cfg, db, stub, []string{origKey, thumbKey, previewKey}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunToS3(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
cfg, db, stub, keys := syncEnv(t)
|
||||||
|
|
||||||
|
src, err := storage.NewLocal(cfg.StoragePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i, k := range keys {
|
||||||
|
put(t, src, k, fmt.Sprintf("photo-bytes-%d", i))
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := Run(ctx, cfg, db, Options{Direction: ToS3, Concurrency: 2})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Total != 3 || res.Copied != 3 || res.Skipped != 0 || res.Failed != 0 || res.Missing != 0 {
|
||||||
|
t.Fatalf("first run = %+v, want 3 total / 3 copied", res)
|
||||||
|
}
|
||||||
|
for i, k := range keys {
|
||||||
|
body, ok := stub.get(k)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%s not uploaded", k)
|
||||||
|
}
|
||||||
|
if want := fmt.Sprintf("photo-bytes-%d", i); string(body) != want {
|
||||||
|
t.Errorf("%s = %q, want %q", k, body, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rerunning is a no-op: everything is already there at the same size.
|
||||||
|
res, err = Run(ctx, cfg, db, Options{Direction: ToS3})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Copied != 0 || res.Skipped != 3 {
|
||||||
|
t.Fatalf("rerun = %+v, want 0 copied / 3 skipped", res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --overwrite re-uploads them.
|
||||||
|
before := stub.puts
|
||||||
|
res, err = Run(ctx, cfg, db, Options{Direction: ToS3, Overwrite: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Copied != 3 || stub.puts != before+3 {
|
||||||
|
t.Fatalf("overwrite run = %+v, puts %d → %d", res, before, stub.puts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunToLocal(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
cfg, db, stub, keys := syncEnv(t)
|
||||||
|
for i, k := range keys {
|
||||||
|
stub.objects[k] = []byte(fmt.Sprintf("s3-bytes-%d", i))
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := Run(ctx, cfg, db, Options{Direction: ToLocal})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Total != 3 || res.Copied != 3 || res.Failed != 0 {
|
||||||
|
t.Fatalf("run = %+v, want 3 total / 3 copied", res)
|
||||||
|
}
|
||||||
|
for i, k := range keys {
|
||||||
|
body, err := os.ReadFile(filepath.Join(cfg.StoragePath, k))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", k, err)
|
||||||
|
}
|
||||||
|
if want := fmt.Sprintf("s3-bytes-%d", i); string(body) != want {
|
||||||
|
t.Errorf("%s = %q, want %q", k, body, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunMissingOnSource(t *testing.T) {
|
||||||
|
cfg, db, _, _ := syncEnv(t)
|
||||||
|
// Nothing on local disk: every object is reported missing, none fail.
|
||||||
|
res, err := Run(context.Background(), cfg, db, Options{Direction: ToS3})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.Missing != 3 || res.Copied != 0 || res.Failed != 0 {
|
||||||
|
t.Fatalf("run = %+v, want 3 missing", res)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunRequiresS3Config(t *testing.T) {
|
||||||
|
cfg, db, _, _ := syncEnv(t)
|
||||||
|
cfg.S3Endpoint, cfg.S3Bucket = "", ""
|
||||||
|
if _, err := Run(context.Background(), cfg, db, Options{Direction: ToS3}); err == nil {
|
||||||
|
t.Fatal("sync without S3 configured should fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,12 @@ type local struct {
|
|||||||
root string
|
root string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewLocal builds the local-disk backend explicitly, whichever backend is
|
||||||
|
// active — `photo-api sync` needs both sides at once.
|
||||||
|
func NewLocal(root string) (Storage, error) {
|
||||||
|
return newLocal(root)
|
||||||
|
}
|
||||||
|
|
||||||
func newLocal(root string) (*local, error) {
|
func newLocal(root string) (*local, error) {
|
||||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||||
return nil, fmt.Errorf("create storage dir %s: %w", root, err)
|
return nil, fmt.Errorf("create storage dir %s: %w", root, err)
|
||||||
@@ -74,6 +80,21 @@ func (l *local) Open(_ context.Context, key string) (io.ReadCloser, int64, error
|
|||||||
return f, info.Size(), nil
|
return f, info.Size(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (l *local) Stat(_ context.Context, key string) (int64, error) {
|
||||||
|
p, err := l.path(key)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
info, err := os.Stat(p)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return 0, ErrNotExist
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return info.Size(), nil
|
||||||
|
}
|
||||||
|
|
||||||
func (l *local) Delete(_ context.Context, key string) error {
|
func (l *local) Delete(_ context.Context, key string) error {
|
||||||
p, err := l.path(key)
|
p, err := l.path(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -2,14 +2,19 @@ package storage
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go-v2/aws"
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
|
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||||
|
"github.com/aws/smithy-go"
|
||||||
|
|
||||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||||
)
|
)
|
||||||
@@ -23,6 +28,12 @@ type s3Store struct {
|
|||||||
bucket string
|
bucket string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewS3 builds the S3 backend explicitly, whichever backend is active —
|
||||||
|
// `photo-api sync` needs both sides at once.
|
||||||
|
func NewS3(cfg config.Config) (Storage, error) {
|
||||||
|
return newS3(cfg)
|
||||||
|
}
|
||||||
|
|
||||||
func newS3(cfg config.Config) (*s3Store, error) {
|
func newS3(cfg config.Config) (*s3Store, error) {
|
||||||
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
|
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
|
||||||
awsconfig.WithRegion(cfg.S3Region),
|
awsconfig.WithRegion(cfg.S3Region),
|
||||||
@@ -65,6 +76,28 @@ func (s *s3Store) Open(ctx context.Context, key string) (io.ReadCloser, int64, e
|
|||||||
return out.Body, aws.ToInt64(out.ContentLength), nil
|
return out.Body, aws.ToInt64(out.ContentLength), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *s3Store) Stat(ctx context.Context, key string) (int64, error) {
|
||||||
|
out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(key),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
// Compatibles differ: some answer NotFound, some NoSuchKey, and a
|
||||||
|
// HEAD carries no body to parse, so fall back to the status code.
|
||||||
|
var notFound *types.NotFound
|
||||||
|
var noKey *types.NoSuchKey
|
||||||
|
var apiErr smithy.APIError
|
||||||
|
var respErr *awshttp.ResponseError
|
||||||
|
if errors.As(err, ¬Found) || errors.As(err, &noKey) ||
|
||||||
|
(errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey")) ||
|
||||||
|
(errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusNotFound) {
|
||||||
|
return 0, ErrNotExist
|
||||||
|
}
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return aws.ToInt64(out.ContentLength), nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *s3Store) Delete(ctx context.Context, key string) error {
|
func (s *s3Store) Delete(ctx context.Context, key string) error {
|
||||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ package storage
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -17,16 +18,30 @@ import (
|
|||||||
// callers then stream the object through the API instead.
|
// callers then stream the object through the API instead.
|
||||||
var ErrNoPresign = errors.New("presigned URLs not supported")
|
var ErrNoPresign = errors.New("presigned URLs not supported")
|
||||||
|
|
||||||
|
// ErrNotExist is what Stat reports for a missing object on either backend.
|
||||||
|
var ErrNotExist = errors.New("object does not exist")
|
||||||
|
|
||||||
type Storage interface {
|
type Storage interface {
|
||||||
Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error
|
Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error
|
||||||
Open(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
Open(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||||
|
// Stat returns the object's size without reading it, or ErrNotExist.
|
||||||
|
Stat(ctx context.Context, key string) (int64, error)
|
||||||
Delete(ctx context.Context, key string) error
|
Delete(ctx context.Context, key string) error
|
||||||
PresignGet(ctx context.Context, key, downloadFilename, contentType string, expiry time.Duration) (string, error)
|
PresignGet(ctx context.Context, key, downloadFilename, contentType string, expiry time.Duration) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// New builds the backend that serves requests (see config.S3Enabled).
|
||||||
func New(cfg config.Config) (Storage, error) {
|
func New(cfg config.Config) (Storage, error) {
|
||||||
if cfg.S3Enabled() {
|
if cfg.S3Enabled() {
|
||||||
return newS3(cfg)
|
return NewS3(cfg)
|
||||||
}
|
}
|
||||||
return newLocal(cfg.StoragePath)
|
return NewLocal(cfg.StoragePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name describes a backend for log lines.
|
||||||
|
func Name(cfg config.Config, s Storage) string {
|
||||||
|
if _, ok := s.(*s3Store); ok {
|
||||||
|
return fmt.Sprintf("S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("local disk at %s", cfg.StoragePath)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,6 +54,19 @@ func Open(cfg config.Config) (*DB, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsUniqueViolation reports whether err is a duplicate-key error. The two
|
||||||
|
// drivers wrap it in unrelated types (pgconn.PgError vs sqlite.Error), and
|
||||||
|
// neither is worth importing here just for one check, so this matches on the
|
||||||
|
// message: pgx renders "(SQLSTATE 23505)", modernc/sqlite "UNIQUE constraint
|
||||||
|
// failed: ...".
|
||||||
|
func IsUniqueViolation(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
return strings.Contains(msg, "SQLSTATE 23505") || strings.Contains(msg, "UNIQUE constraint failed")
|
||||||
|
}
|
||||||
|
|
||||||
// Rebind converts ?-style placeholders to $n for Postgres. Queries in this
|
// Rebind converts ?-style placeholders to $n for Postgres. Queries in this
|
||||||
// package never contain literal question marks in strings.
|
// package never contain literal question marks in strings.
|
||||||
func (db *DB) Rebind(query string) string {
|
func (db *DB) Rebind(query string) string {
|
||||||
|
|||||||
@@ -29,14 +29,17 @@ type Photo struct {
|
|||||||
LastError string
|
LastError string
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
UpdatedAt time.Time
|
UpdatedAt time.Time
|
||||||
|
// Checksum is the sha256 of the original bytes, empty when the row has
|
||||||
|
// not been hashed yet (uploaded before duplicate detection existed).
|
||||||
|
Checksum string
|
||||||
}
|
}
|
||||||
|
|
||||||
const photoColumns = `id, gallery_id, position, original_key, original_filename, content_type,
|
const photoColumns = `id, gallery_id, position, original_key, original_filename, content_type,
|
||||||
size_bytes, width, height, thumb_key, preview_key, taken_at, status, attempts, next_attempt_at,
|
size_bytes, width, height, thumb_key, preview_key, taken_at, status, attempts, next_attempt_at,
|
||||||
last_error, created_at, updated_at`
|
last_error, created_at, updated_at, checksum`
|
||||||
|
|
||||||
func scanPhoto(s scanner) (Photo, error) {
|
func scanPhoto(s scanner) (Photo, error) {
|
||||||
var v [18]any
|
var v [19]any
|
||||||
dest := make([]any, len(v))
|
dest := make([]any, len(v))
|
||||||
for i := range v {
|
for i := range v {
|
||||||
dest[i] = &v[i]
|
dest[i] = &v[i]
|
||||||
@@ -63,20 +66,82 @@ func scanPhoto(s scanner) (Photo, error) {
|
|||||||
LastError: asString(v[15]),
|
LastError: asString(v[15]),
|
||||||
CreatedAt: asTime(v[16]),
|
CreatedAt: asTime(v[16]),
|
||||||
UpdatedAt: asTime(v[17]),
|
UpdatedAt: asTime(v[17]),
|
||||||
|
Checksum: asString(v[18]),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InsertPhoto returns an error satisfying IsUniqueViolation when the gallery
|
||||||
|
// already holds a photo with the same checksum; callers treat that as a
|
||||||
|
// duplicate rather than a failure.
|
||||||
func (db *DB) InsertPhoto(ctx context.Context, p Photo) error {
|
func (db *DB) InsertPhoto(ctx context.Context, p Photo) error {
|
||||||
_, err := db.ExecContext(ctx, db.Rebind(`
|
_, err := db.ExecContext(ctx, db.Rebind(`
|
||||||
INSERT INTO photos_photos
|
INSERT INTO photos_photos
|
||||||
(id, gallery_id, position, original_key, original_filename, content_type, size_bytes,
|
(id, gallery_id, position, original_key, original_filename, content_type, size_bytes,
|
||||||
status, attempts, next_attempt_at, created_at, updated_at)
|
status, attempts, next_attempt_at, created_at, updated_at, checksum)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`),
|
VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?, ?)`),
|
||||||
p.ID, p.GalleryID, p.Position, p.OriginalKey, nullable(p.OriginalFilename), p.ContentType,
|
p.ID, p.GalleryID, p.Position, p.OriginalKey, nullable(p.OriginalFilename), p.ContentType,
|
||||||
p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt))
|
p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt),
|
||||||
|
nullable(p.Checksum))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FindPhotoByChecksum looks for an existing photo with the same content in one
|
||||||
|
// gallery — the duplicate check the upload path runs before storing anything.
|
||||||
|
// Duplicate scope is per gallery: the same image in another gallery keeps its
|
||||||
|
// own row and its own stored object.
|
||||||
|
func (db *DB) FindPhotoByChecksum(ctx context.Context, galleryID, checksum string) (Photo, error) {
|
||||||
|
if checksum == "" {
|
||||||
|
return Photo{}, ErrNotFound // never match the not-yet-hashed rows
|
||||||
|
}
|
||||||
|
row := db.QueryRowContext(ctx, db.Rebind(
|
||||||
|
"SELECT "+photoColumns+" FROM photos_photos WHERE gallery_id = ? AND checksum = ?"),
|
||||||
|
galleryID, checksum)
|
||||||
|
p, err := scanPhoto(row)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return Photo{}, ErrNotFound
|
||||||
|
}
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetPhotoChecksum fills in the hash of an already-stored photo. Used by the
|
||||||
|
// backfill; returns a unique-violation error when the row turns out to
|
||||||
|
// duplicate one already hashed in the same gallery.
|
||||||
|
func (db *DB) SetPhotoChecksum(ctx context.Context, id, checksum string) error {
|
||||||
|
res, err := db.ExecContext(ctx, db.Rebind(
|
||||||
|
"UPDATE photos_photos SET checksum = ? WHERE id = ?"), checksum, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return errIfNoRows(res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PhotosMissingChecksum lists rows still lacking a hash (optionally of one
|
||||||
|
// gallery only), oldest first so backfill keeps the earliest upload of a
|
||||||
|
// duplicate pair as the one that gets the checksum.
|
||||||
|
func (db *DB) PhotosMissingChecksum(ctx context.Context, galleryID string) ([]Photo, error) {
|
||||||
|
q := "SELECT " + photoColumns + " FROM photos_photos WHERE checksum IS NULL"
|
||||||
|
var args []any
|
||||||
|
if galleryID != "" {
|
||||||
|
q += " AND gallery_id = ?"
|
||||||
|
args = append(args, galleryID)
|
||||||
|
}
|
||||||
|
q += " ORDER BY gallery_id, position, created_at"
|
||||||
|
rows, err := db.QueryContext(ctx, db.Rebind(q), args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
photos := []Photo{}
|
||||||
|
for rows.Next() {
|
||||||
|
p, err := scanPhoto(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
photos = append(photos, p)
|
||||||
|
}
|
||||||
|
return photos, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
func (db *DB) GetPhoto(ctx context.Context, id string) (Photo, error) {
|
func (db *DB) GetPhoto(ctx context.Context, id string) (Photo, error) {
|
||||||
row := db.QueryRowContext(ctx,
|
row := db.QueryRowContext(ctx,
|
||||||
db.Rebind("SELECT "+photoColumns+" FROM photos_photos WHERE id = ?"), id)
|
db.Rebind("SELECT "+photoColumns+" FROM photos_photos WHERE id = ?"), id)
|
||||||
@@ -169,6 +234,64 @@ func (db *DB) PhotoKeys(ctx context.Context, galleryID string) ([]string, error)
|
|||||||
return keys, rows.Err()
|
return keys, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PhotoObject is one stored object: a photo variant plus the content type it
|
||||||
|
// should be written with. Used by the storage sync, which treats the rows as
|
||||||
|
// the inventory of what exists.
|
||||||
|
type PhotoObject struct {
|
||||||
|
PhotoID string
|
||||||
|
GalleryID string
|
||||||
|
Variant string // original | thumb | preview
|
||||||
|
Key string
|
||||||
|
ContentType string
|
||||||
|
}
|
||||||
|
|
||||||
|
// AllPhotoObjects lists every object of every photo (optionally of one
|
||||||
|
// gallery only), oldest first. Variant keys are empty until the worker has
|
||||||
|
// processed the photo; those are left out.
|
||||||
|
func (db *DB) AllPhotoObjects(ctx context.Context, galleryID string) ([]PhotoObject, error) {
|
||||||
|
q := `SELECT id, gallery_id, content_type, original_key, thumb_key, preview_key
|
||||||
|
FROM photos_photos`
|
||||||
|
var args []any
|
||||||
|
if galleryID != "" {
|
||||||
|
q += " WHERE gallery_id = ?"
|
||||||
|
args = append(args, galleryID)
|
||||||
|
}
|
||||||
|
q += " ORDER BY gallery_id, position, created_at"
|
||||||
|
rows, err := db.QueryContext(ctx, db.Rebind(q), args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
objects := []PhotoObject{}
|
||||||
|
for rows.Next() {
|
||||||
|
var id, galID, ctype, orig, thumb, preview any
|
||||||
|
if err := rows.Scan(&id, &galID, &ctype, &orig, &thumb, &preview); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
variants := []struct {
|
||||||
|
name, key, contentType string
|
||||||
|
}{
|
||||||
|
{"original", asString(orig), asString(ctype)},
|
||||||
|
{"thumb", asString(thumb), "image/jpeg"},
|
||||||
|
{"preview", asString(preview), "image/jpeg"},
|
||||||
|
}
|
||||||
|
for _, v := range variants {
|
||||||
|
if v.key == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
objects = append(objects, PhotoObject{
|
||||||
|
PhotoID: asString(id),
|
||||||
|
GalleryID: asString(galID),
|
||||||
|
Variant: v.name,
|
||||||
|
Key: v.key,
|
||||||
|
ContentType: v.contentType,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return objects, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
// ClaimNextPhoto picks the oldest due queued/failed photo and marks it
|
// ClaimNextPhoto picks the oldest due queued/failed photo and marks it
|
||||||
// processing. Optimistic claim (RowsAffected check) works identically on
|
// processing. Optimistic claim (RowsAffected check) works identically on
|
||||||
// Postgres and SQLite; returns ErrNotFound when the queue is empty.
|
// Postgres and SQLite; returns ErrNotFound when the queue is empty.
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
-- Content hash of the stored original, for per-gallery duplicate detection on
|
||||||
|
-- upload. NULL means "not hashed yet" — rows predating this migration, until
|
||||||
|
-- `photo-api backfill-checksums` runs. NULLs are distinct under a unique
|
||||||
|
-- index, so those rows never collide with each other.
|
||||||
|
-- (No semicolons in these comments: the migration runner splits on them.)
|
||||||
|
ALTER TABLE photos_photos ADD COLUMN IF NOT EXISTS checksum varchar(64);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS photos_photos_gallery_checksum_idx ON photos_photos (gallery_id, checksum);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
-- Content hash of the stored original, for per-gallery duplicate detection on
|
||||||
|
-- upload. NULL means "not hashed yet" — rows predating this migration, until
|
||||||
|
-- `photo-api backfill-checksums` runs. NULLs are distinct under a unique
|
||||||
|
-- index, so those rows never collide with each other.
|
||||||
|
-- (No semicolons in these comments: the migration runner splits on them.)
|
||||||
|
ALTER TABLE photos_photos ADD COLUMN checksum text;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS photos_photos_gallery_checksum_idx ON photos_photos (gallery_id, checksum);
|
||||||
Reference in New Issue
Block a user