Add photo content-hash dedup and local↔S3 library sync.

Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.

photo-api/internal/httpapi/api_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 
 	"image"
 	"image/color"
@@ -38,10 +39,11 @@ const (
 )
 
 type testEnv struct {
-	handler http.Handler
-	db      *store.DB
-	worker  *worker.Worker
-	pg      bool
+	handler     http.Handler
+	db          *store.DB
+	worker      *worker.Worker
+	storagePath string
+	pg          bool
 }
 
 // setup migrates a scratch DB (SQLite by default; Postgres when
@@ -123,7 +125,13 @@ func setup(t *testing.T) *testEnv {
 	}
 	wrk := worker.New(db, st, nil, cfg.StoragePath, 1)
 	srv := New(cfg, db, st, auth.NewVerifier(db), wrk)
-	return &testEnv{handler: srv.Handler(), db: db, worker: wrk, pg: cfg.DBType == "postgres"}
+	return &testEnv{
+		handler:     srv.Handler(),
+		db:          db,
+		worker:      wrk,
+		storagePath: cfg.StoragePath,
+		pg:          cfg.DBType == "postgres",
+	}
 }
 
 // makeToken inserts a Better Auth session row for the user and returns a
@@ -175,12 +183,16 @@ func decode[T any](t *testing.T, w *httptest.ResponseRecorder) T {
 	return v
 }
 
-func testJPEG(t *testing.T) []byte {
+func testJPEG(t *testing.T) []byte { return testJPEGTinted(t, 128) }
+
+// testJPEGTinted varies the blue channel so tests that need two *different*
+// images (duplicate detection) can get them without a fixture file.
+func testJPEGTinted(t *testing.T, blue uint8) []byte {
 	t.Helper()
 	img := image.NewRGBA(image.Rect(0, 0, 800, 600))
 	for x := 0; x < 800; x += 10 {
 		for y := 0; y < 600; y++ {
-			img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: 128, A: 255})
+			img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: blue, A: 255})
 		}
 	}
 	var buf bytes.Buffer
@@ -296,6 +308,130 @@ func uploadPhoto(t *testing.T, e *testEnv, admin, galleryID string, file []byte)
 	return photos[0]
 }
 
+// uploadFiles posts several parts in one request, the way the API allows even
+// though the admin UI sends one file per request.
+func uploadFiles(t *testing.T, e *testEnv, admin, galleryID string, files ...[]byte) []photoJSON {
+	t.Helper()
+	var buf bytes.Buffer
+	mw := multipart.NewWriter(&buf)
+	for i, f := range files {
+		fw, _ := mw.CreateFormFile("files", fmt.Sprintf("photo-%d.jpg", i))
+		fw.Write(f)
+	}
+	mw.Close()
+	req := httptest.NewRequest("POST", "/api/photos/galleries/"+galleryID+"/photos", &buf)
+	req.Header.Set("Content-Type", mw.FormDataContentType())
+	req.Header.Set("Authorization", "Bearer "+admin)
+	w := httptest.NewRecorder()
+	e.handler.ServeHTTP(w, req)
+	if w.Code != 201 {
+		t.Fatalf("upload: %d %s", w.Code, w.Body.String())
+	}
+	return decode[struct {
+		Photos []photoJSON `json:"photos"`
+	}](t, w).Photos
+}
+
+// countOriginals is how many objects actually landed on disk for a gallery —
+// a duplicate must not add one.
+func countOriginals(t *testing.T, e *testEnv, galleryID string) int {
+	t.Helper()
+	entries, err := os.ReadDir(filepath.Join(e.storagePath, "galleries", galleryID, "orig"))
+	if os.IsNotExist(err) {
+		return 0
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	return len(entries)
+}
+
+func TestUploadSkipsDuplicates(t *testing.T) {
+	e := setup(t)
+	admin := e.makeToken(t, uAdmin)
+	g := createGallery(t, e, admin, map[string]any{"title": "Dupes"})
+	img := testJPEG(t)
+
+	first := uploadPhoto(t, e, admin, g.ID, img)
+	if first.Duplicate {
+		t.Fatalf("first upload flagged as duplicate: %+v", first)
+	}
+
+	// Same bytes again: echoed back as the existing photo, nothing stored.
+	second := uploadPhoto(t, e, admin, g.ID, img)
+	if !second.Duplicate {
+		t.Fatalf("second upload not flagged as duplicate: %+v", second)
+	}
+	if second.ID != first.ID {
+		t.Fatalf("duplicate should echo the existing photo: got %s, want %s", second.ID, first.ID)
+	}
+	photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(photos) != 1 {
+		t.Fatalf("want 1 row after re-upload, got %d", len(photos))
+	}
+	if n := countOriginals(t, e, g.ID); n != 1 {
+		t.Fatalf("want 1 stored original after re-upload, got %d", n)
+	}
+	if photos[0].Checksum == "" {
+		t.Fatal("checksum was not recorded")
+	}
+
+	// A different image is unaffected and takes the next position.
+	other := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))
+	if other.Duplicate || other.ID == first.ID {
+		t.Fatalf("distinct image treated as duplicate: %+v", other)
+	}
+	if other.Position != 1 {
+		t.Fatalf("want position 1 for the second distinct photo, got %d", other.Position)
+	}
+
+	// Duplicate scope is one gallery: the same bytes elsewhere upload normally.
+	g2 := createGallery(t, e, admin, map[string]any{"title": "Other gallery"})
+	elsewhere := uploadPhoto(t, e, admin, g2.ID, img)
+	if elsewhere.Duplicate {
+		t.Fatalf("same image in another gallery must not be a duplicate: %+v", elsewhere)
+	}
+	if elsewhere.ID == first.ID {
+		t.Fatal("second gallery should get its own photo row")
+	}
+}
+
+func TestUploadBatchContinuesPastDuplicate(t *testing.T) {
+	e := setup(t)
+	admin := e.makeToken(t, uAdmin)
+	g := createGallery(t, e, admin, map[string]any{"title": "Batch"})
+	a, b, c := testJPEG(t), testJPEGTinted(t, 32), testJPEGTinted(t, 200)
+
+	if got := uploadFiles(t, e, admin, g.ID, a); len(got) != 1 {
+		t.Fatalf("seed upload: %d photos", len(got))
+	}
+	// b duplicates nothing, a is already there, c is new: the batch must not
+	// abort, and the duplicate must not consume a position.
+	batch := uploadFiles(t, e, admin, g.ID, b, a, c)
+	if len(batch) != 3 {
+		t.Fatalf("want 3 results, got %d", len(batch))
+	}
+	if batch[0].Duplicate || !batch[1].Duplicate || batch[2].Duplicate {
+		t.Fatalf("duplicate flags: %v %v %v", batch[0].Duplicate, batch[1].Duplicate, batch[2].Duplicate)
+	}
+	if batch[0].Position != 1 || batch[2].Position != 2 {
+		t.Fatalf("positions should stay dense: %d, %d", batch[0].Position, batch[2].Position)
+	}
+	photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(photos) != 3 {
+		t.Fatalf("want 3 rows, got %d", len(photos))
+	}
+	if n := countOriginals(t, e, g.ID); n != 3 {
+		t.Fatalf("want 3 stored originals, got %d", n)
+	}
+}
+
 // processQueue runs the worker until the photo is ready or failed.
 func processQueue(t *testing.T, e *testEnv, photoID string) store.Photo {
 	t.Helper()
@@ -465,9 +601,9 @@ func TestReorderAndVisibilityUpdate(t *testing.T) {
 	e := setup(t)
 	admin := e.makeToken(t, uAdmin)
 	g := createGallery(t, e, admin, map[string]any{"title": "Orden", "visibility": "public"})
-	jpg := testJPEG(t)
-	p1 := uploadPhoto(t, e, admin, g.ID, jpg)
-	p2 := uploadPhoto(t, e, admin, g.ID, jpg)
+	// Two distinct images: the same bytes twice would be deduplicated.
+	p1 := uploadPhoto(t, e, admin, g.ID, testJPEG(t))
+	p2 := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))
 
 	if w := e.request(t, "PATCH", "/api/photos/galleries/"+g.ID+"/order", admin,
 		map[string]any{"photoIds": []string{p2.ID, p1.ID}}); w.Code != 200 {

photo-api/internal/httpapi/dto.go

@@ -58,6 +58,10 @@ type photoJSON struct {
 	LastError        string    `json:"lastError,omitempty"` // admin only
 	CreatedAt        string    `json:"createdAt"`
 	URLs             photoURLs `json:"urls"`
+	// Duplicate marks an upload that was skipped because the gallery already
+	// held these bytes; the rest of the object describes the existing photo.
+	// Set by the upload handler only, never persisted.
+	Duplicate bool `json:"duplicate,omitempty"`
 }
 
 // viewTokenFor returns the token to embed in a gallery's file URLs: none

photo-api/internal/httpapi/photos.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
+	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
 	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
 	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
 )
@@ -20,6 +21,10 @@ import (
 // Each file is sniffed by magic bytes (client filename/Content-Type are
 // untrusted, same policy as /api/media/upload), stored as the untouched
 // original, and queued for variant processing.
+//
+// A file whose bytes are already in this gallery is not stored a second time:
+// the incoming copy is discarded, the existing photo is echoed back with
+// "duplicate": true, and the rest of the batch carries on.
 func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.User) {
 	galleryID := r.PathValue("id")
 	g, err := s.db.GetGallery(r.Context(), galleryID)
@@ -57,7 +62,7 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
 			part.Close()
 			continue
 		}
-		photo, uploadErr := s.saveUpload(r, g, part, position, maxFile)
+		photo, duplicate, uploadErr := s.saveUpload(r, g, part, position, maxFile)
 		part.Close()
 		if uploadErr != nil {
 			// One bad file fails the request explicitly rather than silently
@@ -65,7 +70,12 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
 			writeError(w, uploadErr.status, uploadErr.msg)
 			return
 		}
-		created = append(created, s.photoToJSON(photo, s.viewTokenFor(g), true))
+		out := s.photoToJSON(photo, s.viewTokenFor(g), true)
+		out.Duplicate = duplicate
+		created = append(created, out)
+		if duplicate {
+			continue // nothing was stored, so the next file keeps this position
+		}
 		position++
 	}
 
@@ -82,51 +92,67 @@ type uploadError struct {
 	msg    string
 }
 
-func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, *uploadError) {
+// saveUpload stores one part. The bool it returns reports a duplicate: the
+// gallery already holds these bytes, so nothing was written and the photo
+// returned is the existing one.
+func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, bool, *uploadError) {
 	head := make([]byte, 16)
 	n, err := io.ReadFull(part, head)
 	if err != nil && err != io.ErrUnexpectedEOF {
-		return store.Photo{}, &uploadError{http.StatusBadRequest, "Could not read file"}
+		return store.Photo{}, false, &uploadError{http.StatusBadRequest, "Could not read file"}
 	}
 	head = head[:n]
 	contentType, ext, ok, reason := imaging.Sniff(head)
 	if !ok {
-		return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType, reason}
+		return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType, reason}
 	}
 	if contentType == "image/heic" && imaging.DetectHeicConverter(s.cfg.HeicConverter) == nil {
-		return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType,
+		return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType,
 			"HEIC uploads need an image converter on the server (install libvips-tools); please upload JPEG instead"}
 	}
 
 	// Spool to a temp file to learn the size before handing to storage
-	// (S3 wants a length; local rename wants a file anyway).
+	// (S3 wants a length; local rename wants a file anyway). The same pass
+	// hashes the bytes for the duplicate check below.
 	tmp, err := os.CreateTemp(s.cfg.StoragePath, ".incoming-*")
 	if err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 	defer os.Remove(tmp.Name())
 	defer tmp.Close()
 
-	size, err := io.Copy(tmp, io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
+	hasher := checksum.New()
+	size, err := io.Copy(io.MultiWriter(tmp, hasher),
+		io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
 	if err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 	if size > maxFile {
-		return store.Photo{}, &uploadError{http.StatusRequestEntityTooLarge,
+		return store.Photo{}, false, &uploadError{http.StatusRequestEntityTooLarge,
 			fmt.Sprintf("File exceeds the %d MB limit", s.cfg.MaxUploadMB)}
 	}
 	if _, err := tmp.Seek(0, io.SeekStart); err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+	}
+	sum := checksum.Format(hasher)
+
+	// Duplicate check before anything is written: the temp copy is dropped by
+	// the deferred Remove, no object is stored and no row is inserted.
+	if existing, err := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); err == nil {
+		return existing, true, nil
+	} else if err != store.ErrNotFound {
+		log.Printf("Error: %v", err)
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 
 	photoID := newID()
 	key := fmt.Sprintf("galleries/%s/orig/%s%s", g.ID, photoID, ext)
 	if err := s.storage.Put(r.Context(), key, tmp, size, contentType); err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 
 	now := time.Now()
@@ -142,13 +168,22 @@ func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Pa
 		NextAttemptAt:    now,
 		CreatedAt:        now,
 		UpdatedAt:        now,
+		Checksum:         sum,
 	}
 	if err := s.db.InsertPhoto(r.Context(), photo); err != nil {
 		s.storage.Delete(r.Context(), key)
+		// A concurrent upload of the same bytes won the race between the
+		// check above and this insert; the unique index caught it, so report
+		// the winner as the duplicate instead of failing.
+		if store.IsUniqueViolation(err) {
+			if existing, findErr := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); findErr == nil {
+				return existing, true, nil
+			}
+		}
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
-	return photo, nil
+	return photo, false, nil
 }
 
 type reorderBody struct {