Add photo content-hash dedup and local↔S3 library sync.

Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
2026-08-05 05:41:12 +00:00
parent dafa3711f8
commit 498d7d8a7d
31 changed files with 2216 additions and 64 deletions
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"

 	"image"
 	"image/color"
@@ -38,10 +39,11 @@ const (
 )

 type testEnv struct {
-	handler http.Handler
-	db      *store.DB
-	worker  *worker.Worker
-	pg      bool
+	handler     http.Handler
+	db          *store.DB
+	worker      *worker.Worker
+	storagePath string
+	pg          bool
 }

 // setup migrates a scratch DB (SQLite by default; Postgres when
@@ -123,7 +125,13 @@ func setup(t *testing.T) *testEnv {
 	}
 	wrk := worker.New(db, st, nil, cfg.StoragePath, 1)
 	srv := New(cfg, db, st, auth.NewVerifier(db), wrk)
-	return &testEnv{handler: srv.Handler(), db: db, worker: wrk, pg: cfg.DBType == "postgres"}
+	return &testEnv{
+		handler:     srv.Handler(),
+		db:          db,
+		worker:      wrk,
+		storagePath: cfg.StoragePath,
+		pg:          cfg.DBType == "postgres",
+	}
 }

 // makeToken inserts a Better Auth session row for the user and returns a
@@ -175,12 +183,16 @@ func decode[T any](t *testing.T, w *httptest.ResponseRecorder) T {
 	return v
 }

-func testJPEG(t *testing.T) []byte {
+func testJPEG(t *testing.T) []byte { return testJPEGTinted(t, 128) }
+
+// testJPEGTinted varies the blue channel so tests that need two *different*
+// images (duplicate detection) can get them without a fixture file.
+func testJPEGTinted(t *testing.T, blue uint8) []byte {
 	t.Helper()
 	img := image.NewRGBA(image.Rect(0, 0, 800, 600))
 	for x := 0; x < 800; x += 10 {
 		for y := 0; y < 600; y++ {
-			img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: 128, A: 255})
+			img.Set(x, y, color.RGBA{R: uint8(x % 255), G: uint8(y % 255), B: blue, A: 255})
 		}
 	}
 	var buf bytes.Buffer
@@ -296,6 +308,130 @@ func uploadPhoto(t *testing.T, e *testEnv, admin, galleryID string, file []byte)
 	return photos[0]
 }

+// uploadFiles posts several parts in one request, the way the API allows even
+// though the admin UI sends one file per request.
+func uploadFiles(t *testing.T, e *testEnv, admin, galleryID string, files ...[]byte) []photoJSON {
+	t.Helper()
+	var buf bytes.Buffer
+	mw := multipart.NewWriter(&buf)
+	for i, f := range files {
+		fw, _ := mw.CreateFormFile("files", fmt.Sprintf("photo-%d.jpg", i))
+		fw.Write(f)
+	}
+	mw.Close()
+	req := httptest.NewRequest("POST", "/api/photos/galleries/"+galleryID+"/photos", &buf)
+	req.Header.Set("Content-Type", mw.FormDataContentType())
+	req.Header.Set("Authorization", "Bearer "+admin)
+	w := httptest.NewRecorder()
+	e.handler.ServeHTTP(w, req)
+	if w.Code != 201 {
+		t.Fatalf("upload: %d %s", w.Code, w.Body.String())
+	}
+	return decode[struct {
+		Photos []photoJSON `json:"photos"`
+	}](t, w).Photos
+}
+
+// countOriginals is how many objects actually landed on disk for a gallery —
+// a duplicate must not add one.
+func countOriginals(t *testing.T, e *testEnv, galleryID string) int {
+	t.Helper()
+	entries, err := os.ReadDir(filepath.Join(e.storagePath, "galleries", galleryID, "orig"))
+	if os.IsNotExist(err) {
+		return 0
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	return len(entries)
+}
+
+func TestUploadSkipsDuplicates(t *testing.T) {
+	e := setup(t)
+	admin := e.makeToken(t, uAdmin)
+	g := createGallery(t, e, admin, map[string]any{"title": "Dupes"})
+	img := testJPEG(t)
+
+	first := uploadPhoto(t, e, admin, g.ID, img)
+	if first.Duplicate {
+		t.Fatalf("first upload flagged as duplicate: %+v", first)
+	}
+
+	// Same bytes again: echoed back as the existing photo, nothing stored.
+	second := uploadPhoto(t, e, admin, g.ID, img)
+	if !second.Duplicate {
+		t.Fatalf("second upload not flagged as duplicate: %+v", second)
+	}
+	if second.ID != first.ID {
+		t.Fatalf("duplicate should echo the existing photo: got %s, want %s", second.ID, first.ID)
+	}
+	photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(photos) != 1 {
+		t.Fatalf("want 1 row after re-upload, got %d", len(photos))
+	}
+	if n := countOriginals(t, e, g.ID); n != 1 {
+		t.Fatalf("want 1 stored original after re-upload, got %d", n)
+	}
+	if photos[0].Checksum == "" {
+		t.Fatal("checksum was not recorded")
+	}
+
+	// A different image is unaffected and takes the next position.
+	other := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))
+	if other.Duplicate || other.ID == first.ID {
+		t.Fatalf("distinct image treated as duplicate: %+v", other)
+	}
+	if other.Position != 1 {
+		t.Fatalf("want position 1 for the second distinct photo, got %d", other.Position)
+	}
+
+	// Duplicate scope is one gallery: the same bytes elsewhere upload normally.
+	g2 := createGallery(t, e, admin, map[string]any{"title": "Other gallery"})
+	elsewhere := uploadPhoto(t, e, admin, g2.ID, img)
+	if elsewhere.Duplicate {
+		t.Fatalf("same image in another gallery must not be a duplicate: %+v", elsewhere)
+	}
+	if elsewhere.ID == first.ID {
+		t.Fatal("second gallery should get its own photo row")
+	}
+}
+
+func TestUploadBatchContinuesPastDuplicate(t *testing.T) {
+	e := setup(t)
+	admin := e.makeToken(t, uAdmin)
+	g := createGallery(t, e, admin, map[string]any{"title": "Batch"})
+	a, b, c := testJPEG(t), testJPEGTinted(t, 32), testJPEGTinted(t, 200)
+
+	if got := uploadFiles(t, e, admin, g.ID, a); len(got) != 1 {
+		t.Fatalf("seed upload: %d photos", len(got))
+	}
+	// b duplicates nothing, a is already there, c is new: the batch must not
+	// abort, and the duplicate must not consume a position.
+	batch := uploadFiles(t, e, admin, g.ID, b, a, c)
+	if len(batch) != 3 {
+		t.Fatalf("want 3 results, got %d", len(batch))
+	}
+	if batch[0].Duplicate || !batch[1].Duplicate || batch[2].Duplicate {
+		t.Fatalf("duplicate flags: %v %v %v", batch[0].Duplicate, batch[1].Duplicate, batch[2].Duplicate)
+	}
+	if batch[0].Position != 1 || batch[2].Position != 2 {
+		t.Fatalf("positions should stay dense: %d, %d", batch[0].Position, batch[2].Position)
+	}
+	photos, err := e.db.ListPhotos(context.Background(), g.ID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(photos) != 3 {
+		t.Fatalf("want 3 rows, got %d", len(photos))
+	}
+	if n := countOriginals(t, e, g.ID); n != 3 {
+		t.Fatalf("want 3 stored originals, got %d", n)
+	}
+}
+
 // processQueue runs the worker until the photo is ready or failed.
 func processQueue(t *testing.T, e *testEnv, photoID string) store.Photo {
 	t.Helper()
@@ -465,9 +601,9 @@ func TestReorderAndVisibilityUpdate(t *testing.T) {
 	e := setup(t)
 	admin := e.makeToken(t, uAdmin)
 	g := createGallery(t, e, admin, map[string]any{"title": "Orden", "visibility": "public"})
-	jpg := testJPEG(t)
-	p1 := uploadPhoto(t, e, admin, g.ID, jpg)
-	p2 := uploadPhoto(t, e, admin, g.ID, jpg)
+	// Two distinct images: the same bytes twice would be deduplicated.
+	p1 := uploadPhoto(t, e, admin, g.ID, testJPEG(t))
+	p2 := uploadPhoto(t, e, admin, g.ID, testJPEGTinted(t, 32))

 	if w := e.request(t, "PATCH", "/api/photos/galleries/"+g.ID+"/order", admin,
 		map[string]any{"photoIds": []string{p2.ID, p1.ID}}); w.Code != 200 {