Add photo content-hash dedup and local↔S3 library sync.
Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
This commit is contained in:
Michilis
@@ -0,0 +1,175 @@
|
||||
package checksum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
const galleryID = "11111111-1111-1111-1111-111111111111"
|
||||
|
||||
// env builds a migrated scratch SQLite database with one gallery, plus a local
|
||||
// storage backend rooted next to it.
|
||||
func env(t *testing.T) (*store.DB, storage.Storage) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
db, err := store.Open(config.Config{DBType: "sqlite", DatabaseURL: filepath.Join(dir, "test.db")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
ctx := context.Background()
|
||||
if err := db.Migrate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
if err := db.CreateGallery(ctx, store.Gallery{
|
||||
ID: galleryID, Slug: "g", Title: "G", Visibility: "private", ShareToken: "tok",
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := storage.NewLocal(filepath.Join(dir, "photos"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, st
|
||||
}
|
||||
|
||||
// seed inserts an unhashed photo whose original holds body, mimicking a row
|
||||
// uploaded before the checksum column existed.
|
||||
func seed(t *testing.T, db *store.DB, st storage.Storage, id, body string) store.Photo {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
key := "galleries/" + galleryID + "/orig/" + id + ".jpg"
|
||||
if body != "" {
|
||||
if err := st.Put(ctx, key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
p := store.Photo{
|
||||
ID: id, GalleryID: galleryID, OriginalKey: key, OriginalFilename: id + ".jpg",
|
||||
ContentType: "image/jpeg", SizeBytes: int64(len(body)), Status: "ready",
|
||||
NextAttemptAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}
|
||||
if err := db.InsertPhoto(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func checksumOf(t *testing.T, db *store.DB, id string) string {
|
||||
t.Helper()
|
||||
p, err := db.GetPhoto(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p.Checksum
|
||||
}
|
||||
|
||||
func TestBackfill(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("hashes unhashed photos and is idempotent", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
seed(t, db, st, "p2", "beta")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 2 || res.Hashed != 2 || res.Duplicates != 0 || res.Failed != 0 {
|
||||
t.Fatalf("first run: %+v", res)
|
||||
}
|
||||
want, err := Sum(strings.NewReader("alpha"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := checksumOf(t, db, "p1"); got != want {
|
||||
t.Fatalf("p1 checksum = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
res, err = Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 0 || res.Hashed != 0 {
|
||||
t.Fatalf("rerun should find nothing to do: %+v", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reports pre-existing duplicates and leaves them unhashed", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "same")
|
||||
seed(t, db, st, "p2", "same")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{Concurrency: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Hashed != 1 || res.Duplicates != 1 || res.Failed != 0 {
|
||||
t.Fatalf("got %+v, want 1 hashed and 1 duplicate", res)
|
||||
}
|
||||
// The earlier row keeps the checksum, the later one is left alone —
|
||||
// nothing is deleted either way.
|
||||
if checksumOf(t, db, "p1") == "" {
|
||||
t.Fatal("p1 should have been hashed")
|
||||
}
|
||||
if got := checksumOf(t, db, "p2"); got != "" {
|
||||
t.Fatalf("p2 checksum = %q, want empty", got)
|
||||
}
|
||||
if _, err := db.GetPhoto(ctx, "p2"); err != nil {
|
||||
t.Fatalf("duplicate row must survive: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips photos whose object is gone", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "") // row without a stored original
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Missing != 1 || res.Hashed != 0 || res.Failed != 0 {
|
||||
t.Fatalf("got %+v, want 1 missing", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dry run writes nothing", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Hashed != 1 {
|
||||
t.Fatalf("got %+v, want 1 hashed", res)
|
||||
}
|
||||
if got := checksumOf(t, db, "p1"); got != "" {
|
||||
t.Fatalf("dry run recorded %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("limits to one gallery", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{GalleryID: "22222222-2222-2222-2222-222222222222"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 0 {
|
||||
t.Fatalf("other gallery should have nothing to do: %+v", res)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user