Add photo content-hash dedup and local↔S3 library sync.
Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
This commit is contained in:
Michilis
@@ -0,0 +1,145 @@
|
||||
package checksum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// Options configures a backfill run.
|
||||
type Options struct {
|
||||
// GalleryID limits the run to one gallery; empty means the whole library.
|
||||
GalleryID string
|
||||
// Concurrency is how many originals are hashed at a time.
|
||||
Concurrency int
|
||||
// DryRun reports what would be hashed without writing to the database.
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// Result counts what a run did. Duplicates are rows whose bytes match an
|
||||
// earlier photo in the same gallery; they keep a NULL checksum and are listed
|
||||
// so an admin can decide what to do with them.
|
||||
type Result struct {
|
||||
Total int // rows without a checksum
|
||||
Hashed int // hashed and recorded (or, with DryRun, would be)
|
||||
Duplicates int
|
||||
Missing int // original object absent from storage
|
||||
Failed int
|
||||
}
|
||||
|
||||
// Backfill hashes the stored original of every photo that has no checksum yet,
|
||||
// so duplicate detection also catches re-uploads of photos from before the
|
||||
// checksum column existed. It only ever writes the checksum column; no photo,
|
||||
// row or object is deleted.
|
||||
//
|
||||
// It is idempotent: rerunning it finds only what the previous run left behind.
|
||||
func Backfill(ctx context.Context, db *store.DB, st storage.Storage, opts Options) (Result, error) {
|
||||
if opts.Concurrency < 1 {
|
||||
opts.Concurrency = 4
|
||||
}
|
||||
photos, err := db.PhotosMissingChecksum(ctx, opts.GalleryID)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("list photos without a checksum: %w", err)
|
||||
}
|
||||
|
||||
prefix := ""
|
||||
if opts.DryRun {
|
||||
prefix = "[dry-run] "
|
||||
}
|
||||
log.Printf("backfill-checksums: %s%d photos to hash, concurrency %d", prefix, len(photos), opts.Concurrency)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
res = Result{Total: len(photos)}
|
||||
done int64
|
||||
jobs = make(chan store.Photo)
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
// Sequential hashing per worker, but the database update is what can
|
||||
// collide: two photos of the same gallery with identical bytes race, and
|
||||
// the unique index decides which one keeps the checksum.
|
||||
for i := 0; i < opts.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for p := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
sum, err := hashOriginal(ctx, st, p)
|
||||
if err == nil && !opts.DryRun {
|
||||
err = db.SetPhotoChecksum(ctx, p.ID, sum)
|
||||
}
|
||||
n := atomic.AddInt64(&done, 1)
|
||||
|
||||
mu.Lock()
|
||||
switch {
|
||||
case errors.Is(err, storage.ErrNotExist):
|
||||
res.Missing++
|
||||
log.Printf("backfill-checksums: [%d/%d] MISSING object, skipped: %s (photo %s)",
|
||||
n, len(photos), p.OriginalKey, p.ID)
|
||||
case store.IsUniqueViolation(err):
|
||||
res.Duplicates++
|
||||
other := "an earlier photo"
|
||||
if existing, findErr := db.FindPhotoByChecksum(ctx, p.GalleryID, sum); findErr == nil {
|
||||
other = "photo " + existing.ID
|
||||
}
|
||||
log.Printf("backfill-checksums: [%d/%d] DUPLICATE: photo %s (%s) has the same content as %s "+
|
||||
"in gallery %s — left without a checksum, delete it by hand if unwanted",
|
||||
n, len(photos), p.ID, p.OriginalFilename, other, p.GalleryID)
|
||||
case err != nil:
|
||||
res.Failed++
|
||||
log.Printf("backfill-checksums: [%d/%d] FAILED %s (photo %s): %v",
|
||||
n, len(photos), p.OriginalKey, p.ID, err)
|
||||
default:
|
||||
res.Hashed++
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, p := range photos {
|
||||
select {
|
||||
case jobs <- p:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
|
||||
log.Printf("backfill-checksums: %sdone — %d hashed, %d duplicates left unhashed, %d objects missing, %d failed",
|
||||
prefix, res.Hashed, res.Duplicates, res.Missing, res.Failed)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, fmt.Errorf("interrupted after %d/%d photos: %w", res.Hashed, res.Total, err)
|
||||
}
|
||||
if res.Failed > 0 {
|
||||
return res, fmt.Errorf("%d of %d photos failed to hash (rerun to retry; already-hashed photos are skipped)",
|
||||
res.Failed, res.Total)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// hashOriginal reads a photo's stored original. Stat runs first because it is
|
||||
// the only call that reports a missing object as storage.ErrNotExist on both
|
||||
// backends — Open surfaces the driver's own error.
|
||||
func hashOriginal(ctx context.Context, st storage.Storage, p store.Photo) (string, error) {
|
||||
if _, err := st.Stat(ctx, p.OriginalKey); err != nil {
|
||||
return "", err
|
||||
}
|
||||
r, _, err := st.Open(ctx, p.OriginalKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer r.Close()
|
||||
return Sum(r)
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package checksum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
const galleryID = "11111111-1111-1111-1111-111111111111"
|
||||
|
||||
// env builds a migrated scratch SQLite database with one gallery, plus a local
|
||||
// storage backend rooted next to it.
|
||||
func env(t *testing.T) (*store.DB, storage.Storage) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
db, err := store.Open(config.Config{DBType: "sqlite", DatabaseURL: filepath.Join(dir, "test.db")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
ctx := context.Background()
|
||||
if err := db.Migrate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
if err := db.CreateGallery(ctx, store.Gallery{
|
||||
ID: galleryID, Slug: "g", Title: "G", Visibility: "private", ShareToken: "tok",
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := storage.NewLocal(filepath.Join(dir, "photos"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, st
|
||||
}
|
||||
|
||||
// seed inserts an unhashed photo whose original holds body, mimicking a row
|
||||
// uploaded before the checksum column existed.
|
||||
func seed(t *testing.T, db *store.DB, st storage.Storage, id, body string) store.Photo {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
key := "galleries/" + galleryID + "/orig/" + id + ".jpg"
|
||||
if body != "" {
|
||||
if err := st.Put(ctx, key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
p := store.Photo{
|
||||
ID: id, GalleryID: galleryID, OriginalKey: key, OriginalFilename: id + ".jpg",
|
||||
ContentType: "image/jpeg", SizeBytes: int64(len(body)), Status: "ready",
|
||||
NextAttemptAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}
|
||||
if err := db.InsertPhoto(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func checksumOf(t *testing.T, db *store.DB, id string) string {
|
||||
t.Helper()
|
||||
p, err := db.GetPhoto(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p.Checksum
|
||||
}
|
||||
|
||||
func TestBackfill(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("hashes unhashed photos and is idempotent", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
seed(t, db, st, "p2", "beta")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 2 || res.Hashed != 2 || res.Duplicates != 0 || res.Failed != 0 {
|
||||
t.Fatalf("first run: %+v", res)
|
||||
}
|
||||
want, err := Sum(strings.NewReader("alpha"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := checksumOf(t, db, "p1"); got != want {
|
||||
t.Fatalf("p1 checksum = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
res, err = Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 0 || res.Hashed != 0 {
|
||||
t.Fatalf("rerun should find nothing to do: %+v", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reports pre-existing duplicates and leaves them unhashed", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "same")
|
||||
seed(t, db, st, "p2", "same")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{Concurrency: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Hashed != 1 || res.Duplicates != 1 || res.Failed != 0 {
|
||||
t.Fatalf("got %+v, want 1 hashed and 1 duplicate", res)
|
||||
}
|
||||
// The earlier row keeps the checksum, the later one is left alone —
|
||||
// nothing is deleted either way.
|
||||
if checksumOf(t, db, "p1") == "" {
|
||||
t.Fatal("p1 should have been hashed")
|
||||
}
|
||||
if got := checksumOf(t, db, "p2"); got != "" {
|
||||
t.Fatalf("p2 checksum = %q, want empty", got)
|
||||
}
|
||||
if _, err := db.GetPhoto(ctx, "p2"); err != nil {
|
||||
t.Fatalf("duplicate row must survive: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips photos whose object is gone", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "") // row without a stored original
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Missing != 1 || res.Hashed != 0 || res.Failed != 0 {
|
||||
t.Fatalf("got %+v, want 1 missing", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dry run writes nothing", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Hashed != 1 {
|
||||
t.Fatalf("got %+v, want 1 hashed", res)
|
||||
}
|
||||
if got := checksumOf(t, db, "p1"); got != "" {
|
||||
t.Fatalf("dry run recorded %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("limits to one gallery", func(t *testing.T) {
|
||||
db, st := env(t)
|
||||
seed(t, db, st, "p1", "alpha")
|
||||
|
||||
res, err := Backfill(ctx, db, st, Options{GalleryID: "22222222-2222-2222-2222-222222222222"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 0 {
|
||||
t.Fatalf("other gallery should have nothing to do: %+v", res)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Package checksum defines the content hash used to spot duplicate photos and
|
||||
// the backfill that fills it in for photos uploaded before it existed.
|
||||
//
|
||||
// The hash is sha256 over the untouched original bytes, hex encoded, stored in
|
||||
// photos_photos.checksum. Duplicate scope is one gallery — the unique index is
|
||||
// on (gallery_id, checksum) — so the same image may still live in several
|
||||
// galleries, each with its own row and its own stored object.
|
||||
package checksum
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"hash"
|
||||
"io"
|
||||
)
|
||||
|
||||
// New returns a fresh hasher. Upload hashes as it spools the body, so it needs
|
||||
// the writer rather than a finished reader.
|
||||
func New() hash.Hash { return sha256.New() }
|
||||
|
||||
// Format renders a hasher's digest the way it is stored.
|
||||
func Format(h hash.Hash) string { return hex.EncodeToString(h.Sum(nil)) }
|
||||
|
||||
// Sum reads r to EOF and returns its digest.
|
||||
func Sum(r io.Reader) (string, error) {
|
||||
h := New()
|
||||
if _, err := io.Copy(h, r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return Format(h), nil
|
||||
}
|
||||
Reference in New Issue
Block a user