Add photo content-hash dedup and local↔S3 library sync.

Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
2026-08-05 05:41:12 +00:00
parent dafa3711f8
commit 498d7d8a7d
31 changed files with 2216 additions and 64 deletions
@@ -54,6 +54,19 @@ func Open(cfg config.Config) (*DB, error) {
 	}
 }

+// IsUniqueViolation reports whether err is a duplicate-key error. The two
+// drivers wrap it in unrelated types (pgconn.PgError vs sqlite.Error), and
+// neither is worth importing here just for one check, so this matches on the
+// message: pgx renders "(SQLSTATE 23505)", modernc/sqlite "UNIQUE constraint
+// failed: ...".
+func IsUniqueViolation(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "SQLSTATE 23505") || strings.Contains(msg, "UNIQUE constraint failed")
+}
+
 // Rebind converts ?-style placeholders to $n for Postgres. Queries in this
 // package never contain literal question marks in strings.
 func (db *DB) Rebind(query string) string {
@@ -29,14 +29,17 @@ type Photo struct {
 	LastError        string
 	CreatedAt        time.Time
 	UpdatedAt        time.Time
+	// Checksum is the sha256 of the original bytes, empty when the row has
+	// not been hashed yet (uploaded before duplicate detection existed).
+	Checksum string
 }

 const photoColumns = `id, gallery_id, position, original_key, original_filename, content_type,
 	size_bytes, width, height, thumb_key, preview_key, taken_at, status, attempts, next_attempt_at,
-	last_error, created_at, updated_at`
+	last_error, created_at, updated_at, checksum`

 func scanPhoto(s scanner) (Photo, error) {
-	var v [18]any
+	var v [19]any
 	dest := make([]any, len(v))
 	for i := range v {
 		dest[i] = &v[i]
@@ -63,20 +66,82 @@ func scanPhoto(s scanner) (Photo, error) {
 		LastError:        asString(v[15]),
 		CreatedAt:        asTime(v[16]),
 		UpdatedAt:        asTime(v[17]),
+		Checksum:         asString(v[18]),
 	}, nil
 }

+// InsertPhoto returns an error satisfying IsUniqueViolation when the gallery
+// already holds a photo with the same checksum; callers treat that as a
+// duplicate rather than a failure.
 func (db *DB) InsertPhoto(ctx context.Context, p Photo) error {
 	_, err := db.ExecContext(ctx, db.Rebind(`
 		INSERT INTO photos_photos
 			(id, gallery_id, position, original_key, original_filename, content_type, size_bytes,
-			 status, attempts, next_attempt_at, created_at, updated_at)
-		VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?)`),
+			 status, attempts, next_attempt_at, created_at, updated_at, checksum)
+		VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?, ?)`),
 		p.ID, p.GalleryID, p.Position, p.OriginalKey, nullable(p.OriginalFilename), p.ContentType,
-		p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt))
+		p.SizeBytes, db.TimeArg(p.NextAttemptAt), db.TimeArg(p.CreatedAt), db.TimeArg(p.UpdatedAt),
+		nullable(p.Checksum))
 	return err
 }

+// FindPhotoByChecksum looks for an existing photo with the same content in one
+// gallery — the duplicate check the upload path runs before storing anything.
+// Duplicate scope is per gallery: the same image in another gallery keeps its
+// own row and its own stored object.
+func (db *DB) FindPhotoByChecksum(ctx context.Context, galleryID, checksum string) (Photo, error) {
+	if checksum == "" {
+		return Photo{}, ErrNotFound // never match the not-yet-hashed rows
+	}
+	row := db.QueryRowContext(ctx, db.Rebind(
+		"SELECT "+photoColumns+" FROM photos_photos WHERE gallery_id = ? AND checksum = ?"),
+		galleryID, checksum)
+	p, err := scanPhoto(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Photo{}, ErrNotFound
+	}
+	return p, err
+}
+
+// SetPhotoChecksum fills in the hash of an already-stored photo. Used by the
+// backfill; returns a unique-violation error when the row turns out to
+// duplicate one already hashed in the same gallery.
+func (db *DB) SetPhotoChecksum(ctx context.Context, id, checksum string) error {
+	res, err := db.ExecContext(ctx, db.Rebind(
+		"UPDATE photos_photos SET checksum = ? WHERE id = ?"), checksum, id)
+	if err != nil {
+		return err
+	}
+	return errIfNoRows(res)
+}
+
+// PhotosMissingChecksum lists rows still lacking a hash (optionally of one
+// gallery only), oldest first so backfill keeps the earliest upload of a
+// duplicate pair as the one that gets the checksum.
+func (db *DB) PhotosMissingChecksum(ctx context.Context, galleryID string) ([]Photo, error) {
+	q := "SELECT " + photoColumns + " FROM photos_photos WHERE checksum IS NULL"
+	var args []any
+	if galleryID != "" {
+		q += " AND gallery_id = ?"
+		args = append(args, galleryID)
+	}
+	q += " ORDER BY gallery_id, position, created_at"
+	rows, err := db.QueryContext(ctx, db.Rebind(q), args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	photos := []Photo{}
+	for rows.Next() {
+		p, err := scanPhoto(rows)
+		if err != nil {
+			return nil, err
+		}
+		photos = append(photos, p)
+	}
+	return photos, rows.Err()
+}
+
 func (db *DB) GetPhoto(ctx context.Context, id string) (Photo, error) {
 	row := db.QueryRowContext(ctx,
 		db.Rebind("SELECT "+photoColumns+" FROM photos_photos WHERE id = ?"), id)
@@ -169,6 +234,64 @@ func (db *DB) PhotoKeys(ctx context.Context, galleryID string) ([]string, error)
 	return keys, rows.Err()
 }

+// PhotoObject is one stored object: a photo variant plus the content type it
+// should be written with. Used by the storage sync, which treats the rows as
+// the inventory of what exists.
+type PhotoObject struct {
+	PhotoID     string
+	GalleryID   string
+	Variant     string // original | thumb | preview
+	Key         string
+	ContentType string
+}
+
+// AllPhotoObjects lists every object of every photo (optionally of one
+// gallery only), oldest first. Variant keys are empty until the worker has
+// processed the photo; those are left out.
+func (db *DB) AllPhotoObjects(ctx context.Context, galleryID string) ([]PhotoObject, error) {
+	q := `SELECT id, gallery_id, content_type, original_key, thumb_key, preview_key
+		FROM photos_photos`
+	var args []any
+	if galleryID != "" {
+		q += " WHERE gallery_id = ?"
+		args = append(args, galleryID)
+	}
+	q += " ORDER BY gallery_id, position, created_at"
+	rows, err := db.QueryContext(ctx, db.Rebind(q), args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	objects := []PhotoObject{}
+	for rows.Next() {
+		var id, galID, ctype, orig, thumb, preview any
+		if err := rows.Scan(&id, &galID, &ctype, &orig, &thumb, &preview); err != nil {
+			return nil, err
+		}
+		variants := []struct {
+			name, key, contentType string
+		}{
+			{"original", asString(orig), asString(ctype)},
+			{"thumb", asString(thumb), "image/jpeg"},
+			{"preview", asString(preview), "image/jpeg"},
+		}
+		for _, v := range variants {
+			if v.key == "" {
+				continue
+			}
+			objects = append(objects, PhotoObject{
+				PhotoID:     asString(id),
+				GalleryID:   asString(galID),
+				Variant:     v.name,
+				Key:         v.key,
+				ContentType: v.contentType,
+			})
+		}
+	}
+	return objects, rows.Err()
+}
+
 // ClaimNextPhoto picks the oldest due queued/failed photo and marks it
 // processing. Optimistic claim (RowsAffected check) works identically on
 // Postgres and SQLite; returns ErrNotFound when the queue is empty.