Add photo content-hash dedup and local↔S3 library sync.

Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.

photo-api/internal/httpapi/photos.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
+	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
 	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
 	"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
 )
@@ -20,6 +21,10 @@ import (
 // Each file is sniffed by magic bytes (client filename/Content-Type are
 // untrusted, same policy as /api/media/upload), stored as the untouched
 // original, and queued for variant processing.
+//
+// A file whose bytes are already in this gallery is not stored a second time:
+// the incoming copy is discarded, the existing photo is echoed back with
+// "duplicate": true, and the rest of the batch carries on.
 func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.User) {
 	galleryID := r.PathValue("id")
 	g, err := s.db.GetGallery(r.Context(), galleryID)
@@ -57,7 +62,7 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
 			part.Close()
 			continue
 		}
-		photo, uploadErr := s.saveUpload(r, g, part, position, maxFile)
+		photo, duplicate, uploadErr := s.saveUpload(r, g, part, position, maxFile)
 		part.Close()
 		if uploadErr != nil {
 			// One bad file fails the request explicitly rather than silently
@@ -65,7 +70,12 @@ func (s *Server) uploadPhotos(w http.ResponseWriter, r *http.Request, _ auth.Use
 			writeError(w, uploadErr.status, uploadErr.msg)
 			return
 		}
-		created = append(created, s.photoToJSON(photo, s.viewTokenFor(g), true))
+		out := s.photoToJSON(photo, s.viewTokenFor(g), true)
+		out.Duplicate = duplicate
+		created = append(created, out)
+		if duplicate {
+			continue // nothing was stored, so the next file keeps this position
+		}
 		position++
 	}
 
@@ -82,51 +92,67 @@ type uploadError struct {
 	msg    string
 }
 
-func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, *uploadError) {
+// saveUpload stores one part. The bool it returns reports a duplicate: the
+// gallery already holds these bytes, so nothing was written and the photo
+// returned is the existing one.
+func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Part, position int, maxFile int64) (store.Photo, bool, *uploadError) {
 	head := make([]byte, 16)
 	n, err := io.ReadFull(part, head)
 	if err != nil && err != io.ErrUnexpectedEOF {
-		return store.Photo{}, &uploadError{http.StatusBadRequest, "Could not read file"}
+		return store.Photo{}, false, &uploadError{http.StatusBadRequest, "Could not read file"}
 	}
 	head = head[:n]
 	contentType, ext, ok, reason := imaging.Sniff(head)
 	if !ok {
-		return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType, reason}
+		return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType, reason}
 	}
 	if contentType == "image/heic" && imaging.DetectHeicConverter(s.cfg.HeicConverter) == nil {
-		return store.Photo{}, &uploadError{http.StatusUnsupportedMediaType,
+		return store.Photo{}, false, &uploadError{http.StatusUnsupportedMediaType,
 			"HEIC uploads need an image converter on the server (install libvips-tools); please upload JPEG instead"}
 	}
 
 	// Spool to a temp file to learn the size before handing to storage
-	// (S3 wants a length; local rename wants a file anyway).
+	// (S3 wants a length; local rename wants a file anyway). The same pass
+	// hashes the bytes for the duplicate check below.
 	tmp, err := os.CreateTemp(s.cfg.StoragePath, ".incoming-*")
 	if err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 	defer os.Remove(tmp.Name())
 	defer tmp.Close()
 
-	size, err := io.Copy(tmp, io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
+	hasher := checksum.New()
+	size, err := io.Copy(io.MultiWriter(tmp, hasher),
+		io.MultiReader(bytes.NewReader(head), io.LimitReader(part, maxFile+1)))
 	if err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 	if size > maxFile {
-		return store.Photo{}, &uploadError{http.StatusRequestEntityTooLarge,
+		return store.Photo{}, false, &uploadError{http.StatusRequestEntityTooLarge,
 			fmt.Sprintf("File exceeds the %d MB limit", s.cfg.MaxUploadMB)}
 	}
 	if _, err := tmp.Seek(0, io.SeekStart); err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+	}
+	sum := checksum.Format(hasher)
+
+	// Duplicate check before anything is written: the temp copy is dropped by
+	// the deferred Remove, no object is stored and no row is inserted.
+	if existing, err := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); err == nil {
+		return existing, true, nil
+	} else if err != store.ErrNotFound {
+		log.Printf("Error: %v", err)
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 
 	photoID := newID()
 	key := fmt.Sprintf("galleries/%s/orig/%s%s", g.ID, photoID, ext)
 	if err := s.storage.Put(r.Context(), key, tmp, size, contentType); err != nil {
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
 
 	now := time.Now()
@@ -142,13 +168,22 @@ func (s *Server) saveUpload(r *http.Request, g store.Gallery, part *multipart.Pa
 		NextAttemptAt:    now,
 		CreatedAt:        now,
 		UpdatedAt:        now,
+		Checksum:         sum,
 	}
 	if err := s.db.InsertPhoto(r.Context(), photo); err != nil {
 		s.storage.Delete(r.Context(), key)
+		// A concurrent upload of the same bytes won the race between the
+		// check above and this insert; the unique index caught it, so report
+		// the winner as the duplicate instead of failing.
+		if store.IsUniqueViolation(err) {
+			if existing, findErr := s.db.FindPhotoByChecksum(r.Context(), g.ID, sum); findErr == nil {
+				return existing, true, nil
+			}
+		}
 		log.Printf("Error: %v", err)
-		return store.Photo{}, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
+		return store.Photo{}, false, &uploadError{http.StatusInternalServerError, "Internal Server Error"}
 	}
-	return photo, nil
+	return photo, false, nil
 }
 
 type reorderBody struct {