Add photo content-hash dedup and local↔S3 library sync.
Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
This commit is contained in:
Michilis
+135
-15
@@ -1,12 +1,15 @@
|
||||
// photo-api serves event photo galleries for the Spanglish platform.
|
||||
//
|
||||
// photo-api start the HTTP server
|
||||
// photo-api migrate apply pending photos_* migrations and exit
|
||||
// photo-api start the HTTP server
|
||||
// photo-api migrate apply pending photos_* migrations and exit
|
||||
// photo-api sync to-s3|to-local copy the photo library between backends
|
||||
// photo-api backfill-checksums hash photos uploaded before duplicate detection
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -16,9 +19,11 @@ import (
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/httpapi"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/photosync"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/worker"
|
||||
@@ -38,15 +43,21 @@ func main() {
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "migrate" {
|
||||
if err := db.Migrate(context.Background()); err != nil {
|
||||
log.Fatalf("migrate: %v", err)
|
||||
}
|
||||
log.Println("migrations up to date")
|
||||
return
|
||||
}
|
||||
if len(os.Args) > 1 {
|
||||
log.Fatalf("unknown subcommand %q (expected: migrate)", os.Args[1])
|
||||
switch os.Args[1] {
|
||||
case "migrate":
|
||||
if err := db.Migrate(context.Background()); err != nil {
|
||||
log.Fatalf("migrate: %v", err)
|
||||
}
|
||||
log.Println("migrations up to date")
|
||||
case "sync":
|
||||
runSync(cfg, db, os.Args[2:])
|
||||
case "backfill-checksums":
|
||||
runBackfillChecksums(cfg, db, os.Args[2:])
|
||||
default:
|
||||
log.Fatalf("unknown subcommand %q (expected: migrate, sync, backfill-checksums)", os.Args[1])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
@@ -64,11 +75,7 @@ func main() {
|
||||
if err != nil {
|
||||
log.Fatalf("storage: %v", err)
|
||||
}
|
||||
if cfg.S3Enabled() {
|
||||
log.Printf("storage: S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint)
|
||||
} else {
|
||||
log.Printf("storage: local disk at %s", cfg.StoragePath)
|
||||
}
|
||||
log.Printf("storage: %s (STORAGE_BACKEND=%s)", storage.Name(cfg, st), cfg.StorageBackend)
|
||||
|
||||
heic := imaging.DetectHeicConverter(cfg.HeicConverter)
|
||||
if heic == nil {
|
||||
@@ -104,3 +111,116 @@ func main() {
|
||||
log.Printf("shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
const syncUsage = `usage: photo-api sync to-s3|to-local [flags]
|
||||
|
||||
Copies every object of every photo (original, thumb, preview) from one storage
|
||||
backend to the other. Both must be configured in photo-api/.env; the source is
|
||||
never modified, and reruns skip objects already present on the destination.
|
||||
|
||||
Flags:
|
||||
`
|
||||
|
||||
// runSync handles `photo-api sync <direction> [flags]`, the storage migration
|
||||
// used when moving the library between local disk and S3.
|
||||
func runSync(cfg config.Config, db *store.DB, args []string) {
|
||||
fs := flag.NewFlagSet("sync", flag.ExitOnError)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprint(fs.Output(), syncUsage)
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
var (
|
||||
dryRun = fs.Bool("dry-run", false, "report what would be copied without writing")
|
||||
overwrite = fs.Bool("overwrite", false, "re-copy objects already present with the same size")
|
||||
workers = fs.Int("concurrency", 4, "objects copied in parallel")
|
||||
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
||||
)
|
||||
if len(args) == 0 {
|
||||
fs.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
direction, err := photosync.ParseDirection(args[0])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%v\n\n", err)
|
||||
fs.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if _, err := photosync.Run(ctx, cfg, db, photosync.Options{
|
||||
Direction: direction,
|
||||
GalleryID: *gallery,
|
||||
Concurrency: *workers,
|
||||
DryRun: *dryRun,
|
||||
Overwrite: *overwrite,
|
||||
}); err != nil {
|
||||
log.Fatalf("sync: %v", err)
|
||||
}
|
||||
if *dryRun {
|
||||
return
|
||||
}
|
||||
target := "s3"
|
||||
if direction == photosync.ToLocal {
|
||||
target = "local"
|
||||
}
|
||||
log.Printf("sync: set STORAGE_BACKEND=%s in photo-api/.env and restart the service to serve from it "+
|
||||
"(the source copy is left in place — delete it once the switch is verified)", target)
|
||||
}
|
||||
|
||||
const backfillUsage = `usage: photo-api backfill-checksums [flags]
|
||||
|
||||
Hashes the stored original of every photo that has no checksum yet, so uploads
|
||||
of a photo already in a gallery are recognised as duplicates. Runs against the
|
||||
active storage backend (STORAGE_BACKEND) and only ever writes the checksum
|
||||
column — no photo is deleted. Photos whose content already matches an earlier
|
||||
one in the same gallery are reported and left unhashed.
|
||||
|
||||
Flags:
|
||||
`
|
||||
|
||||
// runBackfillChecksums handles `photo-api backfill-checksums [flags]`, the
|
||||
// one-off pass needed after the checksum column is added to an existing
|
||||
// library.
|
||||
func runBackfillChecksums(cfg config.Config, db *store.DB, args []string) {
|
||||
fs := flag.NewFlagSet("backfill-checksums", flag.ExitOnError)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprint(fs.Output(), backfillUsage)
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
var (
|
||||
dryRun = fs.Bool("dry-run", false, "report what would be hashed without writing")
|
||||
workers = fs.Int("concurrency", 4, "originals hashed in parallel")
|
||||
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
||||
)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
st, err := storage.New(cfg)
|
||||
if err != nil {
|
||||
log.Fatalf("storage: %v", err)
|
||||
}
|
||||
log.Printf("backfill-checksums: reading from %s", storage.Name(cfg, st))
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
res, err := checksum.Backfill(ctx, db, st, checksum.Options{
|
||||
GalleryID: *gallery,
|
||||
Concurrency: *workers,
|
||||
DryRun: *dryRun,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("backfill-checksums: %v", err)
|
||||
}
|
||||
if res.Duplicates > 0 {
|
||||
log.Printf("backfill-checksums: %d existing photos duplicate an earlier one in their gallery "+
|
||||
"(listed above); they still show in the gallery — delete the unwanted ones from the admin page",
|
||||
res.Duplicates)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user