Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
227 lines
7.0 KiB
Go
227 lines
7.0 KiB
Go
// photo-api serves event photo galleries for the Spanglish platform.
|
|
//
|
|
// photo-api start the HTTP server
|
|
// photo-api migrate apply pending photos_* migrations and exit
|
|
// photo-api sync to-s3|to-local copy the photo library between backends
|
|
// photo-api backfill-checksums hash photos uploaded before duplicate detection
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/auth"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/checksum"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/httpapi"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/imaging"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/photosync"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
|
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/worker"
|
|
)
|
|
|
|
func main() {
|
|
log.SetFlags(log.LstdFlags | log.Lmsgprefix)
|
|
log.SetPrefix("photo-api: ")
|
|
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
log.Fatalf("config: %v", err)
|
|
}
|
|
db, err := store.Open(cfg)
|
|
if err != nil {
|
|
log.Fatalf("database: %v", err)
|
|
}
|
|
defer db.Close()
|
|
|
|
if len(os.Args) > 1 {
|
|
switch os.Args[1] {
|
|
case "migrate":
|
|
if err := db.Migrate(context.Background()); err != nil {
|
|
log.Fatalf("migrate: %v", err)
|
|
}
|
|
log.Println("migrations up to date")
|
|
case "sync":
|
|
runSync(cfg, db, os.Args[2:])
|
|
case "backfill-checksums":
|
|
runBackfillChecksums(cfg, db, os.Args[2:])
|
|
default:
|
|
log.Fatalf("unknown subcommand %q (expected: migrate, sync, backfill-checksums)", os.Args[1])
|
|
}
|
|
return
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
if err := db.PingContext(ctx); err != nil {
|
|
log.Fatalf("database ping: %v", err)
|
|
}
|
|
// Fail fast if the columns read from Drizzle-owned tables changed.
|
|
if err := db.CheckMainSchema(ctx); err != nil {
|
|
log.Fatalf("%v", err)
|
|
}
|
|
|
|
st, err := storage.New(cfg)
|
|
if err != nil {
|
|
log.Fatalf("storage: %v", err)
|
|
}
|
|
log.Printf("storage: %s (STORAGE_BACKEND=%s)", storage.Name(cfg, st), cfg.StorageBackend)
|
|
|
|
heic := imaging.DetectHeicConverter(cfg.HeicConverter)
|
|
if heic == nil {
|
|
log.Println("WARNING: no HEIC converter found (install libvips-tools or libheif-examples); HEIC uploads will be rejected")
|
|
} else {
|
|
log.Printf("HEIC converter: %s", heic.Name())
|
|
}
|
|
|
|
if err := os.MkdirAll(cfg.StoragePath, 0o755); err != nil {
|
|
log.Fatalf("storage path: %v", err)
|
|
}
|
|
wrk := worker.New(db, st, heic, cfg.StoragePath, cfg.WorkerConcurrency)
|
|
wrk.Run(ctx)
|
|
|
|
server := &http.Server{
|
|
Addr: fmt.Sprintf(":%d", cfg.Port),
|
|
Handler: httpapi.New(cfg, db, st, auth.NewVerifier(db), wrk).Handler(),
|
|
ReadHeaderTimeout: 10 * time.Second,
|
|
IdleTimeout: 60 * time.Second,
|
|
}
|
|
go func() {
|
|
log.Printf("listening on :%d", cfg.Port)
|
|
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
log.Fatalf("serve: %v", err)
|
|
}
|
|
}()
|
|
|
|
<-ctx.Done()
|
|
log.Println("shutting down")
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
if err := server.Shutdown(shutdownCtx); err != nil {
|
|
log.Printf("shutdown: %v", err)
|
|
}
|
|
}
|
|
|
|
const syncUsage = `usage: photo-api sync to-s3|to-local [flags]
|
|
|
|
Copies every object of every photo (original, thumb, preview) from one storage
|
|
backend to the other. Both must be configured in photo-api/.env; the source is
|
|
never modified, and reruns skip objects already present on the destination.
|
|
|
|
Flags:
|
|
`
|
|
|
|
// runSync handles `photo-api sync <direction> [flags]`, the storage migration
|
|
// used when moving the library between local disk and S3.
|
|
func runSync(cfg config.Config, db *store.DB, args []string) {
|
|
fs := flag.NewFlagSet("sync", flag.ExitOnError)
|
|
fs.Usage = func() {
|
|
fmt.Fprint(fs.Output(), syncUsage)
|
|
fs.PrintDefaults()
|
|
}
|
|
var (
|
|
dryRun = fs.Bool("dry-run", false, "report what would be copied without writing")
|
|
overwrite = fs.Bool("overwrite", false, "re-copy objects already present with the same size")
|
|
workers = fs.Int("concurrency", 4, "objects copied in parallel")
|
|
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
|
)
|
|
if len(args) == 0 {
|
|
fs.Usage()
|
|
os.Exit(2)
|
|
}
|
|
direction, err := photosync.ParseDirection(args[0])
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "%v\n\n", err)
|
|
fs.Usage()
|
|
os.Exit(2)
|
|
}
|
|
if err := fs.Parse(args[1:]); err != nil {
|
|
os.Exit(2)
|
|
}
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
if _, err := photosync.Run(ctx, cfg, db, photosync.Options{
|
|
Direction: direction,
|
|
GalleryID: *gallery,
|
|
Concurrency: *workers,
|
|
DryRun: *dryRun,
|
|
Overwrite: *overwrite,
|
|
}); err != nil {
|
|
log.Fatalf("sync: %v", err)
|
|
}
|
|
if *dryRun {
|
|
return
|
|
}
|
|
target := "s3"
|
|
if direction == photosync.ToLocal {
|
|
target = "local"
|
|
}
|
|
log.Printf("sync: set STORAGE_BACKEND=%s in photo-api/.env and restart the service to serve from it "+
|
|
"(the source copy is left in place — delete it once the switch is verified)", target)
|
|
}
|
|
|
|
const backfillUsage = `usage: photo-api backfill-checksums [flags]
|
|
|
|
Hashes the stored original of every photo that has no checksum yet, so uploads
|
|
of a photo already in a gallery are recognised as duplicates. Runs against the
|
|
active storage backend (STORAGE_BACKEND) and only ever writes the checksum
|
|
column — no photo is deleted. Photos whose content already matches an earlier
|
|
one in the same gallery are reported and left unhashed.
|
|
|
|
Flags:
|
|
`
|
|
|
|
// runBackfillChecksums handles `photo-api backfill-checksums [flags]`, the
|
|
// one-off pass needed after the checksum column is added to an existing
|
|
// library.
|
|
func runBackfillChecksums(cfg config.Config, db *store.DB, args []string) {
|
|
fs := flag.NewFlagSet("backfill-checksums", flag.ExitOnError)
|
|
fs.Usage = func() {
|
|
fmt.Fprint(fs.Output(), backfillUsage)
|
|
fs.PrintDefaults()
|
|
}
|
|
var (
|
|
dryRun = fs.Bool("dry-run", false, "report what would be hashed without writing")
|
|
workers = fs.Int("concurrency", 4, "originals hashed in parallel")
|
|
gallery = fs.String("gallery", "", "limit to one gallery id (default: whole library)")
|
|
)
|
|
if err := fs.Parse(args); err != nil {
|
|
os.Exit(2)
|
|
}
|
|
|
|
st, err := storage.New(cfg)
|
|
if err != nil {
|
|
log.Fatalf("storage: %v", err)
|
|
}
|
|
log.Printf("backfill-checksums: reading from %s", storage.Name(cfg, st))
|
|
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
res, err := checksum.Backfill(ctx, db, st, checksum.Options{
|
|
GalleryID: *gallery,
|
|
Concurrency: *workers,
|
|
DryRun: *dryRun,
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("backfill-checksums: %v", err)
|
|
}
|
|
if res.Duplicates > 0 {
|
|
log.Printf("backfill-checksums: %d existing photos duplicate an earlier one in their gallery "+
|
|
"(listed above); they still show in the gallery — delete the unwanted ones from the admin page",
|
|
res.Duplicates)
|
|
}
|
|
}
|