Add photo content-hash dedup and local↔S3 library sync.
Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
This commit is contained in:
Michilis
@@ -0,0 +1,229 @@
|
||||
// Package photosync copies the photo library between the local-disk and the
|
||||
// S3 backend so the active one (STORAGE_BACKEND) can be switched without
|
||||
// losing photos. Both backends must be configured in photo-api/.env for a
|
||||
// sync to run — the direction picks which is the source.
|
||||
//
|
||||
// The photos_photos rows are the inventory: every row contributes its
|
||||
// original key and, once the worker has processed it, its thumb and preview
|
||||
// key. Objects on disk or in the bucket with no row (worker scratch dirs,
|
||||
// leftovers of deleted galleries) are deliberately not copied.
|
||||
//
|
||||
// Sync only ever writes to the destination. The source is left untouched, so
|
||||
// a sync is repeatable, safe to interrupt, and leaves the old backend as a
|
||||
// fallback until it is cleaned up by hand.
|
||||
package photosync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// Direction is which way objects move.
|
||||
type Direction string
|
||||
|
||||
const (
|
||||
ToS3 Direction = "to-s3"
|
||||
ToLocal Direction = "to-local"
|
||||
)
|
||||
|
||||
// ParseDirection accepts the short and the spelled-out form.
|
||||
func ParseDirection(s string) (Direction, error) {
|
||||
switch s {
|
||||
case "to-s3", "local-to-s3":
|
||||
return ToS3, nil
|
||||
case "to-local", "s3-to-local":
|
||||
return ToLocal, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown direction %q (expected to-s3 or to-local)", s)
|
||||
}
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
Direction Direction
|
||||
// GalleryID limits the sync to one gallery; empty means the whole library.
|
||||
GalleryID string
|
||||
// Concurrency is how many objects are copied at a time.
|
||||
Concurrency int
|
||||
// DryRun reports what would be copied without writing anything.
|
||||
DryRun bool
|
||||
// Overwrite re-copies objects that already exist on the destination with
|
||||
// the same size (default: those are skipped, which makes reruns cheap).
|
||||
Overwrite bool
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Total int // objects in the inventory
|
||||
Copied int // copied (or, with DryRun, would be copied)
|
||||
Skipped int // already on the destination
|
||||
Missing int // absent from the source — nothing to copy
|
||||
Failed int // copy attempted and errored
|
||||
Bytes int64 // bytes copied
|
||||
}
|
||||
|
||||
// Run copies the library in the requested direction. It returns a Result even
|
||||
// on error, and an error if any object failed.
|
||||
func Run(ctx context.Context, cfg config.Config, db *store.DB, opts Options) (Result, error) {
|
||||
if !cfg.S3Configured() {
|
||||
return Result{}, errors.New("sync needs both backends configured: set S3_ENDPOINT, S3_BUCKET, " +
|
||||
"S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY (next to STORAGE_PATH) in photo-api/.env")
|
||||
}
|
||||
if opts.Concurrency < 1 {
|
||||
opts.Concurrency = 4
|
||||
}
|
||||
|
||||
local, err := storage.NewLocal(cfg.StoragePath)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("local storage: %w", err)
|
||||
}
|
||||
s3, err := storage.NewS3(cfg)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("s3 storage: %w", err)
|
||||
}
|
||||
src, dst := local, s3
|
||||
srcName, dstName := "local disk "+cfg.StoragePath, "s3 bucket "+cfg.S3Bucket
|
||||
if opts.Direction == ToLocal {
|
||||
src, dst = s3, local
|
||||
srcName, dstName = dstName, srcName
|
||||
}
|
||||
|
||||
objects, err := db.AllPhotoObjects(ctx, opts.GalleryID)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("list photo objects: %w", err)
|
||||
}
|
||||
|
||||
prefix := ""
|
||||
if opts.DryRun {
|
||||
prefix = "[dry-run] "
|
||||
}
|
||||
log.Printf("sync: %s%s → %s: %d objects, concurrency %d",
|
||||
prefix, srcName, dstName, len(objects), opts.Concurrency)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
res = Result{Total: len(objects)}
|
||||
done int64
|
||||
jobs = make(chan store.PhotoObject)
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for i := 0; i < opts.Concurrency; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for o := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
act, size, err := copyObject(ctx, src, dst, o, opts)
|
||||
n := atomic.AddInt64(&done, 1)
|
||||
|
||||
mu.Lock()
|
||||
switch {
|
||||
case err != nil:
|
||||
res.Failed++
|
||||
log.Printf("sync: [%d/%d] FAILED %s (photo %s %s): %v",
|
||||
n, len(objects), o.Key, o.PhotoID, o.Variant, err)
|
||||
case act == actionCopied:
|
||||
res.Copied++
|
||||
res.Bytes += size
|
||||
log.Printf("sync: [%d/%d] %scopied %s (%s)", n, len(objects), prefix, o.Key, humanBytes(size))
|
||||
case act == actionMissing:
|
||||
res.Missing++
|
||||
log.Printf("sync: [%d/%d] MISSING on source, skipped: %s (photo %s %s)",
|
||||
n, len(objects), o.Key, o.PhotoID, o.Variant)
|
||||
default:
|
||||
res.Skipped++
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, o := range objects {
|
||||
select {
|
||||
case jobs <- o:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
|
||||
log.Printf("sync: %sdone — %d copied (%s), %d already present, %d missing on source, %d failed",
|
||||
prefix, res.Copied, humanBytes(res.Bytes), res.Skipped, res.Missing, res.Failed)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, fmt.Errorf("interrupted after %d/%d objects: %w", res.Copied+res.Skipped, res.Total, err)
|
||||
}
|
||||
if res.Failed > 0 {
|
||||
return res, fmt.Errorf("%d of %d objects failed to copy (rerun to retry; already-copied objects are skipped)",
|
||||
res.Failed, res.Total)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
type action int
|
||||
|
||||
const (
|
||||
actionCopied action = iota
|
||||
actionSkipped
|
||||
actionMissing
|
||||
)
|
||||
|
||||
func copyObject(ctx context.Context, src, dst storage.Storage, o store.PhotoObject, opts Options) (action, int64, error) {
|
||||
srcSize, err := src.Stat(ctx, o.Key)
|
||||
if errors.Is(err, storage.ErrNotExist) {
|
||||
return actionMissing, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return actionSkipped, 0, fmt.Errorf("stat source: %w", err)
|
||||
}
|
||||
if !opts.Overwrite {
|
||||
if dstSize, err := dst.Stat(ctx, o.Key); err == nil && dstSize == srcSize {
|
||||
return actionSkipped, 0, nil
|
||||
} else if err != nil && !errors.Is(err, storage.ErrNotExist) {
|
||||
return actionSkipped, 0, fmt.Errorf("stat destination: %w", err)
|
||||
}
|
||||
}
|
||||
if opts.DryRun {
|
||||
return actionCopied, srcSize, nil
|
||||
}
|
||||
|
||||
r, size, err := src.Open(ctx, o.Key)
|
||||
if err != nil {
|
||||
return actionSkipped, 0, fmt.Errorf("read source: %w", err)
|
||||
}
|
||||
defer r.Close()
|
||||
if size <= 0 {
|
||||
size = srcSize // local Open reports the stat size; be defensive anyway
|
||||
}
|
||||
contentType := o.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
if err := dst.Put(ctx, o.Key, r, size, contentType); err != nil {
|
||||
return actionSkipped, 0, fmt.Errorf("write destination: %w", err)
|
||||
}
|
||||
return actionCopied, size, nil
|
||||
}
|
||||
|
||||
func humanBytes(n int64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
v, exp := float64(n), 0
|
||||
for v >= unit && exp < 4 {
|
||||
v /= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", v, "KMGT"[exp-1])
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package photosync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// Two local backends stand in for the real pair: copyObject only talks to the
|
||||
// storage.Storage interface, so the S3 side needs no fake here.
|
||||
func backends(t *testing.T) (src, dst storage.Storage, srcRoot string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
srcRoot = filepath.Join(dir, "src")
|
||||
src, err := storage.NewLocal(srcRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dst, err = storage.NewLocal(filepath.Join(dir, "dst"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return src, dst, srcRoot
|
||||
}
|
||||
|
||||
func put(t *testing.T, s storage.Storage, key, body string) {
|
||||
t.Helper()
|
||||
if err := s.Put(context.Background(), key, strings.NewReader(body), int64(len(body)), "image/jpeg"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
const testKey = "galleries/g1/original/p1.jpg"
|
||||
|
||||
func testObject() store.PhotoObject {
|
||||
return store.PhotoObject{PhotoID: "p1", GalleryID: "g1", Variant: "original", Key: testKey, ContentType: "image/jpeg"}
|
||||
}
|
||||
|
||||
func TestCopyObject(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("copies to destination", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "hello-photo")
|
||||
|
||||
act, size, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||
if err != nil || act != actionCopied || size != 11 {
|
||||
t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err)
|
||||
}
|
||||
if got, err := dst.Stat(ctx, testKey); err != nil || got != 11 {
|
||||
t.Fatalf("destination stat: (%d, %v)", got, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips same-size object already present", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "hello-photo")
|
||||
put(t, dst, testKey, "hello-photo")
|
||||
|
||||
act, _, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||
if err != nil || act != actionSkipped {
|
||||
t.Fatalf("got (%v, %v), want (skipped, nil)", act, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overwrite re-copies", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "new-content")
|
||||
put(t, dst, testKey, "old-content")
|
||||
|
||||
act, _, err := copyObject(ctx, src, dst, testObject(), Options{Overwrite: true})
|
||||
if err != nil || act != actionCopied {
|
||||
t.Fatalf("got (%v, %v), want (copied, nil)", act, err)
|
||||
}
|
||||
r, _, err := dst.Open(ctx, testKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "new-content" {
|
||||
t.Fatalf("destination body = %q, want new-content", body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("reports objects missing on the source", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
|
||||
act, _, err := copyObject(ctx, src, dst, testObject(), Options{})
|
||||
if err != nil || act != actionMissing {
|
||||
t.Fatalf("got (%v, %v), want (missing, nil)", act, err)
|
||||
}
|
||||
if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist {
|
||||
t.Fatalf("destination stat err = %v, want ErrNotExist", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dry run writes nothing", func(t *testing.T) {
|
||||
src, dst, _ := backends(t)
|
||||
put(t, src, testKey, "hello-photo")
|
||||
|
||||
act, size, err := copyObject(ctx, src, dst, testObject(), Options{DryRun: true})
|
||||
if err != nil || act != actionCopied || size != 11 {
|
||||
t.Fatalf("got (%v, %d, %v), want (copied, 11, nil)", act, size, err)
|
||||
}
|
||||
if _, err := dst.Stat(ctx, testKey); err != storage.ErrNotExist {
|
||||
t.Fatalf("destination stat err = %v, want ErrNotExist", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseDirection(t *testing.T) {
|
||||
for in, want := range map[string]Direction{
|
||||
"to-s3": ToS3,
|
||||
"local-to-s3": ToS3,
|
||||
"to-local": ToLocal,
|
||||
"s3-to-local": ToLocal,
|
||||
} {
|
||||
got, err := ParseDirection(in)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("ParseDirection(%q) = (%v, %v), want %v", in, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseDirection("sideways"); err == nil {
|
||||
t.Error("ParseDirection(\"sideways\") should fail")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package photosync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/storage"
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/store"
|
||||
)
|
||||
|
||||
// stubS3 is a minimal path-style S3 (PUT/HEAD/GET on /bucket/key) so the sync
|
||||
// can be driven end to end over the real aws-sdk client.
|
||||
type stubS3 struct {
|
||||
mu sync.Mutex
|
||||
objects map[string][]byte
|
||||
puts int
|
||||
}
|
||||
|
||||
func newStubS3(t *testing.T) (*stubS3, string) {
|
||||
t.Helper()
|
||||
s := &stubS3{objects: map[string][]byte{}}
|
||||
srv := httptest.NewServer(s)
|
||||
t.Cleanup(srv.Close)
|
||||
return s, srv.URL
|
||||
}
|
||||
|
||||
func (s *stubS3) get(key string) ([]byte, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
b, ok := s.objects[key]
|
||||
return b, ok
|
||||
}
|
||||
|
||||
func (s *stubS3) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
key := strings.TrimPrefix(r.URL.Path, "/test-bucket/")
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
body, err := readS3Body(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.objects[key] = body
|
||||
s.puts++
|
||||
s.mu.Unlock()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodHead:
|
||||
body, ok := s.get(key)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound) // a HEAD carries no error body
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodGet:
|
||||
body, ok := s.get(key)
|
||||
if !ok {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
fmt.Fprint(w, `<Error><Code>NoSuchKey</Code></Error>`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.Write(body)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
// readS3Body undoes the SDK's aws-chunked framing when it streams with a
|
||||
// trailing checksum (what it does for the non-seekable S3-to-S3 style reader).
|
||||
func readS3Body(r *http.Request) ([]byte, error) {
|
||||
raw, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") {
|
||||
return raw, nil
|
||||
}
|
||||
var out []byte
|
||||
rest := raw
|
||||
for {
|
||||
nl := strings.Index(string(rest), "\r\n")
|
||||
if nl < 0 {
|
||||
return out, nil
|
||||
}
|
||||
header := string(rest[:nl])
|
||||
rest = rest[nl+2:]
|
||||
size, err := strconv.ParseInt(strings.SplitN(header, ";", 2)[0], 16, 64)
|
||||
if err != nil || size == 0 {
|
||||
return out, nil // trailer section or malformed: body is complete
|
||||
}
|
||||
if int64(len(rest)) < size {
|
||||
return nil, fmt.Errorf("truncated aws-chunked body")
|
||||
}
|
||||
out = append(out, rest[:size]...)
|
||||
rest = rest[size:]
|
||||
if len(rest) >= 2 {
|
||||
rest = rest[2:] // chunk CRLF
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// syncEnv seeds a scratch SQLite DB with one gallery holding one ready photo
|
||||
// (original + thumb + preview) and returns a config wired to the stub bucket.
|
||||
func syncEnv(t *testing.T) (config.Config, *store.DB, *stubS3, []string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
stub, endpoint := newStubS3(t)
|
||||
cfg := config.Config{
|
||||
DBType: "sqlite",
|
||||
DatabaseURL: filepath.Join(dir, "test.db"),
|
||||
ViewTokenSecret: "test-secret",
|
||||
StoragePath: filepath.Join(dir, "photos"),
|
||||
S3Endpoint: endpoint,
|
||||
S3Region: "auto",
|
||||
S3Bucket: "test-bucket",
|
||||
S3AccessKeyID: "key",
|
||||
S3SecretKey: "secret",
|
||||
S3ForcePathStyle: true,
|
||||
}
|
||||
db, err := store.Open(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
ctx := context.Background()
|
||||
if err := db.Migrate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const (
|
||||
galleryID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
|
||||
photoID = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
|
||||
)
|
||||
now := time.Now()
|
||||
if err := db.CreateGallery(ctx, store.Gallery{
|
||||
ID: galleryID, Slug: "trip", Title: "Trip", Visibility: store.VisibilityPublic,
|
||||
ShareToken: "tok", CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
origKey := "galleries/" + galleryID + "/original/" + photoID + ".jpg"
|
||||
thumbKey := "galleries/" + galleryID + "/thumb/" + photoID + ".jpg"
|
||||
previewKey := "galleries/" + galleryID + "/preview/" + photoID + ".jpg"
|
||||
if err := db.InsertPhoto(ctx, store.Photo{
|
||||
ID: photoID, GalleryID: galleryID, OriginalKey: origKey, ContentType: "image/jpeg",
|
||||
SizeBytes: 11, NextAttemptAt: now, CreatedAt: now, UpdatedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.MarkPhotoReady(ctx, photoID, thumbKey, previewKey, 100, 80, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cfg, db, stub, []string{origKey, thumbKey, previewKey}
|
||||
}
|
||||
|
||||
func TestRunToS3(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg, db, stub, keys := syncEnv(t)
|
||||
|
||||
src, err := storage.NewLocal(cfg.StoragePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, k := range keys {
|
||||
put(t, src, k, fmt.Sprintf("photo-bytes-%d", i))
|
||||
}
|
||||
|
||||
res, err := Run(ctx, cfg, db, Options{Direction: ToS3, Concurrency: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 3 || res.Copied != 3 || res.Skipped != 0 || res.Failed != 0 || res.Missing != 0 {
|
||||
t.Fatalf("first run = %+v, want 3 total / 3 copied", res)
|
||||
}
|
||||
for i, k := range keys {
|
||||
body, ok := stub.get(k)
|
||||
if !ok {
|
||||
t.Fatalf("%s not uploaded", k)
|
||||
}
|
||||
if want := fmt.Sprintf("photo-bytes-%d", i); string(body) != want {
|
||||
t.Errorf("%s = %q, want %q", k, body, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Rerunning is a no-op: everything is already there at the same size.
|
||||
res, err = Run(ctx, cfg, db, Options{Direction: ToS3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Copied != 0 || res.Skipped != 3 {
|
||||
t.Fatalf("rerun = %+v, want 0 copied / 3 skipped", res)
|
||||
}
|
||||
|
||||
// --overwrite re-uploads them.
|
||||
before := stub.puts
|
||||
res, err = Run(ctx, cfg, db, Options{Direction: ToS3, Overwrite: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Copied != 3 || stub.puts != before+3 {
|
||||
t.Fatalf("overwrite run = %+v, puts %d → %d", res, before, stub.puts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunToLocal(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg, db, stub, keys := syncEnv(t)
|
||||
for i, k := range keys {
|
||||
stub.objects[k] = []byte(fmt.Sprintf("s3-bytes-%d", i))
|
||||
}
|
||||
|
||||
res, err := Run(ctx, cfg, db, Options{Direction: ToLocal})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Total != 3 || res.Copied != 3 || res.Failed != 0 {
|
||||
t.Fatalf("run = %+v, want 3 total / 3 copied", res)
|
||||
}
|
||||
for i, k := range keys {
|
||||
body, err := os.ReadFile(filepath.Join(cfg.StoragePath, k))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", k, err)
|
||||
}
|
||||
if want := fmt.Sprintf("s3-bytes-%d", i); string(body) != want {
|
||||
t.Errorf("%s = %q, want %q", k, body, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMissingOnSource(t *testing.T) {
|
||||
cfg, db, _, _ := syncEnv(t)
|
||||
// Nothing on local disk: every object is reported missing, none fail.
|
||||
res, err := Run(context.Background(), cfg, db, Options{Direction: ToS3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Missing != 3 || res.Copied != 0 || res.Failed != 0 {
|
||||
t.Fatalf("run = %+v, want 3 missing", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRequiresS3Config(t *testing.T) {
|
||||
cfg, db, _, _ := syncEnv(t)
|
||||
cfg.S3Endpoint, cfg.S3Bucket = "", ""
|
||||
if _, err := Run(context.Background(), cfg, db, Options{Direction: ToS3}); err == nil {
|
||||
t.Fatal("sync without S3 configured should fail")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user