Add photo content-hash dedup and local↔S3 library sync.

Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
This commit is contained in:
Michilis
2026-08-05 05:41:12 +00:00
parent dafa3711f8
commit 498d7d8a7d
31 changed files with 2216 additions and 64 deletions
+21
View File
@@ -17,6 +17,12 @@ type local struct {
root string
}
// NewLocal builds the local-disk backend explicitly, whichever backend is
// active — `photo-api sync` needs both sides at once.
func NewLocal(root string) (Storage, error) {
return newLocal(root)
}
func newLocal(root string) (*local, error) {
if err := os.MkdirAll(root, 0o755); err != nil {
return nil, fmt.Errorf("create storage dir %s: %w", root, err)
@@ -74,6 +80,21 @@ func (l *local) Open(_ context.Context, key string) (io.ReadCloser, int64, error
return f, info.Size(), nil
}
func (l *local) Stat(_ context.Context, key string) (int64, error) {
p, err := l.path(key)
if err != nil {
return 0, err
}
info, err := os.Stat(p)
if os.IsNotExist(err) {
return 0, ErrNotExist
}
if err != nil {
return 0, err
}
return info.Size(), nil
}
func (l *local) Delete(_ context.Context, key string) error {
p, err := l.path(key)
if err != nil {
+33
View File
@@ -2,14 +2,19 @@ package storage
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
)
@@ -23,6 +28,12 @@ type s3Store struct {
bucket string
}
// NewS3 builds the S3 backend explicitly, whichever backend is active —
// `photo-api sync` needs both sides at once.
func NewS3(cfg config.Config) (Storage, error) {
return newS3(cfg)
}
func newS3(cfg config.Config) (*s3Store, error) {
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
awsconfig.WithRegion(cfg.S3Region),
@@ -65,6 +76,28 @@ func (s *s3Store) Open(ctx context.Context, key string) (io.ReadCloser, int64, e
return out.Body, aws.ToInt64(out.ContentLength), nil
}
func (s *s3Store) Stat(ctx context.Context, key string) (int64, error) {
out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(key),
})
if err != nil {
// Compatibles differ: some answer NotFound, some NoSuchKey, and a
// HEAD carries no body to parse, so fall back to the status code.
var notFound *types.NotFound
var noKey *types.NoSuchKey
var apiErr smithy.APIError
var respErr *awshttp.ResponseError
if errors.As(err, &notFound) || errors.As(err, &noKey) ||
(errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey")) ||
(errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusNotFound) {
return 0, ErrNotExist
}
return 0, err
}
return aws.ToInt64(out.ContentLength), nil
}
func (s *s3Store) Delete(ctx context.Context, key string) error {
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
+17 -2
View File
@@ -7,6 +7,7 @@ package storage
import (
"context"
"errors"
"fmt"
"io"
"time"
@@ -17,16 +18,30 @@ import (
// callers then stream the object through the API instead.
var ErrNoPresign = errors.New("presigned URLs not supported")
// ErrNotExist is what Stat reports for a missing object on either backend.
var ErrNotExist = errors.New("object does not exist")
type Storage interface {
Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error
Open(ctx context.Context, key string) (io.ReadCloser, int64, error)
// Stat returns the object's size without reading it, or ErrNotExist.
Stat(ctx context.Context, key string) (int64, error)
Delete(ctx context.Context, key string) error
PresignGet(ctx context.Context, key, downloadFilename, contentType string, expiry time.Duration) (string, error)
}
// New builds the backend that serves requests (see config.S3Enabled).
func New(cfg config.Config) (Storage, error) {
if cfg.S3Enabled() {
return newS3(cfg)
return NewS3(cfg)
}
return newLocal(cfg.StoragePath)
return NewLocal(cfg.StoragePath)
}
// Name describes a backend for log lines.
func Name(cfg config.Config, s Storage) string {
if _, ok := s.(*s3Store); ok {
return fmt.Sprintf("S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint)
}
return fmt.Sprintf("local disk at %s", cfg.StoragePath)
}