Add photo content-hash dedup and local↔S3 library sync.
Uploads skip per-gallery duplicates, checksums can be backfilled, and STORAGE_BACKEND plus sync tooling make switching storage backends safe.
This commit is contained in:
@@ -17,6 +17,12 @@ type local struct {
|
||||
root string
|
||||
}
|
||||
|
||||
// NewLocal builds the local-disk backend explicitly, whichever backend is
|
||||
// active — `photo-api sync` needs both sides at once.
|
||||
func NewLocal(root string) (Storage, error) {
|
||||
return newLocal(root)
|
||||
}
|
||||
|
||||
func newLocal(root string) (*local, error) {
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create storage dir %s: %w", root, err)
|
||||
@@ -74,6 +80,21 @@ func (l *local) Open(_ context.Context, key string) (io.ReadCloser, int64, error
|
||||
return f, info.Size(), nil
|
||||
}
|
||||
|
||||
func (l *local) Stat(_ context.Context, key string) (int64, error) {
|
||||
p, err := l.path(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
info, err := os.Stat(p)
|
||||
if os.IsNotExist(err) {
|
||||
return 0, ErrNotExist
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), nil
|
||||
}
|
||||
|
||||
func (l *local) Delete(_ context.Context, key string) error {
|
||||
p, err := l.path(key)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,14 +2,19 @@ package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
"git.azzamo.net/Michilis/Spanglish/photo-api/internal/config"
|
||||
)
|
||||
@@ -23,6 +28,12 @@ type s3Store struct {
|
||||
bucket string
|
||||
}
|
||||
|
||||
// NewS3 builds the S3 backend explicitly, whichever backend is active —
|
||||
// `photo-api sync` needs both sides at once.
|
||||
func NewS3(cfg config.Config) (Storage, error) {
|
||||
return newS3(cfg)
|
||||
}
|
||||
|
||||
func newS3(cfg config.Config) (*s3Store, error) {
|
||||
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
|
||||
awsconfig.WithRegion(cfg.S3Region),
|
||||
@@ -65,6 +76,28 @@ func (s *s3Store) Open(ctx context.Context, key string) (io.ReadCloser, int64, e
|
||||
return out.Body, aws.ToInt64(out.ContentLength), nil
|
||||
}
|
||||
|
||||
func (s *s3Store) Stat(ctx context.Context, key string) (int64, error) {
|
||||
out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
// Compatibles differ: some answer NotFound, some NoSuchKey, and a
|
||||
// HEAD carries no body to parse, so fall back to the status code.
|
||||
var notFound *types.NotFound
|
||||
var noKey *types.NoSuchKey
|
||||
var apiErr smithy.APIError
|
||||
var respErr *awshttp.ResponseError
|
||||
if errors.As(err, ¬Found) || errors.As(err, &noKey) ||
|
||||
(errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey")) ||
|
||||
(errors.As(err, &respErr) && respErr.HTTPStatusCode() == http.StatusNotFound) {
|
||||
return 0, ErrNotExist
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
return aws.ToInt64(out.ContentLength), nil
|
||||
}
|
||||
|
||||
func (s *s3Store) Delete(ctx context.Context, key string) error {
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
|
||||
@@ -7,6 +7,7 @@ package storage
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
@@ -17,16 +18,30 @@ import (
|
||||
// callers then stream the object through the API instead.
|
||||
var ErrNoPresign = errors.New("presigned URLs not supported")
|
||||
|
||||
// ErrNotExist is what Stat reports for a missing object on either backend.
|
||||
var ErrNotExist = errors.New("object does not exist")
|
||||
|
||||
type Storage interface {
|
||||
Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error
|
||||
Open(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
// Stat returns the object's size without reading it, or ErrNotExist.
|
||||
Stat(ctx context.Context, key string) (int64, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
PresignGet(ctx context.Context, key, downloadFilename, contentType string, expiry time.Duration) (string, error)
|
||||
}
|
||||
|
||||
// New builds the backend that serves requests (see config.S3Enabled).
|
||||
func New(cfg config.Config) (Storage, error) {
|
||||
if cfg.S3Enabled() {
|
||||
return newS3(cfg)
|
||||
return NewS3(cfg)
|
||||
}
|
||||
return newLocal(cfg.StoragePath)
|
||||
return NewLocal(cfg.StoragePath)
|
||||
}
|
||||
|
||||
// Name describes a backend for log lines.
|
||||
func Name(cfg config.Config, s Storage) string {
|
||||
if _, ok := s.(*s3Store); ok {
|
||||
return fmt.Sprintf("S3 bucket %s at %s", cfg.S3Bucket, cfg.S3Endpoint)
|
||||
}
|
||||
return fmt.Sprintf("local disk at %s", cfg.StoragePath)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user