From 035a9718c0e52ea2371a30a99c98740b54ff3d78 Mon Sep 17 00:00:00 2001 From: AzenKain Date: Thu, 9 Apr 2026 15:29:57 +0700 Subject: [PATCH] UPDATE: try fix move s3 --- pkg/storage/s3.go | 71 ++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index 0ea1cad..ab751d1 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -104,7 +104,7 @@ func (s *s3Storage) GetMainBucket() string { return s.bucket } func (s *s3Storage) GetTempBucket() string { return s.tempBucket } func (s *s3Storage) Move(ctx context.Context, src *MoveOptions, dest *MoveOptions) error { - copySource := url.PathEscape(fmt.Sprintf("%s/%s", src.Bucket, src.Key)) + copySource := fmt.Sprintf("%s/%s", src.Bucket, url.PathEscape(src.Key)) _, err := s.client.CopyObject(ctx, &s3.CopyObjectInput{ Bucket: aws.String(dest.Bucket), @@ -115,6 +115,15 @@ func (s *s3Storage) Move(ctx context.Context, src *MoveOptions, dest *MoveOption return fmt.Errorf("failed to copy object: %w", err) } + waiter := s3.NewObjectExistsWaiter(s.client) + err = waiter.Wait(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(dest.Bucket), + Key: aws.String(dest.Key), + }, time.Second*10) + if err != nil { + return fmt.Errorf("object not available after copy: %w", err) + } + _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(src.Bucket), Key: aws.String(src.Key), @@ -190,39 +199,39 @@ func (s *s3Storage) Delete(ctx context.Context, key string) error { } func (s *s3Storage) BulkDelete(ctx context.Context, keys []string) error { - if len(keys) == 0 { - return nil - } + if len(keys) == 0 { + return nil + } - batchSize := 1000 - var hasError bool + batchSize := 1000 + var hasError bool - for i := 0; i < len(keys); i += batchSize { - end := i + batchSize - if end > len(keys) { - end = len(keys) - } + for i := 0; i < len(keys); i += batchSize { + end := i + batchSize + if end > len(keys) { + end = len(keys) + } - batch := keys[i:end] - var objects []types.ObjectIdentifier - for _, k := range batch { - objects = append(objects, types.ObjectIdentifier{Key: aws.String(k)}) - } + batch := keys[i:end] + var objects []types.ObjectIdentifier + for _, k := range batch { + objects = append(objects, types.ObjectIdentifier{Key: aws.String(k)}) + } - _, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ - Bucket: aws.String(s.bucket), - Delete: &types.Delete{Objects: objects}, - }) + _, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(s.bucket), + Delete: &types.Delete{Objects: objects}, + }) - if err != nil { - log.Error().Err(err).Int("start", i).Int("end", end).Msg("S3 batch delete failed") - hasError = true - continue - } - } + if err != nil { + log.Error().Err(err).Int("start", i).Int("end", end).Msg("S3 batch delete failed") + hasError = true + continue + } + } - if hasError { - return fmt.Errorf("one or more batches failed to delete") - } - return nil -} \ No newline at end of file + if hasError { + return fmt.Errorf("one or more batches failed to delete") + } + return nil +}