UPDATE: try fix move s3
All checks were successful
Build and Release / release (push) Successful in 1m2s
All checks were successful
Build and Release / release (push) Successful in 1m2s
This commit is contained in:
@@ -104,7 +104,7 @@ func (s *s3Storage) GetMainBucket() string { return s.bucket }
|
|||||||
func (s *s3Storage) GetTempBucket() string { return s.tempBucket }
|
func (s *s3Storage) GetTempBucket() string { return s.tempBucket }
|
||||||
|
|
||||||
func (s *s3Storage) Move(ctx context.Context, src *MoveOptions, dest *MoveOptions) error {
|
func (s *s3Storage) Move(ctx context.Context, src *MoveOptions, dest *MoveOptions) error {
|
||||||
copySource := url.PathEscape(fmt.Sprintf("%s/%s", src.Bucket, src.Key))
|
copySource := fmt.Sprintf("%s/%s", src.Bucket, url.PathEscape(src.Key))
|
||||||
|
|
||||||
_, err := s.client.CopyObject(ctx, &s3.CopyObjectInput{
|
_, err := s.client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||||
Bucket: aws.String(dest.Bucket),
|
Bucket: aws.String(dest.Bucket),
|
||||||
@@ -115,6 +115,15 @@ func (s *s3Storage) Move(ctx context.Context, src *MoveOptions, dest *MoveOption
|
|||||||
return fmt.Errorf("failed to copy object: %w", err)
|
return fmt.Errorf("failed to copy object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
waiter := s3.NewObjectExistsWaiter(s.client)
|
||||||
|
err = waiter.Wait(ctx, &s3.HeadObjectInput{
|
||||||
|
Bucket: aws.String(dest.Bucket),
|
||||||
|
Key: aws.String(dest.Key),
|
||||||
|
}, time.Second*10)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("object not available after copy: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||||
Bucket: aws.String(src.Bucket),
|
Bucket: aws.String(src.Bucket),
|
||||||
Key: aws.String(src.Key),
|
Key: aws.String(src.Key),
|
||||||
@@ -190,39 +199,39 @@ func (s *s3Storage) Delete(ctx context.Context, key string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *s3Storage) BulkDelete(ctx context.Context, keys []string) error {
|
func (s *s3Storage) BulkDelete(ctx context.Context, keys []string) error {
|
||||||
if len(keys) == 0 {
|
if len(keys) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
batchSize := 1000
|
batchSize := 1000
|
||||||
var hasError bool
|
var hasError bool
|
||||||
|
|
||||||
for i := 0; i < len(keys); i += batchSize {
|
for i := 0; i < len(keys); i += batchSize {
|
||||||
end := i + batchSize
|
end := i + batchSize
|
||||||
if end > len(keys) {
|
if end > len(keys) {
|
||||||
end = len(keys)
|
end = len(keys)
|
||||||
}
|
}
|
||||||
|
|
||||||
batch := keys[i:end]
|
batch := keys[i:end]
|
||||||
var objects []types.ObjectIdentifier
|
var objects []types.ObjectIdentifier
|
||||||
for _, k := range batch {
|
for _, k := range batch {
|
||||||
objects = append(objects, types.ObjectIdentifier{Key: aws.String(k)})
|
objects = append(objects, types.ObjectIdentifier{Key: aws.String(k)})
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
_, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||||
Bucket: aws.String(s.bucket),
|
Bucket: aws.String(s.bucket),
|
||||||
Delete: &types.Delete{Objects: objects},
|
Delete: &types.Delete{Objects: objects},
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Int("start", i).Int("end", end).Msg("S3 batch delete failed")
|
log.Error().Err(err).Int("start", i).Int("end", end).Msg("S3 batch delete failed")
|
||||||
hasError = true
|
hasError = true
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if hasError {
|
if hasError {
|
||||||
return fmt.Errorf("one or more batches failed to delete")
|
return fmt.Errorf("one or more batches failed to delete")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user