From 1d2e00d26fc41607186dde67be2cbc021334012e Mon Sep 17 00:00:00 2001 From: Claudiu Schuster Date: Thu, 27 Aug 2026 19:15:32 +0200 Subject: [PATCH 1/2] protondrive: drain block workers after upload errors Receive every block upload result before returning the first error so all workers can release their semaphore slots. Buffer the result channel and return immediately when slot acquisition fails. Add a regression test which repeats failing batches and then acquires the full semaphore capacity. --- file_upload.go | 23 +++++++++------- file_upload_concurrency_test.go | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 file_upload_concurrency_test.go diff --git a/file_upload.go b/file_upload.go index be3d4d3..2c4cf9b 100644 --- a/file_upload.go +++ b/file_upload.go @@ -19,6 +19,16 @@ import ( "github.com/rclone/go-proton-api" ) +func collectUploadErrors(errChan <-chan error, count int) error { + var firstErr error + for range count { + if err := <-errChan; err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + func (protonDrive *ProtonDrive) handleRevisionConflict(ctx context.Context, link *proton.Link, createFileResp *proton.CreateFileRes) (string, bool, error) { if link != nil { linkID := link.LinkID @@ -292,15 +302,13 @@ func (protonDrive *ProtonDrive) uploadAndCollectBlockData(ctx context.Context, n return err } - errChan := make(chan error) + errChan := make(chan error, len(blockUploadResp)) uploadBlockWrapper := func(ctx context.Context, errChan chan error, bareURL, token string, block io.Reader) { - // log.Println("Before semaphore") if err := protonDrive.blockUploadSemaphore.Acquire(ctx, 1); err != nil { errChan <- err + return } defer protonDrive.blockUploadSemaphore.Release(1) - // log.Println("After semaphore") - // defer log.Println("Release semaphore") errChan <- protonDrive.c.UploadBlock(ctx, bareURL, token, block) } @@ -308,11 +316,8 @@ func (protonDrive *ProtonDrive) uploadAndCollectBlockData(ctx context.Context, n go uploadBlockWrapper(ctx, errChan, blockUploadResp[i].BareURL, blockUploadResp[i].Token, bytes.NewReader(pendingUploadBlocks[i].encData)) } - for i := 0; i < len(blockUploadResp); i++ { - err := <-errChan - if err != nil { - return err - } + if err := collectUploadErrors(errChan, len(blockUploadResp)); err != nil { + return err } pendingUploadBlocks = pendingUploadBlocks[:0] diff --git a/file_upload_concurrency_test.go b/file_upload_concurrency_test.go new file mode 100644 index 0000000..598c1ea --- /dev/null +++ b/file_upload_concurrency_test.go @@ -0,0 +1,48 @@ +package proton_api_bridge + +import ( + "context" + "errors" + "testing" + "time" + + "golang.org/x/sync/semaphore" +) + +func TestCollectUploadErrorsReleasesAllWorkersAfterFailure(t *testing.T) { + const ( + batchSize = int64(8) + slotCount = int64(20) + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + slots := semaphore.NewWeighted(slotCount) + + for batch := 0; batch < 4; batch++ { + results := make(chan error) + for block := int64(0); block < batchSize; block++ { + go func(fail bool) { + if err := slots.Acquire(ctx, 1); err != nil { + results <- err + return + } + defer slots.Release(1) + if fail { + results <- errors.New("synthetic upload failure") + return + } + results <- nil + }(block == 0) + } + + if err := collectUploadErrors(results, int(batchSize)); err == nil { + t.Fatal("expected the first upload failure to be returned") + } + } + + if err := slots.Acquire(ctx, slotCount); err != nil { + t.Fatalf("upload workers leaked semaphore slots: %v", err) + } + slots.Release(slotCount) +} From c059d7573afbc67d9a78e44db9b6aaac384c4263 Mon Sep 17 00:00:00 2001 From: Claudiu Schuster Date: Fri, 28 Aug 2026 09:16:15 +0200 Subject: [PATCH 2/2] protondrive: retry transient block upload failures Retry only failed encrypted blocks with fresh upload links and bounded context-aware backoff. Preserve successful blocks and return terminal or exhausted errors without replaying the complete file stream. Refs oss-singularity/proton-drive-linux#42 --- file_upload.go | 209 ++++++++++++++++++++++++------ file_upload_concurrency_test.go | 221 +++++++++++++++++++++++++++++--- 2 files changed, 374 insertions(+), 56 deletions(-) diff --git a/file_upload.go b/file_upload.go index 2c4cf9b..938ca17 100644 --- a/file_upload.go +++ b/file_upload.go @@ -8,6 +8,8 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" + "fmt" "io" "mime" "os" @@ -19,14 +21,154 @@ import ( "github.com/rclone/go-proton-api" ) -func collectUploadErrors(errChan <-chan error, count int) error { - var firstErr error - for range count { - if err := <-errChan; err != nil && firstErr == nil { - firstErr = err +const ( + blockUploadMaxAttempts = 5 + blockUploadRetryBaseDelay = time.Second + blockUploadRetryMaxDelay = 15 * time.Second +) + +type pendingUploadBlock struct { + blockUploadInfo proton.BlockUploadInfo + encData []byte +} + +type blockUploadResult struct { + index int + err error +} + +type blockUploadRetryLogger interface { + Warnf(format string, v ...interface{}) +} + +func retryableBlockUploadError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + + var apiErr *proton.APIError + if errors.As(err, &apiErr) { + return apiErr.Status >= 500 && apiErr.Status <= 599 + } + + var protonNetErr *proton.NetError + return errors.As(err, &protonNetErr) +} + +func blockUploadRetryDelay(failedAttempt int) time.Duration { + delay := blockUploadRetryBaseDelay + for i := 1; i < failedAttempt && delay < blockUploadRetryMaxDelay; i++ { + delay *= 2 + } + if delay > blockUploadRetryMaxDelay { + return blockUploadRetryMaxDelay + } + return delay +} + +func waitForBlockUploadRetry(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func uploadBlockBatchWithRetry( + ctx context.Context, + blocks []pendingUploadBlock, + maxAttempts int, + requestLinks func(context.Context, []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error), + uploadBlock func(context.Context, proton.BlockUploadLink, []byte) error, + wait func(context.Context, time.Duration) error, + logger blockUploadRetryLogger, +) error { + remaining := append([]pendingUploadBlock(nil), blocks...) + var lastErr error + + for attempt := 1; attempt <= maxAttempts; attempt++ { + blockList := make([]proton.BlockUploadInfo, len(remaining)) + for i := range remaining { + blockList[i] = remaining[i].blockUploadInfo + } + + links, err := requestLinks(ctx, blockList) + if err != nil { + lastErr = err + if !retryableBlockUploadError(err) || attempt == maxAttempts { + return err + } + } else { + if len(links) != len(remaining) { + return fmt.Errorf( + "requested %d Proton block upload links, received %d", + len(remaining), + len(links), + ) + } + + results := make(chan blockUploadResult, len(remaining)) + for i := range remaining { + go func(index int) { + results <- blockUploadResult{ + index: index, + err: uploadBlock(ctx, links[index], remaining[index].encData), + } + }(i) + } + + errorsByIndex := make([]error, len(remaining)) + for range remaining { + result := <-results + errorsByIndex[result.index] = result.err + } + + failed := make([]pendingUploadBlock, 0, len(remaining)) + var terminalErr error + lastErr = nil + for i, uploadErr := range errorsByIndex { + if uploadErr == nil { + continue + } + if !retryableBlockUploadError(uploadErr) && terminalErr == nil { + terminalErr = uploadErr + } + if lastErr == nil { + lastErr = uploadErr + } + failed = append(failed, remaining[i]) + } + if terminalErr != nil { + return terminalErr + } + if len(failed) == 0 { + return nil + } + if attempt == maxAttempts { + return lastErr + } + remaining = failed + } + + delay := blockUploadRetryDelay(attempt) + if logger != nil { + logger.Warnf( + "Retrying %d transient Proton block upload(s) after %s (attempt %d/%d)", + len(remaining), + delay, + attempt+1, + maxAttempts, + ) + } + if err := wait(ctx, delay); err != nil { + return err } } - return firstErr + + return lastErr } func (protonDrive *ProtonDrive) handleRevisionConflict(ctx context.Context, link *proton.Link, createFileResp *proton.CreateFileRes) (string, bool, error) { @@ -267,56 +409,45 @@ func (protonDrive *ProtonDrive) createFileUploadDraft(ctx context.Context, paren } func (protonDrive *ProtonDrive) uploadAndCollectBlockData(ctx context.Context, newSessionKey *crypto.SessionKey, newNodeKR *crypto.KeyRing, file io.Reader, linkID, revisionID string) ([]byte, int64, []int64, string, error) { - type PendingUploadBlocks struct { - blockUploadInfo proton.BlockUploadInfo - encData []byte - } - if newSessionKey == nil || newNodeKR == nil { return nil, 0, nil, "", ErrMissingInputUploadAndCollectBlockData } totalFileSize := int64(0) - pendingUploadBlocks := make([]PendingUploadBlocks, 0) + pendingUploadBlocks := make([]pendingUploadBlock, 0) manifestSignatureData := make([]byte, 0) uploadPendingBlocks := func() error { if len(pendingUploadBlocks) == 0 { return nil } - blockList := make([]proton.BlockUploadInfo, 0) - for i := range pendingUploadBlocks { - blockList = append(blockList, pendingUploadBlocks[i].blockUploadInfo) - } - blockUploadReq := proton.BlockUploadReq{ - AddressID: protonDrive.MainShare.AddressID, - ShareID: protonDrive.MainShare.ShareID, - LinkID: linkID, - RevisionID: revisionID, - - BlockList: blockList, + requestLinks := func(ctx context.Context, blockList []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + return protonDrive.c.RequestBlockUpload(ctx, proton.BlockUploadReq{ + AddressID: protonDrive.MainShare.AddressID, + ShareID: protonDrive.MainShare.ShareID, + LinkID: linkID, + RevisionID: revisionID, + BlockList: blockList, + }) } - blockUploadResp, err := protonDrive.c.RequestBlockUpload(ctx, blockUploadReq) - if err != nil { - return err - } - - errChan := make(chan error, len(blockUploadResp)) - uploadBlockWrapper := func(ctx context.Context, errChan chan error, bareURL, token string, block io.Reader) { + uploadBlock := func(ctx context.Context, link proton.BlockUploadLink, block []byte) error { if err := protonDrive.blockUploadSemaphore.Acquire(ctx, 1); err != nil { - errChan <- err - return + return err } defer protonDrive.blockUploadSemaphore.Release(1) - errChan <- protonDrive.c.UploadBlock(ctx, bareURL, token, block) - } - for i := range blockUploadResp { - go uploadBlockWrapper(ctx, errChan, blockUploadResp[i].BareURL, blockUploadResp[i].Token, bytes.NewReader(pendingUploadBlocks[i].encData)) + return protonDrive.c.UploadBlock(ctx, link.BareURL, link.Token, bytes.NewReader(block)) } - - if err := collectUploadErrors(errChan, len(blockUploadResp)); err != nil { + if err := uploadBlockBatchWithRetry( + ctx, + pendingUploadBlocks, + blockUploadMaxAttempts, + requestLinks, + uploadBlock, + waitForBlockUploadRetry, + protonDrive.Config.GetLogger(), + ); err != nil { return err } @@ -410,7 +541,7 @@ func (protonDrive *ProtonDrive) uploadAndCollectBlockData(ctx context.Context, n } manifestSignatureData = append(manifestSignatureData, hash...) - pendingUploadBlocks = append(pendingUploadBlocks, PendingUploadBlocks{ + pendingUploadBlocks = append(pendingUploadBlocks, pendingUploadBlock{ blockUploadInfo: proton.BlockUploadInfo{ Index: i, // iOS drive: BE starts with 1 Size: int64(len(encData)), diff --git a/file_upload_concurrency_test.go b/file_upload_concurrency_test.go index 598c1ea..06d3677 100644 --- a/file_upload_concurrency_test.go +++ b/file_upload_concurrency_test.go @@ -3,40 +3,227 @@ package proton_api_bridge import ( "context" "errors" + "reflect" + "sync" "testing" "time" + "github.com/rclone/go-proton-api" "golang.org/x/sync/semaphore" ) -func TestCollectUploadErrorsReleasesAllWorkersAfterFailure(t *testing.T) { - const ( - batchSize = int64(8) - slotCount = int64(20) +func testPendingBlocks(indexes ...int) []pendingUploadBlock { + blocks := make([]pendingUploadBlock, len(indexes)) + for i, index := range indexes { + blocks[i] = pendingUploadBlock{ + blockUploadInfo: proton.BlockUploadInfo{Index: index}, + encData: []byte{byte(index)}, + } + } + return blocks +} + +func testUploadLinks(blocks []proton.BlockUploadInfo) []proton.BlockUploadLink { + links := make([]proton.BlockUploadLink, len(blocks)) + for i, block := range blocks { + links[i] = proton.BlockUploadLink{Token: string(rune(block.Index))} + } + return links +} + +func noRetryWait(_ context.Context, _ time.Duration) error { return nil } + +func TestUploadBlockBatchRetriesOnlyTransientFailures(t *testing.T) { + var requestIndexes [][]int + requestLinks := func(_ context.Context, blocks []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + indexes := make([]int, len(blocks)) + for i, block := range blocks { + indexes[i] = block.Index + } + requestIndexes = append(requestIndexes, indexes) + return testUploadLinks(blocks), nil + } + + var mu sync.Mutex + uploads := map[int]int{} + uploadBlock := func(_ context.Context, _ proton.BlockUploadLink, block []byte) error { + index := int(block[0]) + mu.Lock() + uploads[index]++ + attempt := uploads[index] + mu.Unlock() + if attempt == 1 && index != 2 { + return &proton.APIError{Status: 502, Message: "temporary storage failure"} + } + return nil + } + + err := uploadBlockBatchWithRetry( + context.Background(), + testPendingBlocks(1, 2, 3), + blockUploadMaxAttempts, + requestLinks, + uploadBlock, + noRetryWait, + nil, + ) + if err != nil { + t.Fatalf("retrying transient block uploads failed: %v", err) + } + if want := [][]int{{1, 2, 3}, {1, 3}}; !reflect.DeepEqual(requestIndexes, want) { + t.Fatalf("requested block indexes %v, want %v", requestIndexes, want) + } + if want := map[int]int{1: 2, 2: 1, 3: 2}; !reflect.DeepEqual(uploads, want) { + t.Fatalf("block upload counts %v, want %v", uploads, want) + } +} + +func TestUploadBlockBatchReturnsNonRetryableError(t *testing.T) { + terminalErr := &proton.APIError{Status: 422, Message: "draft conflict"} + requests := 0 + err := uploadBlockBatchWithRetry( + context.Background(), + testPendingBlocks(1), + blockUploadMaxAttempts, + func(_ context.Context, blocks []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + requests++ + return testUploadLinks(blocks), nil + }, + func(_ context.Context, _ proton.BlockUploadLink, _ []byte) error { return terminalErr }, + noRetryWait, + nil, + ) + if !errors.Is(err, terminalErr) { + t.Fatalf("returned error %v, want %v", err, terminalErr) + } + if requests != 1 { + t.Fatalf("requested upload links %d times after a terminal error, want 1", requests) + } +} + +func TestUploadBlockBatchRetriesTransientLinkRequest(t *testing.T) { + transientErr := &proton.APIError{Status: 502, Message: "temporary API failure"} + requests := 0 + uploads := 0 + err := uploadBlockBatchWithRetry( + context.Background(), + testPendingBlocks(1), + blockUploadMaxAttempts, + func(_ context.Context, blocks []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + requests++ + if requests == 1 { + return nil, transientErr + } + return testUploadLinks(blocks), nil + }, + func(_ context.Context, _ proton.BlockUploadLink, _ []byte) error { + uploads++ + return nil + }, + noRetryWait, + nil, + ) + if err != nil { + t.Fatalf("retrying a transient link request failed: %v", err) + } + if requests != 2 || uploads != 1 { + t.Fatalf("observed %d link requests and %d uploads, want 2 and 1", requests, uploads) + } +} + +func TestUploadBlockBatchRejectsMismatchedLinkCount(t *testing.T) { + err := uploadBlockBatchWithRetry( + context.Background(), + testPendingBlocks(1, 2), + blockUploadMaxAttempts, + func(_ context.Context, _ []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + return []proton.BlockUploadLink{{}}, nil + }, + func(_ context.Context, _ proton.BlockUploadLink, _ []byte) error { + t.Fatal("upload must not start with a mismatched link response") + return nil + }, + noRetryWait, + nil, + ) + if err == nil { + t.Fatal("expected a mismatched link count to fail") + } +} + +func TestUploadBlockBatchReturnsLastErrorAfterLimit(t *testing.T) { + transientErr := &proton.APIError{Status: 502, Message: "temporary storage failure"} + requests := 0 + err := uploadBlockBatchWithRetry( + context.Background(), + testPendingBlocks(1), + 3, + func(_ context.Context, blocks []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + requests++ + return testUploadLinks(blocks), nil + }, + func(_ context.Context, _ proton.BlockUploadLink, _ []byte) error { return transientErr }, + noRetryWait, + nil, ) + if !errors.Is(err, transientErr) { + t.Fatalf("returned error %v, want %v", err, transientErr) + } + if requests != 3 { + t.Fatalf("requested upload links %d times, want 3", requests) + } +} + +func TestUploadBlockBatchHonorsCancellationDuringBackoff(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := uploadBlockBatchWithRetry( + ctx, + testPendingBlocks(1), + blockUploadMaxAttempts, + func(_ context.Context, blocks []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + return testUploadLinks(blocks), nil + }, + func(_ context.Context, _ proton.BlockUploadLink, _ []byte) error { + return &proton.APIError{Status: 502, Message: "temporary storage failure"} + }, + waitForBlockUploadRetry, + nil, + ) + if !errors.Is(err, context.Canceled) { + t.Fatalf("returned error %v, want context cancellation", err) + } +} + +func TestUploadBlockBatchReleasesAllWorkersAfterFailure(t *testing.T) { + const slotCount = int64(20) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() slots := semaphore.NewWeighted(slotCount) for batch := 0; batch < 4; batch++ { - results := make(chan error) - for block := int64(0); block < batchSize; block++ { - go func(fail bool) { + err := uploadBlockBatchWithRetry( + ctx, + testPendingBlocks(1, 2, 3, 4, 5, 6, 7, 8), + blockUploadMaxAttempts, + func(_ context.Context, blocks []proton.BlockUploadInfo) ([]proton.BlockUploadLink, error) { + return testUploadLinks(blocks), nil + }, + func(_ context.Context, _ proton.BlockUploadLink, block []byte) error { if err := slots.Acquire(ctx, 1); err != nil { - results <- err - return + return err } defer slots.Release(1) - if fail { - results <- errors.New("synthetic upload failure") - return + if block[0] == 1 { + return errors.New("synthetic upload failure") } - results <- nil - }(block == 0) - } - - if err := collectUploadErrors(results, int(batchSize)); err == nil { + return nil + }, + noRetryWait, + nil, + ) + if err == nil { t.Fatal("expected the first upload failure to be returned") } }