feature: multipart uploads and retry

docs/game/ship.md

@@ -45,6 +45,25 @@ shipthis game ship --platform ios --follow --useDemoCredentials --download game.
 shipthis game ship --platform android --follow --useDemoCredentials --downloadAPK game.apk
 ```
 
+### Uploading a large game
+
+ShipThis makes a zip of your game. For a zip of 16MB or more, ShipThis sends the zip in
+several parts at the same time. This is faster than one request.
+
+Each part is separate. If the network fails, ShipThis sends that part again. The parts that
+arrived stay on the server.
+
+ShipThis sends a zip smaller than 16MB in one request. Parts do not make a small zip faster.
+
+To send the zip in one request, use `--skipMultipart`. This method is slower, and the zip
+must be smaller than 5GB. Use this flag only if the upload in parts fails.
+
+```bash
+shipthis game ship --platform android --skipMultipart
+```
+
+To see each part, and to see ShipThis send a part again, add `--verbose`.
+
 ### Overriding the Godot version
 
 You can specify a different Godot version to use only for the current job. This can be helpful if you are upgrading your game to use a newer version of Godot.
@@ -58,7 +77,8 @@ shipthis game ship --platform android --follow --gameEngineVersion 4.5.1 --downl
 ```help
 USAGE
   $ shipthis game ship [-g <value>] [--download <value> --platform android|ios] [--downloadAPK <value> ]
-    [--follow ] [--skipPublish] [--verbose] [--useDemoCredentials ] [--gameEngineVersion <value>] [--dryRun]
+    [--follow ] [--skipMultipart] [--skipPublish] [--verbose] [--useDemoCredentials ]
+    [--gameEngineVersion <value>] [--dryRun]
 
 FLAGS
   -g, --gameId=<value>             The ID of the game
@@ -70,6 +90,8 @@ FLAGS
       --gameEngineVersion=<value>  Override the specified game engine version for this build
       --platform=<option>          The platform to ship the game to. This can be "android" or "ios"
                                    <options: android|ios>
+      --skipMultipart              Upload the zip in one request instead of several parts in parallel (slower, and
+                                   limited to 5GB)
       --skipPublish                Skip the publish step
       --useDemoCredentials         Use demo credentials for this build (requires --platform, implies --skipPublish)
       --verbose                    Enable verbose logging

package.json

@@ -34,6 +34,7 @@
     "marked-terminal": "^6.2.0",
     "node-forge": "^1.3.1",
     "open": "^10.1.0",
+    "p-limit": "^6.2.0",
     "prompts": "^2.4.2",
     "qrcode": "^1.5.4",
     "react": "^18.3.1",

src/api/index.ts

@@ -13,6 +13,8 @@ import {
   GoogleAuthResponse,
   GoogleStatusResponse,
   Job,
+  MultipartPartUrl,
+  MultipartUploadTicket,
   PageAndSortParams,
   Platform,
   Project,
@@ -21,6 +23,7 @@ import {
   Self,
   TermsResponse,
   UploadDetails,
+  UploadedPart,
   UploadTicket,
 } from '@cli/types'
 import {castArrayObjectDates, castJobDates, castObjectDates} from '@cli/utils/dates.js'
@@ -109,6 +112,40 @@ export async function getNewUploadTicket(projectId: string): Promise<UploadTicke
   return data as UploadTicket
 }
 
+// Starts a multipart upload. The size decides the part size the backend returns.
+export async function getNewMultipartUpload(projectId: string, size: number): Promise<MultipartUploadTicket> {
+  const headers = getAuthedHeaders()
+  const opt = {headers}
+  const {data} = await axios.post(`${API_URL}/upload/${projectId}/multipart`, {size}, opt)
+  return data as MultipartUploadTicket
+}
+
+// Returns a signed URL for each part number. A signed URL lasts one hour, so a
+// slow upload asks for a new URL rather than starting again.
+export async function getMultipartPartUrls(
+  uploadTicketId: string,
+  partNumbers: number[],
+): Promise<MultipartPartUrl[]> {
+  const headers = getAuthedHeaders()
+  const opt = {headers}
+  const {data} = await axios.post(`${API_URL}/upload/multipart/${uploadTicketId}/parts`, {partNumbers}, opt)
+  return (data as {parts: MultipartPartUrl[]}).parts
+}
+
+// Joins the uploaded parts into the final object
+export async function completeMultipartUpload(uploadTicketId: string, parts: UploadedPart[]): Promise<void> {
+  const headers = getAuthedHeaders()
+  const opt = {headers}
+  await axios.post(`${API_URL}/upload/multipart/${uploadTicketId}/complete`, {parts}, opt)
+}
+
+// Cancels a multipart upload. This deletes the parts already uploaded.
+export async function abortMultipartUpload(uploadTicketId: string): Promise<void> {
+  const headers = getAuthedHeaders()
+  const opt = {headers}
+  await axios.post(`${API_URL}/upload/multipart/${uploadTicketId}/abort`, {}, opt)
+}
+
 // Tells the backend to start running the jobs for an upload-ticket
 type StartJobsOptions = {
   platform?: Platform

src/commands/game/ship.tsx

@@ -46,6 +46,11 @@ export default class GameShip extends BaseGameCommand<typeof GameShip> {
       options: ['android', 'ios'],
       required: false,
     }),
+    skipMultipart: Flags.boolean({
+      default: false,
+      description: 'Upload the zip in one request instead of several parts in parallel (slower, and limited to 5GB)',
+      required: false,
+    }),
     skipPublish: Flags.boolean({
       default: false,
       description: 'Skip the publish step',

src/types/api.ts

@@ -73,6 +73,24 @@ export interface UploadTicket {
   url: string
 }
 
+// A MultipartUploadTicket is a request to upload a file in parts. The parts go
+// to the signed URLs returned by getMultipartPartUrls.
+export interface MultipartUploadTicket {
+  id: string
+  maxParts: number
+  partSize: number
+}
+
+export interface MultipartPartUrl {
+  partNumber: number
+  url: string
+}
+
+export interface UploadedPart {
+  etag: string
+  partNumber: number
+}
+
 export interface Upload {
   bucketName: string
   createdAt: DateTime

src/types/index.ts

@@ -36,6 +36,7 @@ export type ShipGameFlags = {
   downloadAPK?: string
   follow?: boolean
   platform?: 'android' | 'ios'
+  skipMultipart?: boolean
   skipPublish?: boolean
   verbose?: boolean
   useDemoCredentials?: boolean

src/utils/errors.ts

@@ -9,6 +9,50 @@ export function isNetworkError(exception: any) {
   return ['ECONNABORTED', 'ERR_NETWORK'].includes(`${exception.code}`)
 }
 
+// A 4xx means the request was wrong, so sending it again gives the same answer.
+// These two ask the client to come back later. A 403 is not here - a caller that
+// can recover from one, such as a stale signed URL, handles it itself.
+const RETRYABLE_CLIENT_STATUSES = [408, 429]
+
+// S3 uses 400 for a socket that went quiet, which is temporary. The status
+// cannot tell that apart from a request that was really wrong, so the name does.
+const RETRYABLE_S3_CODES = [
+  'InternalError',
+  'RequestTimeout',
+  'RequestTimeoutException',
+  'ServiceUnavailable',
+  'SlowDown',
+]
+
+// The two fields isRetryable reads. axios already sets `status` on what it throws.
+type RequestError = Error & {code?: string; status?: number}
+
+// Converts a failed S3 request into an error. fetch does not throw on a bad
+// status, and `400 Bad Request` on its own tells nobody anything, so the name
+// and sentence from the small XML body S3 sends go into the message.
+export async function getS3Error(response: Response, what: string) {
+  const body = await response.text().catch(() => '')
+  const code = /<Code>([^<]+)<\/Code>/.exec(body)?.[1]
+  const message = /<Message>([^<]+)<\/Message>/.exec(body)?.[1]
+  const detail = [code ?? response.statusText, message].filter(Boolean).join(' - ')
+
+  const error: RequestError = new Error(`${what} failed: ${response.status} ${detail}`)
+  error.code = code
+  error.status = response.status
+
+  return error
+}
+
+// Decides whether another attempt at a failed request is worth making
+export function isRetryable(error: unknown) {
+  const {code, status} = error as RequestError
+  // An S3 name is more exact than the status, so it answers first
+  if (code !== undefined) return RETRYABLE_S3_CODES.includes(code)
+  // No status means the request never got an answer, which is worth another try
+  if (status === undefined) return true
+  return status >= 500 || RETRYABLE_CLIENT_STATUSES.includes(status)
+}
+
 // Util to extract API error messages if present
 export function getErrorMessage(error: any) {
   try {

src/utils/ship/index.ts

@@ -2,14 +2,15 @@ import fs from 'node:fs'
 
 import {v4 as uuid} from 'uuid'
 
-import {getNewUploadTicket, getProject, startJobsFromUpload} from '@cli/api/index.js'
-import type {Job, Platform, ProjectConfig, ShipGameFlags, UploadDetails, UploadTicket} from '@cli/types'
+import {getProject, startJobsFromUpload} from '@cli/api/index.js'
+import type {Job, Platform, ProjectConfig, ShipGameFlags, UploadDetails} from '@cli/types'
 import {detectGodotVersion, getGodotVersionDrift} from '@cli/utils/godot.js'
 import {getCWDGitInfo, getFileHash} from '@cli/utils/index.js'
 
 import {getFilesToShip} from './glob.js'
+import {MULTIPART_MIN_SIZE, multipartUpload} from './multipartUpload.js'
 import type {ShipOptions} from './types.js'
-import {uploadZip} from './upload.js'
+import {MAX_SINGLE_UPLOAD_SIZE, type ProgressData, singleUpload} from './upload.js'
 import {formatProgressLog, getPlatforms} from './utils.js'
 import {createZip} from './zip.js'
 
@@ -35,6 +36,11 @@ const getMajorDriftError = (detected: string, configured: string) =>
 const getMinorDriftWarning = (detected: string, configured: string) =>
   `${getVersionMismatch(detected, configured)}\n\n` + getVersionFixHint(detected)
 
+const getTooLargeForSingleUploadError = (size: number) =>
+  `This zip is ${(size / 1000 / 1000 / 1000).toFixed(1)}GB. ` +
+  `One request can send at most ${MAX_SINGLE_UPLOAD_SIZE / 1000 / 1000 / 1000}GB.\n\n` +
+  `Remove --skipMultipart to upload it in parts.`
+
 // Main function to ship the game
 export async function ship({command, log, warnLog, shipFlags}: ShipOptions): Promise<Job[]> {
   const commandFlags = command.getFlags() as ShipGameFlags
@@ -99,27 +105,31 @@ export async function ship({command, log, warnLog, shipFlags}: ShipOptions): Pro
     },
   })
 
-  let response: any
   let zipFileMd5 = ''
-  let uploadTicket: UploadTicket | null = null
+  let uploadTicketId = ''
 
   try {
     const {size} = fs.statSync(tmpZipFile)
 
-    vlog('Requesting upload ticket...')
-    uploadTicket = await getNewUploadTicket(projectConfig.project.id)
-
     log('Uploading zip file...')
-    const zipStream = fs.createReadStream(tmpZipFile)
-
-    response = await uploadZip({
-      url: uploadTicket.url,
-      zipStream,
+    const uploadProps = {
+      filePath: tmpZipFile,
+      projectId: projectConfig.project.id,
+      vlog,
       zipSize: size,
-      onProgress: (data) => {
+      onProgress: (data: ProgressData) => {
         log(formatProgressLog('Uploading', data, 'loadedBytes', 'totalBytes', false))
       },
-    })
+    }
+
+    // A small zip goes up in one request. Splitting it buys nothing.
+    const useMultipart = size >= MULTIPART_MIN_SIZE && !finalFlags.skipMultipart
+
+    if (!useMultipart && size > MAX_SINGLE_UPLOAD_SIZE) {
+      throw new Error(getTooLargeForSingleUploadError(size))
+    }
+
+    uploadTicketId = useMultipart ? await multipartUpload(uploadProps) : await singleUpload(uploadProps)
 
     vlog('Computing zip file hash...')
     zipFileMd5 = await getFileHash(tmpZipFile)
@@ -136,10 +146,6 @@ export async function ship({command, log, warnLog, shipFlags}: ShipOptions): Pro
     }
   }
 
-  if (!response.ok) {
-    throw new Error(`Upload failed: ${response.status} ${response.statusText}`)
-  }
-
   log(`Upload complete`)
 
   vlog('Fetching Git info...')
@@ -160,7 +166,7 @@ export async function ship({command, log, warnLog, shipFlags}: ShipOptions): Pro
     gameEngineVersion: finalFlags.gameEngineVersion,
   }
 
-  const jobs = await startJobsFromUpload(uploadTicket.id, startJobsOptions)
+  const jobs = await startJobsFromUpload(uploadTicketId, startJobsOptions)
 
   vlog('Job submission complete.')