From bf524162d61a651346bb4b4e1c9fc48dd8f49bd6 Mon Sep 17 00:00:00 2001 From: Veronika Al Raheem Date: Thu, 23 Jul 2026 16:51:03 +0200 Subject: [PATCH] chore: migrate cos-sdk examples from ibm-cos-sdk v1 to v2 --- examples/cos-sdk/README.md | 31 +++++++------------ examples/cos-sdk/cos-backup-file.js | 32 ++++++++++--------- examples/cos-sdk/cos-backup-stream.js | 29 +++++++++-------- examples/cos-sdk/cos-restore-file.js | 33 ++++++++++++-------- examples/cos-sdk/cos-restore-stream.js | 43 +++++++++++++++----------- examples/cos-sdk/package.json | 3 +- 6 files changed, 92 insertions(+), 79 deletions(-) diff --git a/examples/cos-sdk/README.md b/examples/cos-sdk/README.md index 5a9ae632..fee54b78 100644 --- a/examples/cos-sdk/README.md +++ b/examples/cos-sdk/README.md @@ -17,7 +17,8 @@ checked out version. ### IBM COS SDK configuration The scripts expect the following values: -* shared credentials file `~/.bluemix/cos_credentials` or target file from `COS_CREDENTIALS_FILE` environment variable +* `IBM_API_KEY_ID` environment variable set to the `apikey` from your COS service credentials +* `IBM_SERVICE_INSTANCE_ID` environment variable set to the `resource_instance_id` from your COS service credentials * `CLOUDANT_IAM_API_KEY` environment variable set to API key with permission to the Cloudant instance * (optional) `CLOUDANT_IAM_TOKEN_URL` environment variable set to the URL of token endpoint (defaults to `https://iam.cloud.ibm.com`) @@ -25,32 +26,24 @@ The scripts expect the following values: When using IBM Cloud Object Storage create a service credential with __disabled__ `Include HMAC Credential` option. -Copy the credentials into `~/.bluemix/cos_credentials` or generate it using the `ibmcloud` CLI tool: +To retrieve your COS credentials using the `ibmcloud` CLI tool: ```bash ibmcloud resource service-key-create --instance-name -ibmcloud resource service-key --output JSON | jq '.[].credentials' > ~/.bluemix/cos_credentials +ibmcloud resource service-key --output JSON | jq '.[].credentials' ``` More info on generating the credentials: https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-service-credentials -#### Service Credentials file structure: -```json -{ - "apikey": "", - "endpoints": "https://control.cloud-object-storage.cloud.ibm.com/v2/endpoints", - "iam_apikey_description": "Auto-generated for key crn:v1:...f9d5b", - "iam_apikey_id": "ApiKey-6f...b1", - "iam_apikey_name": "", - "iam_role_crn": "...Writer", - "iam_serviceid_crn": "crn:v1:...", - "resource_instance_id": "crn:v1:..." -} +From the output, set the environment variables: +```bash +export IBM_API_KEY_ID= +export IBM_SERVICE_INSTANCE_ID= ``` #### IBM COS -Run the scripts with the `--cos_url` option pointing to your COS instance S3 endpoint. +Run the scripts with the `--cos_url` option pointing to your COS instance S3 endpoint and `--cos_region` set to the region of your bucket (e.g. `us-south`, `eu-de`). Corresponding endpoint URLs can be found under the link found in the Service Credentials file or on the IBM Cloud UI (`endpoints` field). @@ -68,7 +61,7 @@ The source database and destination bucket are required options. The minimum needed to run the backup scripts are thus: ```bash -node cos-backup-file.js -s 'https://~replaceWithYourUniqueHost~.cloudantnosqldb.appdomain.cloud/sourcedb' -b 'examplebucket' --cos_url 's3.eu-de.cloud-object-storage.appdomain.cloud' +node cos-backup-file.js -s 'https://~replaceWithYourUniqueHost~.cloudantnosqldb.appdomain.cloud/sourcedb' -b 'examplebucket' --cos_url 's3.eu-de.cloud-object-storage.appdomain.cloud' --cos_region 'eu-de' ``` The object created in the bucket for the backup file will be @@ -88,7 +81,7 @@ The target database URL, source bucket, and backup object name are required opti The minimum needed to run the restore scripts are thus: ```bash -node cos-restore-file.js -t 'https://~replaceWithYourUniqueHost~.cloudantnosqldb.appdomain.cloud/targetdb' -b 'examplebucket' -o 'couchbackup-sourcedb-2024-01-25T09:45:11.730Z' --cos_url 's3.eu-de.cloud-object-storage.appdomain.cloud' +node cos-restore-file.js -t 'https://~replaceWithYourUniqueHost~.cloudantnosqldb.appdomain.cloud/targetdb' -b 'examplebucket' -o 'couchbackup-sourcedb-2024-01-25T09:45:11.730Z' --cos_url 's3.eu-de.cloud-object-storage.appdomain.cloud' --cos_region 'eu-de' ``` ## Progress and debug @@ -97,7 +90,7 @@ To see detailed progress of the backup/restore and upload/download or additional use the `DEBUG` environment variable with label `couchbackup-cos` e.g. ```bash -DEBUG='couchbackup-cos' node cos-backup-file.js -s 'https://~replaceWithYourUniqueHost~.cloudantnosqldb.appdomain.cloud/sourcedb' -b 'couchbackup-example' --cos_url "s3.eu-de.cloud-object-storage.appdomain.cloud" +DEBUG='couchbackup-cos' node cos-backup-file.js -s 'https://~replaceWithYourUniqueHost~.cloudantnosqldb.appdomain.cloud/sourcedb' -b 'couchbackup-example' --cos_url "s3.eu-de.cloud-object-storage.appdomain.cloud" --cos_region "eu-de" ``` ``` diff --git a/examples/cos-sdk/cos-backup-file.js b/examples/cos-sdk/cos-backup-file.js index cb2e0b27..05e05925 100644 --- a/examples/cos-sdk/cos-backup-file.js +++ b/examples/cos-sdk/cos-backup-file.js @@ -18,7 +18,8 @@ // The script generates the backup object name by combining together the path // part of the database URL and the current time. -const IBM_COS = require('ibm-cos-sdk'); +const { S3Client, HeadBucketCommand } = require('ibm-cos-sdk-v2'); +const { Upload } = require('@ibm-cos/lib-storage'); const fs = require('fs'); const VError = require('verror'); const tmp = require('tmp'); @@ -38,6 +39,7 @@ function main() { bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' }, prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' }, cos_url: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint URL' }, // An endpoint from 'endpoints' list in cos_credentials + cos_region: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint region' }, shallow: { describe: 'Backup the documents winning revisions only', type: 'boolean' } }) .help('h').alias('h', 'help') @@ -49,23 +51,23 @@ function main() { const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-'); const backupKeyPrefix = `${argv.prefix}-${backupName}`; const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`; - const cosEndpoint = argv.cos_url; + const cosEndpoint = argv.cos_url.startsWith('https://') ? argv.cos_url : `https://${argv.cos_url}`; + const cosRegion = argv.cos_region; const cloudantApiKey = process.env.CLOUDANT_IAM_API_KEY; const mode = argv.shallow ? 'shallow' : 'full'; const backupTmpFile = tmp.fileSync(); /* - * Creds are from ~/.bluemix/cos_credentials, generated by ibmcloud CLI tool + * Creds are from IBM_API_KEY_ID and IBM_SERVICE_INSTANCE_ID environment variables. * See: https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-service-credentials * - * corresponding 'endpoint' can be found on IBM Cloud UI at the COS instance, - * or under the 'endpoints' link provided in the generated file (~/.bluemix/cos_credentials) + * corresponding 'endpoint' and 'region' can be found on IBM Cloud UI at the COS instance. * */ const config = { endpoint: cosEndpoint, - credentials: new IBM_COS.SharedJSONFileCredentials(), + region: cosRegion, }; - const COS = new IBM_COS.S3(config); + const COS = new S3Client(config); debug(`Creating a new backup of ${sourceUrl} at ${backupBucket}/${backupKey}...`); bucketAccessible(COS, backupBucket) .then(() => { @@ -87,7 +89,7 @@ function main() { * Return a promise that resolves if the bucket is available and * rejects if not. * - * @param {IBM_COS.S3} s3 IBM COS S3 client object + * @param {S3Client} s3 IBM COS S3 client object * @param {any} bucketName Bucket name * @returns Promise */ @@ -95,7 +97,7 @@ function bucketAccessible(s3, bucketName) { const params = { Bucket: bucketName }; - return s3.headBucket(params).promise() + return s3.send(new HeadBucketCommand(params)) .then(() => { debug('Bucket is accessible'); }) .catch((reason) => { console.error(reason); @@ -145,7 +147,7 @@ function createBackupFile(sourceUrl, backupTmpFilePath, cloudantApiKey, mode) { /** * Upload a backup file to an IBM COS bucket. * - * @param {IBM_COS.S3} cos Object store client + * @param {S3Client} cos Object store client * @param {any} backupTmpFilePath Path of backup file to write. * @param {any} bucket Object store bucket name * @param {any} key Object store key name @@ -160,17 +162,17 @@ function uploadNewBackup(cos, backupTmpFilePath, bucket, key) { Key: key, Body: inputStream }; - const options = { + const upload = new Upload({ + client: cos, + params: params, partSize: 5 * 1024 * 1024, // max 5 MB part size (minimum size) queueSize: 5 // allow 5 parts at a time - }; - - const upload = cos.upload(params, options); + }); upload.on('httpUploadProgress', (progress) => { debug(`IBM COS S3 upload progress: ${JSON.stringify(progress)}`); }); - return upload.promise() + return upload.done() .then(() => { debug('Upload succeeded!'); }) diff --git a/examples/cos-sdk/cos-backup-stream.js b/examples/cos-sdk/cos-backup-stream.js index abe7512f..c5f9c874 100644 --- a/examples/cos-sdk/cos-backup-stream.js +++ b/examples/cos-sdk/cos-backup-stream.js @@ -19,7 +19,8 @@ // part of the database URL and the current time. const stream = require('stream'); -const IBM_COS = require('ibm-cos-sdk'); +const { S3Client, HeadBucketCommand } = require('ibm-cos-sdk-v2'); +const { Upload } = require('@ibm-cos/lib-storage'); const VError = require('verror').VError; const couchbackup = require('@cloudant/couchbackup'); const debug = require('debug')('couchbackup-cos'); @@ -37,6 +38,7 @@ function main() { bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Destination bucket' }, prefix: { alias: 'p', nargs: 1, describe: 'Prefix for backup object key', default: 'couchbackup' }, cos_url: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint URL' }, // An endpoint from 'endpoints' list in cos_credentials + cos_region: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint region' }, shallow: { describe: 'Backup the documents winning revisions only', type: 'boolean' } }) .help('h').alias('h', 'help') @@ -48,22 +50,22 @@ function main() { const backupName = new url.URL(sourceUrl).pathname.split('/').filter(function(x) { return x; }).join('-'); const backupKeyPrefix = `${argv.prefix}-${backupName}`; const backupKey = `${backupKeyPrefix}-${new Date().toISOString()}`; - const cosEndpoint = argv.cos_url; + const cosEndpoint = argv.cos_url.startsWith('https://') ? argv.cos_url : `https://${argv.cos_url}`; + const cosRegion = argv.cos_region; const cloudantApiKey = process.env.CLOUDANT_IAM_API_KEY; const mode = argv.shallow ? 'shallow' : 'full'; /* - * Creds are from ~/.bluemix/cos_credentials, generated by ibmcloud CLI tool + * Creds are from IBM_API_KEY_ID and IBM_SERVICE_INSTANCE_ID environment variables. * See: https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-service-credentials * - * corresponding 'endpoint' can be found on IBM Cloud UI at the COS instance, - * or under the 'endpoints' link provided in the generated file (~/.bluemix/cos_credentials) + * corresponding 'endpoint' and 'region' can be found on IBM Cloud UI at the COS instance. * */ const config = { endpoint: cosEndpoint, - credentials: new IBM_COS.SharedJSONFileCredentials(), + region: cosRegion, }; - const COS = new IBM_COS.S3(config); + const COS = new S3Client(config); debug(`Creating a new backup of ${sourceUrl} at ${backupBucket}/${backupKey}...`); bucketAccessible(COS, backupBucket) .then(() => { @@ -82,7 +84,7 @@ function main() { * Return a promise that resolves if the bucket is available and * rejects if not. * - * @param {IBM_COS.S3} s3 IBM COS S3 client object + * @param {S3Client} s3 IBM COS S3 client object * @param {any} bucketName Bucket name * @returns Promise */ @@ -90,7 +92,7 @@ function bucketAccessible(s3, bucketName) { const params = { Bucket: bucketName }; - return s3.headBucket(params).promise() + return s3.send(new HeadBucketCommand(params)) .then(() => { debug('Bucket is accessible'); }) .catch((reason) => { console.error(reason); @@ -123,11 +125,12 @@ function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, cloudantApiKey, mode) Key: s3Key, Body: streamToUpload }; - const options = { + const upload = new Upload({ + client: s3Client, + params: params, partSize: 5 * 1024 * 1024, // max 5 MB part size (minimum size) queueSize: 5 // allow 5 parts at a time - }; - const upload = s3Client.upload(params, options); + }); upload.on('httpUploadProgress', (progress) => { debug(`IBM COS upload progress: ${JSON.stringify(progress)}`); }); @@ -155,7 +158,7 @@ function backupToS3(sourceUrl, s3Client, s3Bucket, s3Key, cloudantApiKey, mode) .on('changes', batch => debug('Couchbackup changes batch: ', batch)) .on('written', progress => debug('Fetched batch:', progress.batch, 'Total document revisions written:', progress.total, 'Time:', progress.time)) ); - return Promise.all([backupPromise, upload.promise()]); + return Promise.all([backupPromise, upload.done()]); } main(); diff --git a/examples/cos-sdk/cos-restore-file.js b/examples/cos-sdk/cos-restore-file.js index 178025f0..af32c825 100644 --- a/examples/cos-sdk/cos-restore-file.js +++ b/examples/cos-sdk/cos-restore-file.js @@ -15,7 +15,7 @@ // Small script which restores a Cloudant or CouchDB database from an IBM Cloud Object Storage (COS) // bucket using an intermediary file on disk -const IBM_COS = require('ibm-cos-sdk'); +const { S3Client, HeadObjectCommand, GetObjectCommand } = require('ibm-cos-sdk-v2'); const VError = require('verror'); const couchbackup = require('@cloudant/couchbackup'); const debug = require('debug')('couchbackup-cos'); @@ -33,6 +33,7 @@ function main() { bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Source bucket containing backup' }, object: { alias: 'o', nargs: 1, demandOption: true, describe: 'Backup Object name in IBM COS' }, cos_url: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint URL' }, + cos_region: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint region' }, }) .help('h').alias('h', 'help') .epilog('Copyright (C) IBM 2025') @@ -40,7 +41,8 @@ function main() { const restoreBucket = argv.bucket; const objectKey = argv.object; - const cosEndpoint = argv.cos_url; + const cosEndpoint = argv.cos_url.startsWith('https://') ? argv.cos_url : `https://${argv.cos_url}`; + const cosRegion = argv.cos_region; const targetUrl = argv.target; const cloudantApiKey = process.env.CLOUDANT_IAM_API_KEY; @@ -48,9 +50,9 @@ function main() { const config = { endpoint: cosEndpoint, - credentials: new IBM_COS.SharedJSONFileCredentials(), + region: cosRegion, }; - const COS = new IBM_COS.S3(config); + const COS = new S3Client(config); restoreProcess(COS, restoreBucket, objectKey, targetUrl, cloudantApiKey, restoreTmpFile.name) .then(() => { debug('Restore completed successfully'); @@ -73,7 +75,7 @@ async function restoreProcess(COS, restoreBucket, objectKey, targetUrl, cloudant /** * Check if object is accessible in COS - * @param {IBM_COS.S3} s3 + * @param {S3Client} s3 * @param {string} bucketName * @param {string} objectKey */ @@ -83,7 +85,7 @@ async function objectAccessible(s3, bucketName, objectKey) { Bucket: bucketName, }; try { - await s3.headObject(params).promise(); + await s3.send(new HeadObjectCommand(params)); debug(`Object '${objectKey}' is accessible`); } catch (reason) { debug(reason); @@ -93,7 +95,7 @@ async function objectAccessible(s3, bucketName, objectKey) { /** * Download backup file from COS to local temporary file - * @param {IBM_COS.S3} COS + * @param {S3Client} COS * @param {string} restoreBucket * @param {string} objectKey * @param {string} restoreTmpFilePath @@ -101,12 +103,17 @@ async function objectAccessible(s3, bucketName, objectKey) { async function createRestoreFile(COS, restoreBucket, objectKey, restoreTmpFilePath) { debug(`Downloading from ${restoreBucket}/${objectKey} to ${restoreTmpFilePath}`); - const inputStream = COS.getObject({ - Bucket: restoreBucket, - Key: objectKey - }).createReadStream({ - highWaterMark: 16 * 1024 * 1024 // 16MB buffer - }); + let inputStream; + try { + const response = await COS.send(new GetObjectCommand({ + Bucket: restoreBucket, + Key: objectKey + })); + inputStream = response.Body; + } catch (err) { + debug('COS GetObjectCommand error:', err); + throw new VError(err, 'Failed to fetch object from COS'); + } const outputStream = fs.createWriteStream(restoreTmpFilePath, { highWaterMark: 16 * 1024 * 1024 // 16MB buffer diff --git a/examples/cos-sdk/cos-restore-stream.js b/examples/cos-sdk/cos-restore-stream.js index 0b7f941a..6ef4a728 100644 --- a/examples/cos-sdk/cos-restore-stream.js +++ b/examples/cos-sdk/cos-restore-stream.js @@ -15,7 +15,7 @@ // Small script which restores a Cloudant or CouchDB database from an IBM Cloud Object Storage (COS) // bucket via direct stream rather than on-disk file -const IBM_COS = require('ibm-cos-sdk'); +const { S3Client, HeadObjectCommand, GetObjectCommand } = require('ibm-cos-sdk-v2'); const VError = require('verror'); const couchbackup = require('@cloudant/couchbackup'); const debug = require('debug')('couchbackup-cos'); @@ -30,6 +30,7 @@ function main() { bucket: { alias: 'b', nargs: 1, demandOption: true, describe: 'Source bucket containing backup' }, object: { alias: 'o', nargs: 1, demandOption: true, describe: 'Backup Object name in IBM COS' }, cos_url: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint URL' }, + cos_region: { nargs: 1, demandOption: true, describe: 'IBM COS S3 endpoint region' }, }) .help('h').alias('h', 'help') .epilog('Copyright (C) IBM 2025') @@ -37,17 +38,18 @@ function main() { const restoreBucket = argv.bucket; const objectKey = argv.object; - const cosEndpoint = argv.cos_url; + const cosEndpoint = argv.cos_url.startsWith('https://') ? argv.cos_url : `https://${argv.cos_url}`; + const cosRegion = argv.cos_region; const targetUrl = argv.target; const cloudantApiKey = process.env.CLOUDANT_IAM_API_KEY; const config = { endpoint: cosEndpoint, - credentials: new IBM_COS.SharedJSONFileCredentials(), + region: cosRegion, }; - const COS = new IBM_COS.S3(config); + const COS = new S3Client(config); objectAccessible(COS, restoreBucket, objectKey) .then(() => { @@ -65,7 +67,7 @@ function main() { /** * Check if object is accessible in COS - * @param {IBM_COS.S3} s3 + * @param {S3Client} s3 * @param {string} bucketName * @param {string} objectKey */ @@ -75,7 +77,7 @@ async function objectAccessible(s3, bucketName, objectKey) { Bucket: bucketName, }; try { - await s3.headObject(params).promise(); + await s3.send(new HeadObjectCommand(params)); debug(`Object '${objectKey}' is accessible`); } catch (reason) { debug(reason); @@ -87,7 +89,7 @@ async function objectAccessible(s3, bucketName, objectKey) { * Restore directly from a backup file on IBM COS S3 to a new and empty CouchDB or Cloudant database. * Uses direct streaming without intermediate files. * - * @param {IBM_COS.S3} cosClient Object store client + * @param {S3Client} cosClient Object store client * @param {string} cosBucket Backup source bucket * @param {string} cosObjectKey Backup file name on IBM COS * @param {string} targetUrl URL of database @@ -96,19 +98,24 @@ async function objectAccessible(s3, bucketName, objectKey) { async function restoreFromCOS(cosClient, cosBucket, cosObjectKey, targetUrl, cloudantApiKey) { debug(`Starting direct stream restore from ${cosBucket}/${cosObjectKey} to ${s(targetUrl)}`); - const cosInputStream = cosClient.getObject({ - Bucket: cosBucket, - Key: cosObjectKey - }).createReadStream({ - highWaterMark: 16 * 1024 * 1024 // 16MB buffer - }); - - cosInputStream.on('error', (err) => { - debug('COS input stream error:', err); - throw new VError(err, 'Failed to read from COS object'); - }); + let cosInputStream; + try { + const response = await cosClient.send(new GetObjectCommand({ + Bucket: cosBucket, + Key: cosObjectKey, + })); + cosInputStream = response.Body; + } catch (err) { + debug('COS GetObjectCommand error:', err); + throw new VError(err, 'Failed to fetch object from COS'); + } const restorePromise = new Promise((resolve, reject) => { + cosInputStream.on('error', (err) => { + debug('COS input stream error:', err); + reject(new VError(err, 'Failed to read from COS object')); + }); + const params = { iamApiKey: cloudantApiKey, ...(process.env.CLOUDANT_IAM_TOKEN_URL && { iamTokenUrl: process.env.CLOUDANT_IAM_TOKEN_URL }), diff --git a/examples/cos-sdk/package.json b/examples/cos-sdk/package.json index f8f135e1..f353dbed 100644 --- a/examples/cos-sdk/package.json +++ b/examples/cos-sdk/package.json @@ -4,7 +4,8 @@ "description": "Examples of using CouchBackup as a library", "dependencies": { "@cloudant/couchbackup": "^2.11.6", - "ibm-cos-sdk": "^1.14.1", + "@ibm-cos/lib-storage": "^1.0.0", + "ibm-cos-sdk-v2": "^1.0.0", "tmp": "^0.2.3", "verror": "^1.10.1", "yargs": "^17.7.2"