Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .talismanrc
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,6 @@ fileignoreconfig:
checksum: abc5ac707341760cf59d5b8b1c4e13cf2c79955e2735c33e2db3ec6bc48eddb6
- filename: packages/contentstack-import/src/import/modules/assets.ts
checksum: cda61a9c90bb39f27c09951b8a2623851296aefd0d3220d066032287a712d899
- filename: packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts
checksum: 63c6bff4d51842d8fa3cce88545259d0a2c3cfe71df95d303d993f692cee883b
version: '1.0'
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ export const FALLBACK_AM_API_CONCURRENCY = 5;
export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY;
export const FALLBACK_AM_API_PAGE_SIZE = 100;
export const FALLBACK_AM_API_FETCH_CONCURRENCY = 5;
/** Max assets/uids the CS Assets bulk delete/move endpoints accept per request. */
export const CS_ASSETS_BULK_MUTATE_MAX_ITEMS = 100;

/** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */
export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,18 +127,45 @@ export type BulkDeleteAssetItem = { uid: string; locale: string };

export type BulkDeleteAssetsPayload = { assets: BulkDeleteAssetItem[] };

/** One failed batch when a bulk mutate is split across multiple ≤100-item requests. */
export type BulkMutateFailure = {
batchIndex: number;
count: number;
status?: number;
error: string;
/** Asset uids in the failed batch, so callers can re-run just the failures. */
uids: string[];
};

/** Raw response of a single bulk-mutate request (one ≤100-item POST) as returned by the API. */
export type CsAssetsMutateBatchResponse = { notice?: string; job_id?: string };

/** Aggregate result of a bulk delete, combining every dispatched ≤100-item batch. */
export type BulkDeleteAssetsResponse = {
/** First notice returned; a human-facing message, safe to surface as the summary line. */
notice?: string;
job_id?: string;
/** One submitted job id, for a short summary line; `job_ids` holds all of them (batches
* run concurrently, so this is whichever completed first — not a stable "first batch"). */
primaryJobId?: string;
notices?: string[];
job_ids?: string[];
failures?: BulkMutateFailure[];
batchesTotal?: number;
batchesSucceeded?: number;
};

export type BulkMoveAssetsPayload = {
asset_uids: string[];
target_folder_uid: string;
};

/** Aggregate result of a bulk move (sync; no job ids). */
export type BulkMoveAssetsResponse = {
notice?: string;
notices?: string[];
failures?: BulkMutateFailure[];
batchesTotal?: number;
batchesSucceeded?: number;
};

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@ import chunk from 'lodash/chunk';
import { HttpClient, log, authenticationHandler, handleAndLogError } from '@contentstack/cli-utilities';

import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from './retry';
import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE } from '../constants/index';
import {
CS_ASSETS_BULK_MUTATE_MAX_ITEMS,
FALLBACK_AM_API_CONCURRENCY,
FALLBACK_AM_API_FETCH_CONCURRENCY,
FALLBACK_AM_API_PAGE_SIZE,
} from '../constants/index';

import type {
CSAssetsAPIConfig,
Expand All @@ -14,6 +19,8 @@ import type {
BulkDeleteAssetsResponse,
BulkMoveAssetsPayload,
BulkMoveAssetsResponse,
BulkMutateFailure,
CsAssetsMutateBatchResponse,
CreateAssetMetadata,
CreateAssetTypePayload,
CreateFieldPayload,
Expand Down Expand Up @@ -93,6 +100,21 @@ export type CustomPromiseHandlerInput = {

export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise<any>;

/**
* Error thrown by {@link CSAssetsAdapter.postJson} for a failed POST. Carries the HTTP
* `status` (undefined for network/transport failures) so callers can classify failures
* without parsing the message string.
*/
export class CsAssetsPostError extends Error {
constructor(
message: string,
public readonly status?: number,
) {
super(message);
this.name = 'CsAssetsPostError';
}
}

export class CSAssetsAdapter implements ICSAssetsAdapter {
private readonly config: CSAssetsAPIConfig;
private readonly apiClient: HttpClient;
Expand Down Expand Up @@ -649,10 +671,11 @@ export class CSAssetsAdapter implements ICSAssetsAdapter {
}
const text = await response.text().catch(() => '');
const bodySnippet = this.formatResponseBodyForError(text);
throw new Error(
throw new CsAssetsPostError(
`CS Assets API POST failed: status ${response.status} path ${path}${
bodySnippet ? `\nResponse: ${bodySnippet}` : ''
}`,
response.status,
);
}
return response.json() as Promise<T>;
Expand All @@ -669,12 +692,17 @@ export class CSAssetsAdapter implements ICSAssetsAdapter {
: await doPost();
} catch (error) {
if (error instanceof RetryableHttpError) {
throw new Error(`CS Assets API POST failed: path ${path} (status ${error.status ?? 'network'}) - ${error.message}`);
throw new CsAssetsPostError(
`CS Assets API POST failed: path ${path} (status ${error.status ?? 'network'}) - ${error.message}`,
error.status,
);
}
if (error instanceof Error && error.message.includes('CS Assets API POST failed')) {
if (error instanceof CsAssetsPostError) {
throw error;
}
throw new Error(`CS Assets API POST failed: path ${path} - ${error instanceof Error ? error.message : String(error)}`);
throw new CsAssetsPostError(
`CS Assets API POST failed: path ${path} - ${error instanceof Error ? error.message : String(error)}`,
);
}
}

Expand Down Expand Up @@ -779,18 +807,76 @@ export class CSAssetsAdapter implements ICSAssetsAdapter {
payload: BulkDeleteAssetsPayload,
): Promise<BulkDeleteAssetsResponse> {
const path = `/api/spaces/${encodeURIComponent(spaceUid)}/assets/bulk/delete?workspace=${encodeURIComponent(workspaceUid)}`;
return this.postJson<BulkDeleteAssetsResponse>(path, payload, { space_key: spaceUid });
const bodies = chunk(payload.assets, CS_ASSETS_BULK_MUTATE_MAX_ITEMS).map((assets) => ({ assets }));
const { notices, jobIds, failures, batchesTotal } = await this.dispatchBulkMutateBatches(spaceUid, path, bodies);
return {
notice: notices[0],
primaryJobId: jobIds[0],
notices,
job_ids: jobIds,
failures,
batchesTotal,
batchesSucceeded: batchesTotal - failures.length,
};
}

/**
* POST /api/spaces/{spaceUid}/assets/bulk-move — move assets into a folder.
* Split into ≤{@link CS_ASSETS_BULK_MUTATE_MAX_ITEMS}-item requests (same cap as delete).
*/
async bulkMoveAssets(
spaceUid: string,
workspaceUid: string = 'main',
payload: BulkMoveAssetsPayload,
): Promise<BulkMoveAssetsResponse> {
const path = `/api/spaces/${encodeURIComponent(spaceUid)}/assets/bulk-move?workspace=${encodeURIComponent(workspaceUid)}`;
return this.postJson<BulkMoveAssetsResponse>(path, payload, { space_key: spaceUid });
const bodies = chunk(payload.asset_uids, CS_ASSETS_BULK_MUTATE_MAX_ITEMS).map((asset_uids) => ({
asset_uids,
target_folder_uid: payload.target_folder_uid,
}));
const { notices, failures, batchesTotal } = await this.dispatchBulkMutateBatches(spaceUid, path, bodies);
return {
notice: notices[0],
notices,
failures,
batchesTotal,
batchesSucceeded: batchesTotal - failures.length,
};
}

/**
* Dispatch pre-chunked bulk-mutate request bodies (each already ≤100 items) through
* {@link postJson} with bounded concurrency via {@link makeConcurrentCall}. A batch
* failure is collected — never rethrown — because the CS Assets bulk endpoints commit
* each request independently, so earlier batches are already applied server-side and
* callers must be able to report partial outcomes. `postJson` retries transient
* (429/5xx) failures; 4xx like the 422 item-cap are not retried.
*/
private async dispatchBulkMutateBatches(
spaceUid: string,
path: string,
bodies: unknown[],
): Promise<{ notices: string[]; jobIds: string[]; failures: BulkMutateFailure[]; batchesTotal: number }> {
const notices: string[] = [];
const jobIds: string[] = [];
const failures: BulkMutateFailure[] = [];
const apiBatches = chunk(bodies, FALLBACK_AM_API_CONCURRENCY);

await this.makeConcurrentCall({ module: `bulk-mutate ${path}`, apiBatches }, async ({ element, batchIndex, index }) => {
const globalIndex = batchIndex * FALLBACK_AM_API_CONCURRENCY + index;
const body = element as { assets?: { uid: string }[]; asset_uids?: string[] };
const uids = body.assets ? body.assets.map((a) => a.uid) : (body.asset_uids ?? []);
try {
const r = await this.postJson<CsAssetsMutateBatchResponse>(path, body, { space_key: spaceUid }, { retry: true });
if (typeof r.notice === 'string') notices.push(r.notice);
if (typeof r.job_id === 'string') jobIds.push(r.job_id);
} catch (e) {
const status = e instanceof CsAssetsPostError ? e.status : undefined;
const message = e instanceof Error ? e.message : String(e);
failures.push({ batchIndex: globalIndex, count: uids.length, status, error: message, uids });
}
});

return { notices, jobIds, failures, batchesTotal: bodies.length };
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -791,39 +791,94 @@ describe('CSAssetsAdapter', () => {
});
});

const deleteItems = (n: number) =>
Array.from({ length: n }, (_, i) => ({ uid: `a${i}`, locale: 'en-us' }));

describe('bulkDeleteAssets', () => {
it('POSTs to the bulk delete endpoint with workspace query param', async () => {
fetchStub.resolves(okJsonResponse({ deleted: 2 }));
fetchStub.resolves(okJsonResponse({ notice: 'ok', job_id: 'j1' }));
const adapter = new CSAssetsAdapter(baseConfig);
await adapter.bulkDeleteAssets('sp-1', 'ws-main', { asset_uids: ['a1', 'a2'] } as any);
await adapter.bulkDeleteAssets('sp-1', 'ws-main', { assets: deleteItems(2) });

const [url, opts] = fetchStub.firstCall.args;
expect(url).to.include('/api/spaces/sp-1/assets/bulk/delete');
expect(url).to.include('workspace=ws-main');
expect(opts.headers['space_key']).to.equal('sp-1');
expect(JSON.parse(opts.body).assets).to.have.length(2);
});

it('uses "main" as default workspace uid', async () => {
fetchStub.resolves(okJsonResponse({}));
const adapter = new CSAssetsAdapter(baseConfig);
await adapter.bulkDeleteAssets('sp-1', undefined as any, {} as any);
await adapter.bulkDeleteAssets('sp-1', undefined as any, { assets: deleteItems(1) });

const [url] = fetchStub.firstCall.args;
expect(url).to.include('workspace=main');
});

it('splits >100 assets into ≤100-item batches and aggregates job ids', async () => {
fetchStub.callsFake(async () => okJsonResponse({ notice: 'batch ok', job_id: 'job-x' }));
const adapter = new CSAssetsAdapter(baseConfig);
const result = await adapter.bulkDeleteAssets('sp-1', 'main', { assets: deleteItems(250) });

expect(fetchStub.callCount).to.equal(3); // 100 + 100 + 50
for (const call of fetchStub.getCalls()) {
expect(JSON.parse(call.args[1].body).assets.length).to.be.at.most(100);
}
expect(result.batchesTotal).to.equal(3);
expect(result.batchesSucceeded).to.equal(3);
expect(result.job_ids).to.have.length(3);
expect(result.failures).to.have.length(0);
});

it('keeps succeeded batches and records failures when one batch fails (partial)', async () => {
// 3 batches; the 2nd (second fetch) returns 422, others succeed.
let n = 0;
fetchStub.callsFake(async () => {
n += 1;
return n === 2 ? failResponse(422, 'Assets cannot exceed the max limit of 100.') : okJsonResponse({ job_id: `j${n}` });
});
const adapter = new CSAssetsAdapter(baseConfig);
const result = await adapter.bulkDeleteAssets('sp-1', 'main', { assets: deleteItems(250) });

expect(result.batchesTotal).to.equal(3);
expect(result.batchesSucceeded).to.equal(2);
expect(result.failures).to.have.length(1);
expect(result.failures![0].status).to.equal(422);
expect(result.failures![0].error).to.include('422');
// Failed batch carries its uids so callers can re-run just the failures.
expect(result.failures![0].uids).to.have.length(100);
expect(result.failures![0].uids.every((u) => u.startsWith('a'))).to.equal(true);
});
});

describe('bulkMoveAssets', () => {
it('POSTs to the bulk-move endpoint with workspace query param', async () => {
fetchStub.resolves(okJsonResponse({ moved: 1 }));
const adapter = new CSAssetsAdapter(baseConfig);
await adapter.bulkMoveAssets('sp-1', 'ws-main', { asset_uids: ['a1'], folder_uid: 'f1' } as any);
await adapter.bulkMoveAssets('sp-1', 'ws-main', { asset_uids: ['a1'], target_folder_uid: 'f1' });

const [url, opts] = fetchStub.firstCall.args;
expect(url).to.include('/api/spaces/sp-1/assets/bulk-move');
expect(url).to.include('workspace=ws-main');
expect(opts.headers['space_key']).to.equal('sp-1');
});

it('splits >100 uids into ≤100-item batches, re-attaching target_folder_uid', async () => {
fetchStub.callsFake(async () => okJsonResponse({ notice: 'moved' }));
const adapter = new CSAssetsAdapter(baseConfig);
const uids = Array.from({ length: 150 }, (_, i) => `u${i}`);
const result = await adapter.bulkMoveAssets('sp-1', 'main', { asset_uids: uids, target_folder_uid: 'f1' });

expect(fetchStub.callCount).to.equal(2); // 100 + 50
for (const call of fetchStub.getCalls()) {
const body = JSON.parse(call.args[1].body);
expect(body.asset_uids.length).to.be.at.most(100);
expect(body.target_folder_uid).to.equal('f1');
}
expect(result.batchesTotal).to.equal(2);
expect(result.batchesSucceeded).to.equal(2);
});
});

describe('postJson error handling', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,12 @@ export interface CsAssetsBulkOperationResult {
notice?: string;
jobId?: string;
error?: string;
/** Aggregate across the ≤100-item batches a single delete/move is split into. */
jobIds?: string[];
batchesTotal?: number;
batchesSucceeded?: number;
batchesFailed?: number;
failures?: { batchIndex: number; count: number; error: string; uids: string[] }[];
}

/** Typed flags for CS Assets delete/move operations (cm:stacks:bulk-assets). */
Expand Down
9 changes: 9 additions & 0 deletions packages/contentstack-bulk-operations/src/messages/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -247,11 +247,20 @@ const csAssetsBulkMsg = {
CS_ASSETS_INVALID_OPERATION: 'Invalid operation: {operation}. Must be delete or move',
CS_ASSETS_CONFIRM_SUMMARY: 'Proceed with CS Assets {operation} on {count} item(s)?',
CS_ASSETS_DELETE_SUCCESS: 'CS Assets bulk delete job submitted successfully!',
CS_ASSETS_DELETE_JOBS_SUBMITTED:
'{count} bulk delete job(s) submitted. Deletion runs asynchronously — this confirms submission, not completion. Verify at the status URL below:',
CS_ASSETS_DELETE_JOB_ID: 'Job ID: {jobId}',
CS_ASSETS_DELETE_ASYNC_NOTE: 'The job runs asynchronously — check the bulk task queue for status:',
CS_ASSETS_MOVE_SUCCESS: 'CS Assets bulk move completed successfully!',
CS_ASSETS_MOVE_ASSETS_COUNT: '{count} asset(s) moved to folder: {folderUid}',
CS_ASSETS_OPERATION_FAILED: 'CS Assets {operation} failed.',
CS_ASSETS_BATCH_SUMMARY: 'Dispatched in {batchesTotal} batch(es) of up to 100 — {batchesSucceeded} succeeded.',
CS_ASSETS_PARTIAL_FAILURE: 'CS Assets {operation} partially failed: {batchesFailed} of {batchesTotal} batch(es) failed.',
CS_ASSETS_FAILED_BATCH: 'Batch {batchIndex} ({count} item(s)) failed: {error}',
CS_ASSETS_FAILED_UIDS_WRITTEN:
'Uids whose {operation} request did not confirm success written to: {path} (these requests failed to return success — the server may or may not have applied them).',
CS_ASSETS_RETRY_HINT:
'Re-run just these with: --operation {operation} --asset-uids-file {path} (plus the same --space-uid/--org-uid, and --locale for delete). Safe to re-run — the operation is idempotent, so assets already applied are a no-op.',

// Merged-command flag matrix validation
FLAG_NOT_ALLOWED_FOR_OPERATION: '{flag} is not valid for operation "{operation}".{hint}',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,17 @@ export class CsAssetsService {
const response = await this.adapter.bulkDeleteAssets(spaceUid, workspaceUid ?? 'main', {
assets: items,
});
const failures = response.failures ?? [];
return {
success: true,
success: failures.length === 0,
notice: typeof response.notice === 'string' ? response.notice : undefined,
jobId: typeof response.job_id === 'string' ? response.job_id : undefined,
jobId: typeof response.primaryJobId === 'string' ? response.primaryJobId : undefined,
jobIds: response.job_ids,
batchesTotal: response.batchesTotal,
batchesSucceeded: response.batchesSucceeded,
batchesFailed: failures.length,
failures: failures.map((f) => ({ batchIndex: f.batchIndex, count: f.count, error: f.error, uids: f.uids })),
error: failures.length > 0 ? failures.map((f) => f.error).join('; ') : undefined,
};
} catch (e: unknown) {
return {
Expand All @@ -48,9 +55,15 @@ export class CsAssetsService {
asset_uids: assetUids,
target_folder_uid: targetFolderUid,
});
const failures = response.failures ?? [];
return {
success: true,
success: failures.length === 0,
notice: typeof response.notice === 'string' ? response.notice : undefined,
batchesTotal: response.batchesTotal,
batchesSucceeded: response.batchesSucceeded,
batchesFailed: failures.length,
failures: failures.map((f) => ({ batchIndex: f.batchIndex, count: f.count, error: f.error, uids: f.uids })),
error: failures.length > 0 ? failures.map((f) => f.error).join('; ') : undefined,
};
} catch (e: unknown) {
return {
Expand Down
Loading
Loading