Multipart upload

const fs = require('fs');
const fetch = require('node-fetch');


const API_BASE_URL = process.env.API_BASE_URL || 'https://api.cmp.optimizely.com';


class MultipartUploader {
    constructor(apiToken, partSize) {
        this.apiToken = apiToken;
        this.partSize = partSize;
    }

    async uploadLargeFile(file) {
        console.log(`Initiating multipart upload..., file size: ${file.size}, part size: ${this.partSize}`);
        const { uploadId, uploadUrls } = await this.initiateMultipartUpload(file.size);

        console.log('Uploading parts...');
        await this.uploadParts(file, uploadUrls);

        console.log('Completing upload...');
        const fileKey = await this.completeUpload(uploadId);

        console.log('Waiting for completion...');
        await this.waitForCompletion(uploadId);

        console.log('Upload complete. File key:', fileKey);
        return fileKey;
    }

    async getFileSize(filePath) {
        const stats = fs.statSync(filePath);
        return stats.size;
    }

    async initiateMultipartUpload(fileSize) {
        const response = await fetch(`${API_BASE_URL}/v3/multipart-uploads`, {
            method: 'POST',
            headers: {
                'Authorization': `Bearer ${this.apiToken}`,
                'Content-Type': 'application/json'
            },
            body: JSON.stringify({
                file_size: fileSize,
                part_size: this.partSize
            })
        });

        if (!response.ok) {
            const responseJson = await response.json();
            throw new Error(`Failed to initiate upload. response: ${JSON.stringify(responseJson)}`);
        }

        const result = await response.json();
        return {
            uploadId: result.id,
            uploadUrls: result.upload_part_urls,
            partCount: result.upload_part_count,
            expiresAt: result.expires_at
        };
    }

    async uploadParts(file, uploadUrls) {
        const uploadPromises = uploadUrls.map(async (url, index) => {
            const start = index * this.partSize;
            const end = Math.min(start + this.partSize, file.size);
            const chunk = await file.slice(start, end);

            const response = await fetch(url, {
                method: 'PUT',
                body: chunk,
            });

            if (!response.ok) {
                const responseText = await response.text();
                throw new Error(`Failed to upload part ${index + 1}, status: ${response.status}, statusText: ${response.statusText}, responseText: ${responseText}`);
            }

            console.log(`Part ${index + 1} uploaded successfully`);
        });

        // Upload all parts in parallel
        await Promise.all(uploadPromises);
    }

    async completeUpload(uploadId) {
        const response = await fetch(`${API_BASE_URL}/v3/multipart-uploads/${uploadId}/complete`, {
            method: 'POST',
            headers: {
                'Authorization': `Bearer ${this.apiToken}`
            }
        });

        if (!response.ok) {
            throw new Error('Failed to complete upload');
        }

        const result = await response.json();
        return result.key;
    }

    async checkUploadStatus(uploadId) {
        const response = await fetch(`${API_BASE_URL}/v3/multipart-uploads/${uploadId}/status`, {
            headers: {
                'Authorization': `Bearer ${this.apiToken}`
            }
        });

        if (!response.ok) {
            throw new Error('Failed to check upload status');
        }
        return await response.json();
    }

    async waitForCompletion(uploadId) {
        while (true) {
            const status = await this.checkUploadStatus(uploadId);
            console.log(`status: ${status.status}`);

            switch (status.status) {
                case 'UPLOAD_COMPLETION_SUCCEEDED':
                    return status.key;
                case 'UPLOAD_COMPLETION_FAILED':
                    throw new Error(`Upload failed: ${status.status_message}`);
                case 'UPLOAD_COMPLETION_IN_PROGRESS':
                    await new Promise(resolve => setTimeout(resolve, 1000));
                    continue;
                case 'UPLOAD_COMPLETION_NOT_STARTED':
                    throw new Error('Upload completion not initiated');
                default:
                    throw new Error(`Unexpected status: ${status.status}`);
            }
        }
    }
}

async function uploadFile(filePath, partSize, apiToken) {
    const uploader = new MultipartUploader(apiToken, partSize);

    const stats = fs.statSync(filePath);

    const file = {
        size: stats.size,
        slice: (start, end) => {
            return new Promise((resolve, reject) => {
                const chunk = [];
                const sliceStream = fs.createReadStream(filePath, { start, end: end - 1 });

                sliceStream.on('data', data => chunk.push(data));
                sliceStream.on('end', () => resolve(Buffer.concat(chunk)));
                sliceStream.on('error', reject);
            });
        }
    };

    try {
        const fileKey = await uploader.uploadLargeFile(file);
        return fileKey;
    } catch (error) {
        console.error('Upload failed:', error);
        throw error;
    }
}

async function addFileToLibrary(key, name, apiToken) {
    const response = await fetch(`${API_BASE_URL}/v3/assets`, {
        method: 'POST',
        headers: {
            'Authorization': `Bearer ${apiToken}`,
            'Content-Type': 'application/json'
        },
        body: JSON.stringify({
            key,
            title: name,
        })
    });

    if (!response.ok) {
        const responseJson = await response.json();
        throw new Error(`Failed to add file to library. response: ${JSON.stringify(responseJson)}`);
    }

    const result = await response.json();
    return result.id;
}

async function main() {
    const FILE_PATH = process.env.FILE_PATH;
    const PART_SIZE = parseInt(process.env.PART_SIZE) || 5 * 1024 * 1024;
    const API_TOKEN = process.env.API_TOKEN;

    const key = await uploadFile(FILE_PATH, PART_SIZE, API_TOKEN);
    console.log('File uploaded successfully. Key:', key);

    console.log('Adding file to library...');
    const libraryAssetId = await addFileToLibrary(key, 'large-file', API_TOKEN);

    console.log('File added to library successfully. ID:', libraryAssetId);
}

main().catch(console.error);

FILE_PATH=sample-large-file.dat API_TOKEN=********** node multipart_uploader.js