diff --git a/config/default.js b/config/default.js index 0766f4d..1df223b 100755 --- a/config/default.js +++ b/config/default.js @@ -126,7 +126,7 @@ module.exports = { orgField: process.env.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME || 'skillProviders' } }, - MAX_BATCH_SIZE: parseInt(process.env.MAX_RESULT_SIZE, 10) || 10000, + MAX_BATCH_SIZE: parseInt(process.env.MAX_BATCH_SIZE, 10) || 10000, MAX_RESULT_SIZE: parseInt(process.env.MAX_RESULT_SIZE, 10) || 1000, MAX_BULK_SIZE: parseInt(process.env.MAX_BULK_SIZE, 10) || 100 } diff --git a/scripts/db/dumpDbToEs.js b/scripts/db/dumpDbToEs.js index 4f6147c..6f3d85d 100644 --- a/scripts/db/dumpDbToEs.js +++ b/scripts/db/dumpDbToEs.js @@ -391,14 +391,12 @@ async function main () { for (let i = 0; i < keys.length; i++) { const key = keys[i] + const queryPage = { perPage: parseInt(config.get('ES.MAX_BATCH_SIZE'), 10), page: 1 } try { - const allData = await dbHelper.find(models[key], {}) - let j = 0 - const dataset = _.chunk(allData, config.get('ES.MAX_BATCH_SIZE')) - for (const data of dataset) { + while (true) { + const data = await dbHelper.find(models[key], { ...queryPage }) for (let i = 0; i < data.length; i++) { - j++ - logger.info(`Inserting data ${j} of ${allData.length}`) + logger.info(`Inserting data ${(i + 1) + (queryPage.perPage * (queryPage.page - 1))}`) logger.info(JSON.stringify(data[i])) if (!_.isString(data[i].created)) { data[i].created = new Date() @@ -414,14 +412,18 @@ async function main () { } } await insertIntoES(key, data) + if (data.length < queryPage.perPage) { + logger.info('import data for ' + key + ' done') + break + } else { + queryPage.page = queryPage.page + 1 + } } - logger.info('import data for ' + key + ' done') } catch (e) { logger.error(e) logger.warn('import data for ' + key + ' failed') continue } - try { await createAndExecuteEnrichPolicy(key) logger.info('create and execute enrich policy for ' + key + ' done')