From 3445da0138f9eed9d73d2b3f5f451fcc1fa2e3fe Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Fri, 23 Sep 2022 13:50:57 -0700 Subject: [PATCH 1/9] feat: timings are now written alongside debug log files BREAKING CHANGE: `--timing` file changes: - When run with the `--timing` flag, `npm` now writes timing data to a file alongside the debug log data, respecting the `logs-dir` option and falling back to `/_logs/` dir, instead of directly inside the cache directory. - The timing file data is no longer newline delimited JSON, and instead each run will create a uniquely named `-timing.json` file, with the `` portion being the same as the debug log. - Finally, the data inside the file now has three top level keys, `metadata`, `timers, and `unfinishedTimers` instead of everything being a top level key. Closes https://github.com/npm/statusboard/issues/456 --- docs/content/using-npm/config.md | 7 ++- docs/content/using-npm/logging.md | 8 +-- lib/npm.js | 16 ++---- lib/utils/config/definitions.js | 8 ++- lib/utils/display.js | 29 +++++----- lib/utils/log-file.js | 54 ++++++++----------- lib/utils/run-id.js | 3 ++ lib/utils/timers.js | 43 ++++++++------- .../lib/utils/config/definitions.js.test.cjs | 7 ++- .../lib/utils/config/describe-all.js.test.cjs | 7 ++- test/fixtures/mock-npm.js | 4 +- test/lib/npm.js | 20 ++++--- test/lib/utils/exit-handler.js | 30 ++++++----- test/lib/utils/log-file.js | 3 +- test/lib/utils/timers.js | 10 ++-- 15 files changed, 118 insertions(+), 131 deletions(-) create mode 100644 lib/utils/run-id.js diff --git a/docs/content/using-npm/config.md b/docs/content/using-npm/config.md index 2b05c842f1190..2de35fa2a46c3 100644 --- a/docs/content/using-npm/config.md +++ b/docs/content/using-npm/config.md @@ -1707,12 +1707,11 @@ particular, use care when overriding this setting for public packages. * Default: false * Type: Boolean -If true, writes a debug log to `logs-dir` and timing information to -`_timing.json` in the cache, even if the command completes successfully. -`_timing.json` is a newline delimited list of JSON objects. +If true, writes timing information to a process specific json file in the +cache or `logs-dir`. The file name ends with `-timing.json`. You can quickly view it with this [json](https://npm.im/json) command line: -`npm exec -- json -g < ~/.npm/_timing.json`. +`cat ~/.npm/_logs/*-timing.json | npm exec -- json -g`. diff --git a/docs/content/using-npm/logging.md b/docs/content/using-npm/logging.md index d699c8dca6ccc..e52c19004d542 100644 --- a/docs/content/using-npm/logging.md +++ b/docs/content/using-npm/logging.md @@ -12,8 +12,7 @@ The `npm` CLI has various mechanisms for showing different levels of information All logs are written to a debug log, with the path to that file printed if the execution of a command fails. -The default location of the logs directory is a directory named `_logs` inside the npm cache. This can be changed -with the `logs-dir` config option. +The default location of the logs directory is a directory named `_logs` inside the npm cache. This can be changed with the `logs-dir` config option. Log files will be removed from the `logs-dir` when the number of log files exceeds `logs-max`, with the oldest logs being deleted first. @@ -62,9 +61,10 @@ The [`--timing` config](/using-npm/config#timing) can be set which does two things: 1. Always shows the full path to the debug log regardless of command exit status -1. Write timing information to a timing file in the cache or `logs-dir` +1. Write timing information to a process specific timing file in the cache or `logs-dir` -This file is a newline delimited list of JSON objects that can be inspected to see timing data for each task in a `npm` CLI run. +This file contains a `timers` object where the keys are an identifier for the +portion of the process being timed and the value is the number of milliseconds it took to complete. ### Registry Response Headers diff --git a/lib/npm.js b/lib/npm.js index b116ec5cc68a4..a178dca727e30 100644 --- a/lib/npm.js +++ b/lib/npm.js @@ -18,9 +18,11 @@ const replaceInfo = require('./utils/replace-info.js') const updateNotifier = require('./utils/update-notifier.js') const pkg = require('../package.json') const cmdList = require('./utils/cmd-list.js') +const runId = require('./utils/run-id.js') let warnedNonDashArg = false const _load = Symbol('_load') +const RUN_ID = runId() class Npm extends EventEmitter { static get version () { @@ -38,9 +40,10 @@ class Npm extends EventEmitter { #argvClean = [] #chalk = null - #logFile = new LogFile() + #logFile = new LogFile({ id: RUN_ID }) #display = new Display() #timers = new Timers({ + id: RUN_ID, start: 'npm', listener: (name, ms) => { const args = ['timing', name, `Completed in ${ms}ms`] @@ -207,10 +210,6 @@ class Npm extends EventEmitter { writeTimingFile () { this.#timers.writeFile({ command: this.#argvClean, - // We used to only ever report a single log file - // so to be backwards compatible report the last logfile - // XXX: remove this in npm 9 or just keep it forever - logfile: this.logFiles[this.logFiles.length - 1], logfiles: this.logFiles, version: this.version, }) @@ -298,7 +297,7 @@ class Npm extends EventEmitter { this.time('npm:load:timers', () => this.#timers.load({ - dir: this.config.get('timing') ? this.timingDir : null, + dir: this.config.get('timing') ? this.logsDir : null, }) ) @@ -376,11 +375,6 @@ class Npm extends EventEmitter { return this.#timers.file } - get timingDir () { - // XXX(npm9): make this always in logs-dir - return this.config.get('logs-dir') || this.cache - } - get cache () { return this.config.get('cache') } diff --git a/lib/utils/config/definitions.js b/lib/utils/config/definitions.js index fc39bddc70a42..ae38efc32b5ae 100644 --- a/lib/utils/config/definitions.js +++ b/lib/utils/config/definitions.js @@ -2062,13 +2062,11 @@ define('timing', { default: false, type: Boolean, description: ` - If true, writes a debug log to \`logs-dir\` and timing information - to \`_timing.json\` in the cache, even if the command completes - successfully. \`_timing.json\` is a newline delimited list of JSON - objects. + If true, writes timing information to a process specific json file in + the cache or \`logs-dir\`. The file name ends with \`-timing.json\`. You can quickly view it with this [json](https://npm.im/json) command - line: \`npm exec -- json -g < ~/.npm/_timing.json\`. + line: \`cat ~/.npm/_logs/*-timing.json | npm exec -- json -g\`. `, }) diff --git a/lib/utils/display.js b/lib/utils/display.js index a19f72297e838..b23b2467ff4e6 100644 --- a/lib/utils/display.js +++ b/lib/utils/display.js @@ -3,11 +3,6 @@ const npmlog = require('npmlog') const log = require('./log-shim.js') const { explain } = require('./explain-eresolve.js') -const _logHandler = Symbol('logHandler') -const _eresolveWarn = Symbol('eresolveWarn') -const _log = Symbol('log') -const _npmlog = Symbol('npmlog') - class Display { constructor () { // pause by default until config is loaded @@ -16,11 +11,11 @@ class Display { } on () { - process.on('log', this[_logHandler]) + process.on('log', this.#logHandler) } off () { - process.off('log', this[_logHandler]) + process.off('log', this.#logHandler) // Unbalanced calls to enable/disable progress // will leave change listeners on the tracker // This pretty much only happens in tests but @@ -72,16 +67,16 @@ class Display { } log (...args) { - this[_logHandler](...args) + this.#logHandler(...args) } - [_logHandler] = (level, ...args) => { + #logHandler = (level, ...args) => { try { - this[_log](level, ...args) + this.#log(level, ...args) } catch (ex) { try { // if it crashed once, it might again! - this[_npmlog]('verbose', `attempt to log ${inspect(args)} crashed`, ex) + this.#npmlog('verbose', `attempt to log ${inspect(args)} crashed`, ex) } catch (ex2) { // eslint-disable-next-line no-console console.error(`attempt to log ${inspect(args)} crashed`, ex, ex2) @@ -89,13 +84,13 @@ class Display { } } - [_log] (...args) { - return this[_eresolveWarn](...args) || this[_npmlog](...args) + #log (...args) { + return this.#eresolveWarn(...args) || this.#npmlog(...args) } // Explicitly call these on npmlog and not log shim // This is the final place we should call npmlog before removing it. - [_npmlog] (level, ...args) { + #npmlog (level, ...args) { npmlog[level](...args) } @@ -103,13 +98,13 @@ class Display { // log.warn() method so that when we see a peerDep override // explanation from Arborist, we can replace the object with a // highly abbreviated explanation of what's being overridden. - [_eresolveWarn] (level, heading, message, expl) { + #eresolveWarn (level, heading, message, expl) { if (level === 'warn' && heading === 'ERESOLVE' && expl && typeof expl === 'object' ) { - this[_npmlog](level, heading, message) - this[_npmlog](level, '', explain(expl, log.useColor(), 2)) + this.#npmlog(level, heading, message) + this.#npmlog(level, '', explain(expl, log.useColor(), 2)) // Return true to short circuit other log in chain return true } diff --git a/lib/utils/log-file.js b/lib/utils/log-file.js index d62329c8551e2..a18eb2878f0be 100644 --- a/lib/utils/log-file.js +++ b/lib/utils/log-file.js @@ -7,18 +7,11 @@ const MiniPass = require('minipass') const fsMiniPass = require('fs-minipass') const fs = require('@npmcli/fs') const log = require('./log-shim') +const runId = require('./run-id') const padZero = (n, length) => n.toString().padStart(length.toString().length, '0') const globify = pattern => pattern.split('\\').join('/') -const _logHandler = Symbol('logHandler') -const _formatLogItem = Symbol('formatLogItem') -const _getLogFilePath = Symbol('getLogFilePath') -const _openLogFile = Symbol('openLogFile') -const _cleanLogs = Symbol('cleanlogs') -const _endStream = Symbol('endStream') -const _isBuffered = Symbol('isBuffered') - class LogFiles { // If we write multiple log files we want them all to have the same // identifier for sorting and matching purposes @@ -46,19 +39,16 @@ class LogFiles { #files = [] constructor ({ + id = runId(), maxLogsPerFile = 50_000, maxFilesPerProcess = 5, } = {}) { - this.#logId = LogFiles.logId(new Date()) + this.#logId = id this.#MAX_LOGS_PER_FILE = maxLogsPerFile this.#MAX_FILES_PER_PROCESS = maxFilesPerProcess this.on() } - static logId (d) { - return d.toISOString().replace(/[.:]/g, '_') - } - static format (count, level, title, ...args) { let prefix = `${count} ${level}` if (title) { @@ -75,12 +65,12 @@ class LogFiles { on () { this.#logStream = new MiniPass() - process.on('log', this[_logHandler]) + process.on('log', this.#logHandler) } off () { - process.off('log', this[_logHandler]) - this[_endStream]() + process.off('log', this.#logHandler) + this.#endStream() } load ({ dir, logsMax = Infinity } = {}) { @@ -100,7 +90,7 @@ class LogFiles { // set that as the new log logstream for future writes // if logs max is 0 then the user does not want a log file if (this.#logsMax > 0) { - const initialFile = this[_openLogFile]() + const initialFile = this.#openLogFile() if (initialFile) { this.#logStream = this.#logStream.pipe(initialFile) } @@ -109,29 +99,29 @@ class LogFiles { // Kickoff cleaning process, even if we aren't writing a logfile. // This is async but it will always ignore the current logfile // Return the result so it can be awaited in tests - return this[_cleanLogs]() + return this.#cleanLogs() } log (...args) { - this[_logHandler](...args) + this.#logHandler(...args) } get files () { return this.#files } - get [_isBuffered] () { + get #isBuffered () { return this.#logStream instanceof MiniPass } - [_endStream] (output) { + #endStream (output) { if (this.#logStream) { this.#logStream.end(output) this.#logStream = null } } - [_logHandler] = (level, ...args) => { + #logHandler = (level, ...args) => { // Ignore pause and resume events since we // write everything to the log file if (level === 'pause' || level === 'resume') { @@ -143,9 +133,9 @@ class LogFiles { return } - const logOutput = this[_formatLogItem](level, ...args) + const logOutput = this.#formatLogItem(level, ...args) - if (this[_isBuffered]) { + if (this.#isBuffered) { // Cant do anything but buffer the output if we dont // have a file stream yet this.#logStream.write(logOutput) @@ -155,29 +145,29 @@ class LogFiles { // Open a new log file if we've written too many logs to this one if (this.#fileLogCount >= this.#MAX_LOGS_PER_FILE) { // Write last chunk to the file and close it - this[_endStream](logOutput) + this.#endStream(logOutput) if (this.#files.length >= this.#MAX_FILES_PER_PROCESS) { // but if its way too many then we just stop listening this.off() } else { // otherwise we are ready for a new file for the next event - this.#logStream = this[_openLogFile]() + this.#logStream = this.#openLogFile() } } else { this.#logStream.write(logOutput) } } - [_formatLogItem] (...args) { + #formatLogItem (...args) { this.#fileLogCount += 1 return LogFiles.format(this.#totalLogCount++, ...args) } - [_getLogFilePath] (count = '') { + #getLogFilePath (count = '') { return path.resolve(this.#dir, `${this.#logId}-debug-${count}.log`) } - [_openLogFile] () { + #openLogFile () { // Count in filename will be 0 indexed const count = this.#files.length @@ -186,7 +176,7 @@ class LogFiles { // We never want to write files ending in `-9.log` and `-10.log` because // log file cleaning is done by deleting the oldest so in this example // `-10.log` would be deleted next - const f = this[_getLogFilePath](padZero(count, this.#MAX_FILES_PER_PROCESS)) + const f = this.#getLogFilePath(padZero(count, this.#MAX_FILES_PER_PROCESS)) // Some effort was made to make the async, but we need to write logs // during process.on('exit') which has to be synchronous. So in order // to never drop log messages, it is easiest to make it sync all the time @@ -210,7 +200,7 @@ class LogFiles { } } - async [_cleanLogs] () { + async #cleanLogs () { // module to clean out the old log files // this is a best-effort attempt. if a rm fails, we just // log a message about it and move on. We do return a @@ -218,7 +208,7 @@ class LogFiles { // just for the benefit of testing this function properly. try { - const logPath = this[_getLogFilePath]() + const logPath = this.#getLogFilePath() const logGlob = path.join(path.dirname(logPath), path.basename(logPath) // tell glob to only match digits .replace(/\d/g, '[0123456789]') diff --git a/lib/utils/run-id.js b/lib/utils/run-id.js new file mode 100644 index 0000000000000..b7e7cf797da3c --- /dev/null +++ b/lib/utils/run-id.js @@ -0,0 +1,3 @@ +module.exports = (d = new Date()) => { + return d.toISOString().replace(/[.:]/g, '_') +} diff --git a/lib/utils/timers.js b/lib/utils/timers.js index 3336c3b519259..d4f8d8bdedd6d 100644 --- a/lib/utils/timers.js +++ b/lib/utils/timers.js @@ -2,10 +2,7 @@ const EE = require('events') const { resolve } = require('path') const fs = require('@npmcli/fs') const log = require('./log-shim') - -const _timeListener = Symbol('timeListener') -const _timeEndListener = Symbol('timeEndListener') -const _init = Symbol('init') +const runId = require('./run-id') // This is an event emiiter but on/off // only listen on a single internal event that gets @@ -13,17 +10,19 @@ const _init = Symbol('init') class Timers extends EE { file = null + #id = null #unfinished = new Map() #finished = {} #onTimeEnd = Symbol('onTimeEnd') #initialListener = null #initialTimer = null - constructor ({ listener = null, start = 'npm' } = {}) { + constructor ({ id = runId(), listener = null, start = 'npm' } = {}) { super() + this.#id = id this.#initialListener = listener this.#initialTimer = start - this[_init]() + this.#init() } get unfinished () { @@ -34,7 +33,7 @@ class Timers extends EE { return this.#finished } - [_init] () { + #init () { this.on() if (this.#initialListener) { this.on(this.#initialListener) @@ -47,8 +46,8 @@ class Timers extends EE { if (listener) { super.on(this.#onTimeEnd, listener) } else { - process.on('time', this[_timeListener]) - process.on('timeEnd', this[_timeEndListener]) + process.on('time', this.#timeListener) + process.on('timeEnd', this.#timeEndListener) } } @@ -57,8 +56,8 @@ class Timers extends EE { super.off(this.#onTimeEnd, listener) } else { this.removeAllListeners(this.#onTimeEnd) - process.off('time', this[_timeListener]) - process.off('timeEnd', this[_timeEndListener]) + process.off('time', this.#timeListener) + process.off('timeEnd', this.#timeEndListener) } } @@ -74,11 +73,11 @@ class Timers extends EE { load ({ dir } = {}) { if (dir) { - this.file = resolve(dir, '_timing.json') + this.file = resolve(dir, `${this.#id}-timing.json`) } } - writeFile (fileData) { + writeFile (metadata) { if (!this.file) { return } @@ -87,20 +86,20 @@ class Timers extends EE { const globalStart = this.started const globalEnd = this.#finished.npm || Date.now() const content = { - ...fileData, - ...this.#finished, + metadata: { + id: this.#id, + ...metadata, + }, + timers: this.#finished, // add any unfinished timers with their relative start/end - unfinished: [...this.#unfinished.entries()].reduce((acc, [name, start]) => { + unfinishedTimers: [...this.#unfinished.entries()].reduce((acc, [name, start]) => { acc[name] = [start - globalStart, globalEnd - globalStart] return acc }, {}), } - // we append line delimited json to this file...forever - // XXX: should we also write a process specific timing file? - // with similar rules to the debug log (max files, etc) fs.withOwnerSync( this.file, - () => fs.appendFileSync(this.file, JSON.stringify(content) + '\n'), + () => fs.writeFileSync(this.file, JSON.stringify(content) + '\n'), { owner: 'inherit' } ) } catch (e) { @@ -109,11 +108,11 @@ class Timers extends EE { } } - [_timeListener] = (name) => { + #timeListener = (name) => { this.#unfinished.set(name, Date.now()) } - [_timeEndListener] = (name) => { + #timeEndListener = (name) => { if (this.#unfinished.has(name)) { const ms = Date.now() - this.#unfinished.get(name) this.#finished[name] = ms diff --git a/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs b/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs index ddf2396a6fe00..df9313270d056 100644 --- a/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs +++ b/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs @@ -1762,12 +1762,11 @@ exports[`test/lib/utils/config/definitions.js TAP > config description for timin * Default: false * Type: Boolean -If true, writes a debug log to \`logs-dir\` and timing information to -\`_timing.json\` in the cache, even if the command completes successfully. -\`_timing.json\` is a newline delimited list of JSON objects. +If true, writes timing information to a process specific json file in the +cache or \`logs-dir\`. The file name ends with \`-timing.json\`. You can quickly view it with this [json](https://npm.im/json) command line: -\`npm exec -- json -g < ~/.npm/_timing.json\`. +\`cat ~/.npm/_logs/*-timing.json | npm exec -- json -g\`. ` exports[`test/lib/utils/config/definitions.js TAP > config description for tmp 1`] = ` diff --git a/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs b/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs index 2c51efec54afe..89ca7c952b182 100644 --- a/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs +++ b/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs @@ -1580,12 +1580,11 @@ particular, use care when overriding this setting for public packages. * Default: false * Type: Boolean -If true, writes a debug log to \`logs-dir\` and timing information to -\`_timing.json\` in the cache, even if the command completes successfully. -\`_timing.json\` is a newline delimited list of JSON objects. +If true, writes timing information to a process specific json file in the +cache or \`logs-dir\`. The file name ends with \`-timing.json\`. You can quickly view it with this [json](https://npm.im/json) command line: -\`npm exec -- json -g < ~/.npm/_timing.json\`. +\`cat ~/.npm/_logs/*-timing.json | npm exec -- json -g\`. diff --git a/test/fixtures/mock-npm.js b/test/fixtures/mock-npm.js index 330c341474da3..72cfa4b0c4beb 100644 --- a/test/fixtures/mock-npm.js +++ b/test/fixtures/mock-npm.js @@ -174,8 +174,8 @@ const LoadMockNpm = async (t, { .join('\n') }, timingFile: async () => { - const data = await fs.readFile(path.resolve(dirs.cache, '_timing.json'), 'utf8') - return JSON.parse(data) // XXX: this fails if multiple timings are written + const data = await fs.readFile(npm.timingFile, 'utf8') + return JSON.parse(data) }, } } diff --git a/test/lib/npm.js b/test/lib/npm.js index 135a02cc98a3d..2bf7f426db409 100644 --- a/test/lib/npm.js +++ b/test/lib/npm.js @@ -513,19 +513,23 @@ t.test('timings', async t => { process.emit('timeEnd', 'foo') process.emit('time', 'bar') npm.writeTimingFile() - t.equal(npm.timingFile, join(cache, '_timing.json')) + t.match(npm.timingFile, cache) + t.match(npm.timingFile, /-timing.json$/) const timings = await timingFile() t.match(timings, { - command: [], - logfile: String, - logfiles: [String], - version: String, - unfinished: { + metadata: { + command: [], + logfiles: [String], + version: String, + }, + unfinishedTimers: { bar: [Number, Number], npm: [Number, Number], }, - foo: Number, - 'npm:load': Number, + timers: { + foo: Number, + 'npm:load': Number, + }, }) }) diff --git a/test/lib/utils/exit-handler.js b/test/lib/utils/exit-handler.js index 23942cca1c078..de4ce61aaa6af 100644 --- a/test/lib/utils/exit-handler.js +++ b/test/lib/utils/exit-handler.js @@ -326,7 +326,7 @@ t.test('timing with no error', async (t) => { t.match(msg, /Timing info written to:/) t.match( - timingFileData, + timingFileData.timers, Object.keys(npm.finishedTimers).reduce((acc, k) => { acc[k] = Number return acc @@ -334,11 +334,14 @@ t.test('timing with no error', async (t) => { ) t.strictSame(npm.unfinishedTimers, new Map()) t.match(timingFileData, { - command: [], - version: '1.0.0', - npm: Number, - logfile: String, - logfiles: [String], + metadata: { + command: [], + version: '1.0.0', + logfiles: [String], + }, + timers: { + npm: Number, + }, }) }) @@ -356,12 +359,15 @@ t.test('unfinished timers', async (t) => { t.equal(process.exitCode, 0) t.match(npm.unfinishedTimers, new Map([['foo', Number], ['bar', Number]])) t.match(timingFileData, { - command: [], - version: '1.0.0', - npm: Number, - logfile: String, - logfiles: [String], - unfinished: { + metadata: { + command: [], + version: '1.0.0', + logfiles: [String], + }, + timers: { + npm: Number, + }, + unfinishedTimers: { foo: [Number, Number], bar: [Number, Number], }, diff --git a/test/lib/utils/log-file.js b/test/lib/utils/log-file.js index ce6f0bf4cf51f..3c7bb3fe8324a 100644 --- a/test/lib/utils/log-file.js +++ b/test/lib/utils/log-file.js @@ -6,6 +6,7 @@ const os = require('os') const fsMiniPass = require('fs-minipass') const rimraf = require('rimraf') const LogFile = require('../../../lib/utils/log-file.js') +const runId = require('../../../lib/utils/run-id.js') const { cleanCwd } = require('../../fixtures/clean-snapshot') t.cleanSnapshot = (path) => cleanCwd(path) @@ -19,7 +20,7 @@ const makeOldLogs = (count, oldStyle) => { return range(oldStyle ? count : (count / 2)).reduce((acc, i) => { const cloneDate = new Date(d.getTime()) cloneDate.setSeconds(i) - const dateId = LogFile.logId(cloneDate) + const dateId = runId(cloneDate) if (oldStyle) { acc[`${dateId}-debug.log`] = 'hello' } else { diff --git a/test/lib/utils/timers.js b/test/lib/utils/timers.js index 30e54700c63a9..259ecd5dd4893 100644 --- a/test/lib/utils/timers.js +++ b/test/lib/utils/timers.js @@ -68,17 +68,17 @@ t.test('finish unstarted timer', async (t) => { }) t.test('writes file', async (t) => { - const { timers } = mockTimers(t) + const { timers } = mockTimers(t, { id: 'TIMING_FILE' }) const dir = t.testdir() process.emit('time', 'foo') process.emit('timeEnd', 'foo') timers.load({ dir }) timers.writeFile({ some: 'data' }) - const data = JSON.parse(fs.readFileSync(resolve(dir, '_timing.json'))) + const data = JSON.parse(fs.readFileSync(resolve(dir, 'TIMING_FILE-timing.json'))) t.match(data, { - some: 'data', - foo: Number, - unfinished: { + metadata: { some: 'data' }, + timers: { foo: Number }, + unfinishedTimers: { npm: [Number, Number], }, }) From e64d69aedecc0943425605b3a6dc68aec3ad93aa Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Sun, 25 Sep 2022 01:31:56 -0700 Subject: [PATCH 2/9] feat: write eresolve error files to the logs directory Also refactor all files written to the logs directory to use the same code path for file name creation. --- lib/npm.js | 15 +- lib/utils/error-message.js | 10 +- lib/utils/exit-handler.js | 35 +++-- lib/utils/explain-eresolve.js | 35 +++-- lib/utils/log-file.js | 17 +-- lib/utils/run-id.js | 3 - lib/utils/timers.js | 17 +-- .../test/lib/utils/error-message.js.test.cjs | 22 +-- .../test/lib/utils/exit-handler.js.test.cjs | 4 +- .../lib/utils/explain-eresolve.js.test.cjs | 136 +++--------------- .../test/lib/utils/log-file.js.test.cjs | 2 +- test/lib/utils/error-message.js | 19 +-- test/lib/utils/exit-handler.js | 53 ++++++- test/lib/utils/explain-eresolve.js | 37 ++--- test/lib/utils/log-file.js | 15 +- test/lib/utils/timers.js | 6 +- 16 files changed, 192 insertions(+), 234 deletions(-) delete mode 100644 lib/utils/run-id.js diff --git a/lib/npm.js b/lib/npm.js index a178dca727e30..852d91ad3f890 100644 --- a/lib/npm.js +++ b/lib/npm.js @@ -18,11 +18,9 @@ const replaceInfo = require('./utils/replace-info.js') const updateNotifier = require('./utils/update-notifier.js') const pkg = require('../package.json') const cmdList = require('./utils/cmd-list.js') -const runId = require('./utils/run-id.js') let warnedNonDashArg = false const _load = Symbol('_load') -const RUN_ID = runId() class Npm extends EventEmitter { static get version () { @@ -34,16 +32,16 @@ class Npm extends EventEmitter { loadErr = null argv = [] + #runId = new Date().toISOString().replace(/[.:]/g, '_') #loadPromise = null #tmpFolder = null #title = 'npm' #argvClean = [] #chalk = null - #logFile = new LogFile({ id: RUN_ID }) + #logFile = new LogFile() #display = new Display() #timers = new Timers({ - id: RUN_ID, start: 'npm', listener: (name, ms) => { const args = ['timing', name, `Completed in ${ms}ms`] @@ -209,6 +207,7 @@ class Npm extends EventEmitter { writeTimingFile () { this.#timers.writeFile({ + id: this.#runId, command: this.#argvClean, logfiles: this.logFiles, version: this.version, @@ -289,7 +288,7 @@ class Npm extends EventEmitter { this.time('npm:load:logFile', () => { this.#logFile.load({ - dir: this.logsDir, + path: this.logPath, logsMax: this.config.get('logs-max'), }) log.verbose('logfile', this.#logFile.files[0] || 'no logfile created') @@ -297,7 +296,7 @@ class Npm extends EventEmitter { this.time('npm:load:timers', () => this.#timers.load({ - dir: this.config.get('timing') ? this.logsDir : null, + path: this.config.get('timing') ? this.logPath : null, }) ) @@ -371,6 +370,10 @@ class Npm extends EventEmitter { return this.config.get('logs-dir') || join(this.cache, '_logs') } + get logPath () { + return resolve(this.logsDir, `${this.#runId}-`) + } + get timingFile () { return this.#timers.file } diff --git a/lib/utils/error-message.js b/lib/utils/error-message.js index adf10a56f6d66..aee376120ba27 100644 --- a/lib/utils/error-message.js +++ b/lib/utils/error-message.js @@ -8,6 +8,7 @@ const log = require('./log-shim') module.exports = (er, npm) => { const short = [] const detail = [] + const files = [] if (er.message) { er.message = replaceInfo(er.message) @@ -17,14 +18,17 @@ module.exports = (er, npm) => { } switch (er.code) { - case 'ERESOLVE': + case 'ERESOLVE': { short.push(['ERESOLVE', er.message]) detail.push(['', '']) // XXX(display): error messages are logged so we use the logColor since that is based // on stderr. This should be handled solely by the display layer so it could also be // printed to stdout if necessary. - detail.push(['', report(er, !!npm.logColor, resolve(npm.cache, 'eresolve-report.txt'))]) + const { explanation, file } = report(er, !!npm.logColor) + detail.push(['', explanation]) + files.push(['eresolve-report.txt', file]) break + } case 'ENOLOCK': { const cmd = npm.command || '' @@ -398,5 +402,5 @@ module.exports = (er, npm) => { break } - return { summary: short, detail: detail } + return { summary: short, detail, files } } diff --git a/lib/utils/exit-handler.js b/lib/utils/exit-handler.js index d8ae9994dfecc..f1843347bc324 100644 --- a/lib/utils/exit-handler.js +++ b/lib/utils/exit-handler.js @@ -1,4 +1,5 @@ const os = require('os') +const fs = require('@npmcli/fs') const log = require('./log-shim.js') const errorMessage = require('./error-message.js') @@ -18,11 +19,10 @@ process.on('exit', code => { // unfinished timer check below process.emit('timeEnd', 'npm') - const hasNpm = !!npm - const hasLoadedNpm = hasNpm && npm.config.loaded + const hasLoadedNpm = npm?.config.loaded // Unfinished timers can be read before config load - if (hasNpm) { + if (npm) { for (const [name, timer] of npm.unfinishedTimers) { log.verbose('unfinished npm timer', name, timer) } @@ -111,10 +111,9 @@ const exitHandler = err => { log.disableProgress() - const hasNpm = !!npm - const hasLoadedNpm = hasNpm && npm.config.loaded + const hasLoadedNpm = npm?.config.loaded - if (!hasNpm) { + if (!npm) { err = err || new Error('Exit prior to setting npm in exit handler') // eslint-disable-next-line no-console console.error(err.stack || err.message) @@ -181,8 +180,24 @@ const exitHandler = err => { } } - const msg = errorMessage(err, npm) - for (const errline of [...msg.summary, ...msg.detail]) { + const { summary, detail, files = [] } = errorMessage(err, npm) + + for (let [file, content] of files) { + file = `${npm.logPath}${file}` + content = `'Log files:\n${npm.logFiles.join('\n')}\n\n${content.trim()}\n` + try { + fs.withOwnerSync( + file, + () => fs.writeFileSync(file, content), + { owner: 'inherit' } + ) + detail.push(['', `\n\nFor a full report see:\n${file}`]) + } catch (err) { + log.warn('', `Could not write error message to ${file} due to ${err}`) + } + } + + for (const errline of [...summary, ...detail]) { log.error(...errline) } @@ -190,8 +205,8 @@ const exitHandler = err => { const error = { error: { code: err.code, - summary: messageText(msg.summary), - detail: messageText(msg.detail), + summary: messageText(summary), + detail: messageText(detail), }, } npm.outputError(JSON.stringify(error, null, 2)) diff --git a/lib/utils/explain-eresolve.js b/lib/utils/explain-eresolve.js index 7f6a10869c73c..480cd8e5cd4e6 100644 --- a/lib/utils/explain-eresolve.js +++ b/lib/utils/explain-eresolve.js @@ -1,7 +1,6 @@ // this is called when an ERESOLVE error is caught in the exit-handler, // or when there's a log.warn('eresolve', msg, explanation), to turn it // into a human-intelligible explanation of what's wrong and how to fix. -const { writeFileSync } = require('fs') const { explainEdge, explainNode, printNode } = require('./explain-dep.js') // expl is an explanation object that comes from Arborist. It looks like: @@ -45,27 +44,25 @@ const explain = (expl, color, depth) => { } // generate a full verbose report and tell the user how to fix it -const report = (expl, color, fullReport) => { - const orNoStrict = expl.strictPeerDeps ? '--no-strict-peer-deps, ' : '' - const fix = `Fix the upstream dependency conflict, or retry -this command with ${orNoStrict}--force, or --legacy-peer-deps -to accept an incorrect (and potentially broken) dependency resolution.` - - writeFileSync(fullReport, `# npm resolution error report - -${new Date().toISOString()} - -${explain(expl, false, Infinity)} +const report = (expl, color) => { + const flags = [ + expl.strictPeerDeps ? '--no-strict-peer-deps' : '', + '--force', + '--legacy-peer-deps', + ].filter(Boolean) -${fix} + const or = (arr) => arr.length <= 2 + ? arr.join(' or ') : + arr.map((v, i, l) => i + 1 === l.length ? `or ${v}` : v).join(', ') -Raw JSON explanation object: - -${JSON.stringify(expl, null, 2)} -`, 'utf8') + const fix = `Fix the upstream dependency conflict, or retry +this command with ${or(flags)} +to accept an incorrect (and potentially broken) dependency resolution.` - return explain(expl, color, 4) + - `\n\n${fix}\n\nSee ${fullReport} for a full report.` + return { + explanation: `${explain(expl, color, 4)}\n\n${fix}`, + file: `# npm resolution error report\n\n${explain(expl, false, Infinity)}\n\n${fix}`, + } } module.exports = { diff --git a/lib/utils/log-file.js b/lib/utils/log-file.js index a18eb2878f0be..3351ae6cdd3d5 100644 --- a/lib/utils/log-file.js +++ b/lib/utils/log-file.js @@ -7,16 +7,11 @@ const MiniPass = require('minipass') const fsMiniPass = require('fs-minipass') const fs = require('@npmcli/fs') const log = require('./log-shim') -const runId = require('./run-id') const padZero = (n, length) => n.toString().padStart(length.toString().length, '0') const globify = pattern => pattern.split('\\').join('/') class LogFiles { - // If we write multiple log files we want them all to have the same - // identifier for sorting and matching purposes - #logId = null - // Default to a plain minipass stream so we can buffer // initial writes before we know the cache location #logStream = null @@ -34,16 +29,14 @@ class LogFiles { #fileLogCount = 0 #totalLogCount = 0 - #dir = null + #path = null #logsMax = null #files = [] constructor ({ - id = runId(), maxLogsPerFile = 50_000, maxFilesPerProcess = 5, } = {}) { - this.#logId = id this.#MAX_LOGS_PER_FILE = maxLogsPerFile this.#MAX_FILES_PER_PROCESS = maxFilesPerProcess this.on() @@ -73,10 +66,10 @@ class LogFiles { this.#endStream() } - load ({ dir, logsMax = Infinity } = {}) { + load ({ path, logsMax = Infinity } = {}) { // dir is user configurable and is required to exist so // this can error if the dir is missing or not configured correctly - this.#dir = dir + this.#path = path this.#logsMax = logsMax // Log stream has already ended @@ -84,7 +77,7 @@ class LogFiles { return } - log.verbose('logfile', `logs-max:${logsMax} dir:${dir}`) + log.verbose('logfile', `logs-max:${logsMax} dir:${this.#path}`) // Pipe our initial stream to our new file stream and // set that as the new log logstream for future writes @@ -164,7 +157,7 @@ class LogFiles { } #getLogFilePath (count = '') { - return path.resolve(this.#dir, `${this.#logId}-debug-${count}.log`) + return `${this.#path}debug-${count}.log` } #openLogFile () { diff --git a/lib/utils/run-id.js b/lib/utils/run-id.js deleted file mode 100644 index b7e7cf797da3c..0000000000000 --- a/lib/utils/run-id.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = (d = new Date()) => { - return d.toISOString().replace(/[.:]/g, '_') -} diff --git a/lib/utils/timers.js b/lib/utils/timers.js index d4f8d8bdedd6d..c84bbba3819ab 100644 --- a/lib/utils/timers.js +++ b/lib/utils/timers.js @@ -1,8 +1,6 @@ const EE = require('events') -const { resolve } = require('path') const fs = require('@npmcli/fs') const log = require('./log-shim') -const runId = require('./run-id') // This is an event emiiter but on/off // only listen on a single internal event that gets @@ -10,16 +8,14 @@ const runId = require('./run-id') class Timers extends EE { file = null - #id = null #unfinished = new Map() #finished = {} #onTimeEnd = Symbol('onTimeEnd') #initialListener = null #initialTimer = null - constructor ({ id = runId(), listener = null, start = 'npm' } = {}) { + constructor ({ listener = null, start = 'npm' } = {}) { super() - this.#id = id this.#initialListener = listener this.#initialTimer = start this.#init() @@ -71,9 +67,9 @@ class Timers extends EE { return end } - load ({ dir } = {}) { - if (dir) { - this.file = resolve(dir, `${this.#id}-timing.json`) + load ({ path } = {}) { + if (path) { + this.file = `${path}timing.json` } } @@ -86,10 +82,7 @@ class Timers extends EE { const globalStart = this.started const globalEnd = this.#finished.npm || Date.now() const content = { - metadata: { - id: this.#id, - ...metadata, - }, + metadata, timers: this.#finished, // add any unfinished timers with their relative start/end unfinishedTimers: [...this.#unfinished.entries()].reduce((acc, [name, start]) => { diff --git a/tap-snapshots/test/lib/utils/error-message.js.test.cjs b/tap-snapshots/test/lib/utils/error-message.js.test.cjs index 8e772e8691860..de21a46d2f6e8 100644 --- a/tap-snapshots/test/lib/utils/error-message.js.test.cjs +++ b/tap-snapshots/test/lib/utils/error-message.js.test.cjs @@ -509,7 +509,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-false-cacheDest-false-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-false-cacheDest-false-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -549,7 +549,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-false-cacheDest-true-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-false-cacheDest-true-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -592,7 +592,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-true-cacheDest-false-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-true-cacheDest-false-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -635,7 +635,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-true-cacheDest-true-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-false-loaded-true-cachePath-true-cacheDest-true-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -825,7 +825,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-false-cacheDest-false-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-false-cacheDest-false-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -876,7 +876,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-false-cacheDest-true-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-false-cacheDest-true-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -927,7 +927,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-true-cacheDest-false-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-true-cacheDest-false-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -978,7 +978,7 @@ Array [ ], Array [ "logfile", - "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-true-cacheDest-true-/cache/_logs", + "logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-error-message-eacces-eperm--windows-true-loaded-true-cachePath-true-cacheDest-true-/cache/_logs/{DATE}-", ], Array [ "logfile", @@ -1176,6 +1176,12 @@ Object { "explanation", ], ], + "files": Array [ + Array [ + "eresolve-report.txt", + "report", + ], + ], "summary": Array [ Array [ "ERESOLVE", diff --git a/tap-snapshots/test/lib/utils/exit-handler.js.test.cjs b/tap-snapshots/test/lib/utils/exit-handler.js.test.cjs index edb7edaa5d5f5..4d52c02d97dd3 100644 --- a/tap-snapshots/test/lib/utils/exit-handler.js.test.cjs +++ b/tap-snapshots/test/lib/utils/exit-handler.js.test.cjs @@ -15,7 +15,7 @@ exports[`test/lib/utils/exit-handler.js TAP handles unknown error with logs and 20 verbose argv 21 timing npm:load:setTitle Completed in {TIME}ms 23 timing npm:load:display Completed in {TIME}ms -24 verbose logfile logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-exit-handler-handles-unknown-error-with-logs-and-debug-file/cache/_logs +24 verbose logfile logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-exit-handler-handles-unknown-error-with-logs-and-debug-file/cache/_logs/{DATE}- 25 verbose logfile {CWD}/test/lib/utils/tap-testdir-exit-handler-handles-unknown-error-with-logs-and-debug-file/cache/_logs/{DATE}-debug-0.log 26 timing npm:load:logFile Completed in {TIME}ms 27 timing npm:load:timers Completed in {TIME}ms @@ -47,7 +47,7 @@ verbose title npm verbose argv timing npm:load:setTitle Completed in {TIME}ms timing npm:load:display Completed in {TIME}ms -verbose logfile logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-exit-handler-handles-unknown-error-with-logs-and-debug-file/cache/_logs +verbose logfile logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-exit-handler-handles-unknown-error-with-logs-and-debug-file/cache/_logs/{DATE}- verbose logfile {CWD}/test/lib/utils/tap-testdir-exit-handler-handles-unknown-error-with-logs-and-debug-file/cache/_logs/{DATE}-debug-0.log timing npm:load:logFile Completed in {TIME}ms timing npm:load:timers Completed in {TIME}ms diff --git a/tap-snapshots/test/lib/utils/explain-eresolve.js.test.cjs b/tap-snapshots/test/lib/utils/explain-eresolve.js.test.cjs index 354081d110319..99ad5c0f31e90 100644 --- a/tap-snapshots/test/lib/utils/explain-eresolve.js.test.cjs +++ b/tap-snapshots/test/lib/utils/explain-eresolve.js.test.cjs @@ -29,11 +29,9 @@ node_modules/@isaacs/testing-peer-dep-conflict-chain-c @isaacs/testing-peer-dep-conflict-chain-c@"1" from the root project ` -exports[`test/lib/utils/explain-eresolve.js TAP chain-conflict > report 1`] = ` +exports[`test/lib/utils/explain-eresolve.js TAP chain-conflict > report from color 1`] = ` # npm resolution error report -\${TIME} - While resolving: project@1.2.3 Found: @isaacs/testing-peer-dep-conflict-chain-d@2.0.0 node_modules/@isaacs/testing-peer-dep-conflict-chain-d @@ -45,16 +43,8 @@ node_modules/@isaacs/testing-peer-dep-conflict-chain-c @isaacs/testing-peer-dep-conflict-chain-c@"1" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -Raw JSON explanation object: - -{ - "name": "chain-conflict", - "json": true -} - ` exports[`test/lib/utils/explain-eresolve.js TAP chain-conflict > report with color 1`] = ` @@ -69,10 +59,8 @@ Could not resolve dependency: @isaacs/testing-peer-dep-conflict-chain-c@"1" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP chain-conflict > report with no color 1`] = ` @@ -87,10 +75,8 @@ node_modules/@isaacs/testing-peer-dep-conflict-chain-c @isaacs/testing-peer-dep-conflict-chain-c@"1" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP cycleNested > explain with color, depth of 2 1`] = ` @@ -130,11 +116,9 @@ node_modules/@isaacs/peer-dep-cycle-c @isaacs/peer-dep-cycle-a@"1.x" from the root project ` -exports[`test/lib/utils/explain-eresolve.js TAP cycleNested > report 1`] = ` +exports[`test/lib/utils/explain-eresolve.js TAP cycleNested > report from color 1`] = ` # npm resolution error report -\${TIME} - Found: @isaacs/peer-dep-cycle-c@2.0.0 node_modules/@isaacs/peer-dep-cycle-c @isaacs/peer-dep-cycle-c@"2.x" from the root project @@ -155,14 +139,6 @@ node_modules/@isaacs/peer-dep-cycle-c Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -Raw JSON explanation object: - -{ - "name": "cycleNested", - "json": true -} - ` exports[`test/lib/utils/explain-eresolve.js TAP cycleNested > report with color 1`] = ` @@ -186,8 +162,6 @@ Conflicting peer dependency: @isaacs/peer-dep-cycle-c@1.0.0[2 Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP cycleNested > report with no color 1`] = ` @@ -211,8 +185,6 @@ node_modules/@isaacs/peer-dep-cycle-c Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP eslint-plugin case > explain with color, depth of 2 1`] = ` @@ -255,11 +227,9 @@ node_modules/eslint dev eslint-plugin-eslint-plugin@"^3.1.0" from the root project ` -exports[`test/lib/utils/explain-eresolve.js TAP eslint-plugin case > report 1`] = ` +exports[`test/lib/utils/explain-eresolve.js TAP eslint-plugin case > report from color 1`] = ` # npm resolution error report -\${TIME} - While resolving: eslint-plugin-react@7.24.0 Found: eslint@6.8.0 node_modules/eslint @@ -287,16 +257,8 @@ node_modules/eslint dev eslint-plugin-eslint-plugin@"^3.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -Raw JSON explanation object: - -{ - "name": "eslint-plugin case", - "json": true -} - ` exports[`test/lib/utils/explain-eresolve.js TAP eslint-plugin case > report with color 1`] = ` @@ -319,10 +281,8 @@ Conflicting peer dependency: eslint@7.31.0 dev eslint-plugin-eslint-plugin@"^3.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP eslint-plugin case > report with no color 1`] = ` @@ -345,10 +305,8 @@ node_modules/eslint dev eslint-plugin-eslint-plugin@"^3.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP gatsby > explain with color, depth of 2 1`] = ` @@ -391,11 +349,9 @@ node_modules/ink-box gatsby@"" from the root project ` -exports[`test/lib/utils/explain-eresolve.js TAP gatsby > report 1`] = ` +exports[`test/lib/utils/explain-eresolve.js TAP gatsby > report from color 1`] = ` # npm resolution error report -\${TIME} - While resolving: gatsby-recipes@0.2.31 Found: ink@3.0.0-7 node_modules/ink @@ -421,14 +377,6 @@ node_modules/ink-box Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -Raw JSON explanation object: - -{ - "name": "gatsby", - "json": true -} - ` exports[`test/lib/utils/explain-eresolve.js TAP gatsby > report with color 1`] = ` @@ -456,8 +404,6 @@ Could not resolve dependency: Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP gatsby > report with no color 1`] = ` @@ -485,8 +431,6 @@ node_modules/ink-box Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP no current node, but has current edge > explain with color, depth of 2 1`] = ` @@ -509,11 +453,9 @@ node_modules/eslint-plugin-jsdoc dev eslint-plugin-jsdoc@"^22.1.0" from the root project ` -exports[`test/lib/utils/explain-eresolve.js TAP no current node, but has current edge > report 1`] = ` +exports[`test/lib/utils/explain-eresolve.js TAP no current node, but has current edge > report from color 1`] = ` # npm resolution error report -\${TIME} - While resolving: eslint@7.22.0 Found: dev eslint@"file:." from the root project @@ -523,16 +465,8 @@ node_modules/eslint-plugin-jsdoc dev eslint-plugin-jsdoc@"^22.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -Raw JSON explanation object: - -{ - "name": "no current node, but has current edge", - "json": true -} - ` exports[`test/lib/utils/explain-eresolve.js TAP no current node, but has current edge > report with color 1`] = ` @@ -545,10 +479,8 @@ Could not resolve dependency: dev eslint-plugin-jsdoc@"^22.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP no current node, but has current edge > report with no color 1`] = ` @@ -561,10 +493,8 @@ node_modules/eslint-plugin-jsdoc dev eslint-plugin-jsdoc@"^22.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP no current node, no current edge, idk > explain with color, depth of 2 1`] = ` @@ -591,11 +521,9 @@ node_modules/eslint-plugin-jsdoc dev eslint-plugin-jsdoc@"^22.1.0" from the root project ` -exports[`test/lib/utils/explain-eresolve.js TAP no current node, no current edge, idk > report 1`] = ` +exports[`test/lib/utils/explain-eresolve.js TAP no current node, no current edge, idk > report from color 1`] = ` # npm resolution error report -\${TIME} - While resolving: eslint@7.22.0 Found: peer eslint@"^6.0.0" from eslint-plugin-jsdoc@22.2.0 node_modules/eslint-plugin-jsdoc @@ -607,16 +535,8 @@ node_modules/eslint-plugin-jsdoc dev eslint-plugin-jsdoc@"^22.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -Raw JSON explanation object: - -{ - "name": "no current node, no current edge, idk", - "json": true -} - ` exports[`test/lib/utils/explain-eresolve.js TAP no current node, no current edge, idk > report with color 1`] = ` @@ -631,10 +551,8 @@ Could not resolve dependency: dev eslint-plugin-jsdoc@"^22.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP no current node, no current edge, idk > report with no color 1`] = ` @@ -649,10 +567,8 @@ node_modules/eslint-plugin-jsdoc dev eslint-plugin-jsdoc@"^22.1.0" from the root project Fix the upstream dependency conflict, or retry -this command with --force, or --legacy-peer-deps +this command with --force or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP withShrinkwrap > explain with color, depth of 2 1`] = ` @@ -682,11 +598,9 @@ node_modules/@isaacs/peer-dep-cycle-b @isaacs/peer-dep-cycle-a@"1.x" from the root project ` -exports[`test/lib/utils/explain-eresolve.js TAP withShrinkwrap > report 1`] = ` +exports[`test/lib/utils/explain-eresolve.js TAP withShrinkwrap > report from color 1`] = ` # npm resolution error report -\${TIME} - While resolving: @isaacs/peer-dep-cycle-b@1.0.0 Found: @isaacs/peer-dep-cycle-c@2.0.0 node_modules/@isaacs/peer-dep-cycle-c @@ -702,14 +616,6 @@ node_modules/@isaacs/peer-dep-cycle-b Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -Raw JSON explanation object: - -{ - "name": "withShrinkwrap", - "json": true -} - ` exports[`test/lib/utils/explain-eresolve.js TAP withShrinkwrap > report with color 1`] = ` @@ -728,8 +634,6 @@ Could not resolve dependency: Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` exports[`test/lib/utils/explain-eresolve.js TAP withShrinkwrap > report with no color 1`] = ` @@ -748,6 +652,4 @@ node_modules/@isaacs/peer-dep-cycle-b Fix the upstream dependency conflict, or retry this command with --no-strict-peer-deps, --force, or --legacy-peer-deps to accept an incorrect (and potentially broken) dependency resolution. - -See \${REPORT} for a full report. ` diff --git a/tap-snapshots/test/lib/utils/log-file.js.test.cjs b/tap-snapshots/test/lib/utils/log-file.js.test.cjs index 7a39184939026..912a4365f6810 100644 --- a/tap-snapshots/test/lib/utils/log-file.js.test.cjs +++ b/tap-snapshots/test/lib/utils/log-file.js.test.cjs @@ -6,7 +6,7 @@ */ 'use strict' exports[`test/lib/utils/log-file.js TAP snapshot > must match snapshot 1`] = ` -0 verbose logfile logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-log-file-snapshot +0 verbose logfile logs-max:10 dir:{CWD}/test/lib/utils/tap-testdir-log-file-snapshot/{DATE}- 1 silly logfile done cleaning log files 2 error no prefix 3 error prefix with prefix diff --git a/test/lib/utils/error-message.js b/test/lib/utils/error-message.js index 3fec501ef9ff7..29753c3039f36 100644 --- a/test/lib/utils/error-message.js +++ b/test/lib/utils/error-message.js @@ -4,6 +4,12 @@ const { load: _loadMockNpm } = require('../../fixtures/mock-npm.js') const mockGlobals = require('../../fixtures/mock-globals.js') const { cleanCwd, cleanDate } = require('../../fixtures/clean-snapshot.js') +t.formatSnapshot = (p) => { + if (Array.isArray(p.files) && !p.files.length) { + delete p.files + } + return p +} t.cleanSnapshot = p => cleanDate(cleanCwd(p)) mockGlobals(t, { @@ -420,7 +426,7 @@ t.test('bad platform', async t => { }) t.test('explain ERESOLVE errors', async t => { - const npm = await loadMockNpm(t) + const { npm, ...rest } = await loadMockNpm(t) const EXPLAIN_CALLED = [] const er = Object.assign(new Error('could not resolve'), { @@ -428,19 +434,16 @@ t.test('explain ERESOLVE errors', async t => { }) t.matchSnapshot(errorMessage(er, { - ...npm, + npm, + ...rest, mocks: { '../../../lib/utils/explain-eresolve.js': { report: (...args) => { EXPLAIN_CALLED.push(args) - return 'explanation' + return { explanation: 'explanation', file: 'report' } }, }, }, })) - t.match(EXPLAIN_CALLED, [[ - er, - false, - path.resolve(npm.cache, 'eresolve-report.txt'), - ]]) + t.match(EXPLAIN_CALLED, [[er, false]]) }) diff --git a/test/lib/utils/exit-handler.js b/test/lib/utils/exit-handler.js index de4ce61aaa6af..7d2e7e061c365 100644 --- a/test/lib/utils/exit-handler.js +++ b/test/lib/utils/exit-handler.js @@ -1,5 +1,7 @@ const t = require('tap') const os = require('os') +const fs = require('@npmcli/fs') +const { join } = require('path') const EventEmitter = require('events') const { format } = require('../../../lib/utils/log-file') const { load: loadMockNpm } = require('../../fixtures/mock-npm') @@ -45,7 +47,7 @@ mockGlobals(t, { }), }, { replace: true }) -const mockExitHandler = async (t, { init, load, testdir, config, globals, mocks } = {}) => { +const mockExitHandler = async (t, { init, load, testdir, config, mocks, files } = {}) => { const errors = [] const { npm, logMocks, ...rest } = await loadMockNpm(t, { @@ -69,9 +71,9 @@ const mockExitHandler = async (t, { init, load, testdir, config, globals, mocks const exitHandler = t.mock('../../../lib/utils/exit-handler.js', { '../../../lib/utils/error-message.js': (err) => ({ - ...err, summary: [['ERR SUMMARY', err.message]], detail: [['ERR DETAIL', err.message]], + ...(files ? { files } : {}), }), os: { type: () => 'Foo', @@ -311,6 +313,53 @@ t.test('log file error', async (t) => { t.match(logs.error.filter(([t]) => t === ''), [['', `error writing to the directory`]]) }) +t.test('files from error message', async (t) => { + const { exitHandler, logs, cache } = await mockExitHandler(t, { + files: [ + ['error-file.txt', '# error file content'], + ], + }) + + await exitHandler(err('Error message')) + + const logFiles = fs.readdirSync(join(cache, '_logs')) + const errorFileName = logFiles.find(f => f.endsWith('error-file.txt')) + const errorFile = fs.readFileSync(join(cache, '_logs', errorFileName)).toString() + + const [log] = logs.error.filter(([t]) => t === '') + + t.match(log[1], /For a full report see:\n.*-error-file\.txt/) + t.match(errorFile, '# error file content') + t.match(errorFile, 'Log files:') +}) + +t.test('files from error message with error', async (t) => { + const { exitHandler, logs } = await mockExitHandler(t, { + config: { + 'logs-dir': 'LOGS_DIR', + }, + files: [ + ['error-file.txt', '# error file content'], + ], + mocks: { + '@npmcli/fs': { + ...fs, + writeFileSync: (dir) => { + if (dir.includes('LOGS_DIR') && dir.endsWith('error-file.txt')) { + throw new Error('err') + } + }, + }, + }, + }) + + await exitHandler(err('Error message')) + + const [log] = logs.warn.filter(([t]) => t === '') + + t.match(log[1], /Could not write error message to.*error-file\.txt.*err/) +}) + t.test('timing with no error', async (t) => { const { exitHandler, timingFile, npm, logs } = await mockExitHandler(t, { config: { timing: true }, diff --git a/test/lib/utils/explain-eresolve.js b/test/lib/utils/explain-eresolve.js index f9710ee889ab1..2c1fed77899e9 100644 --- a/test/lib/utils/explain-eresolve.js +++ b/test/lib/utils/explain-eresolve.js @@ -1,41 +1,32 @@ const t = require('tap') -const npm = {} -const { explain, report } = require('../../../lib/utils/explain-eresolve.js') -const { statSync, readFileSync, unlinkSync } = require('fs') -// strip out timestamps from reports -const read = f => readFileSync(f, 'utf8') - .replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, '${TIME}') - const { resolve } = require('path') +const { explain, report } = require('../../../lib/utils/explain-eresolve.js') const cases = require('../../fixtures/eresolve-explanations.js') +const { cleanDate } = require('../../fixtures/clean-snapshot.js') for (const [name, expl] of Object.entries(cases)) { // no sense storing the whole contents of each object in the snapshot // we can trust that JSON.stringify still works just fine. - expl.toJSON = () => { - return { name, json: true } - } + expl.toJSON = () => ({ name, json: true }) t.test(name, t => { - npm.cache = t.testdir() - const reportFile = resolve(npm.cache, 'eresolve-report.txt') - t.cleanSnapshot = str => str.split(reportFile).join('${REPORT}') + const dir = t.testdir() + const fileReport = resolve(dir, 'eresolve-report.txt') + const opts = { file: fileReport, date: new Date().toISOString() } + + t.cleanSnapshot = str => cleanDate(str.split(fileReport).join('${REPORT}')) - npm.color = true - t.matchSnapshot(report(expl, true, reportFile), 'report with color') - const reportData = read(reportFile) - t.matchSnapshot(reportData, 'report') - unlinkSync(reportFile) + const color = report(expl, true, opts) + t.matchSnapshot(color.explanation, 'report with color') + t.matchSnapshot(color.file, 'report from color') - t.matchSnapshot(report(expl, false, reportFile), 'report with no color') - t.equal(read(reportFile), reportData, 'same report written for object') - unlinkSync(reportFile) + const noColor = report(expl, false, opts) + t.matchSnapshot(noColor.explanation, 'report with no color') + t.equal(noColor.file, color.file, 'same report written for object') t.matchSnapshot(explain(expl, true, 2), 'explain with color, depth of 2') - t.throws(() => statSync(reportFile), { code: 'ENOENT' }, 'no report') t.matchSnapshot(explain(expl, false, 6), 'explain with no color, depth of 6') - t.throws(() => statSync(reportFile), { code: 'ENOENT' }, 'no report') t.end() }) diff --git a/test/lib/utils/log-file.js b/test/lib/utils/log-file.js index 3c7bb3fe8324a..4be5231c1c4fa 100644 --- a/test/lib/utils/log-file.js +++ b/test/lib/utils/log-file.js @@ -6,11 +6,11 @@ const os = require('os') const fsMiniPass = require('fs-minipass') const rimraf = require('rimraf') const LogFile = require('../../../lib/utils/log-file.js') -const runId = require('../../../lib/utils/run-id.js') -const { cleanCwd } = require('../../fixtures/clean-snapshot') +const { cleanCwd, cleanDate } = require('../../fixtures/clean-snapshot') -t.cleanSnapshot = (path) => cleanCwd(path) +t.cleanSnapshot = (path) => cleanDate(cleanCwd(path)) +const getId = (d = new Date()) => d.toISOString().replace(/[.:]/g, '_') const last = arr => arr[arr.length - 1] const range = (n) => Array.from(Array(n).keys()) const makeOldLogs = (count, oldStyle) => { @@ -20,7 +20,7 @@ const makeOldLogs = (count, oldStyle) => { return range(oldStyle ? count : (count / 2)).reduce((acc, i) => { const cloneDate = new Date(d.getTime()) cloneDate.setSeconds(i) - const dateId = runId(cloneDate) + const dateId = getId(cloneDate) if (oldStyle) { acc[`${dateId}-debug.log`] = 'hello' } else { @@ -42,10 +42,15 @@ const cleanErr = (message) => { const loadLogFile = async (t, { buffer = [], mocks, testdir = {}, ...options } = {}) => { const root = t.testdir(testdir) + const MockLogFile = t.mock('../../../lib/utils/log-file.js', mocks) const logFile = new MockLogFile(Object.keys(options).length ? options : undefined) + buffer.forEach((b) => logFile.log(...b)) - await logFile.load({ dir: root, ...options }) + + const id = getId() + await logFile.load({ path: path.join(root, `${id}-`), ...options }) + t.teardown(() => logFile.off()) return { root, diff --git a/test/lib/utils/timers.js b/test/lib/utils/timers.js index 259ecd5dd4893..23d8eb6e2cafe 100644 --- a/test/lib/utils/timers.js +++ b/test/lib/utils/timers.js @@ -68,11 +68,11 @@ t.test('finish unstarted timer', async (t) => { }) t.test('writes file', async (t) => { - const { timers } = mockTimers(t, { id: 'TIMING_FILE' }) + const { timers } = mockTimers(t) const dir = t.testdir() process.emit('time', 'foo') process.emit('timeEnd', 'foo') - timers.load({ dir }) + timers.load({ path: resolve(dir, `TIMING_FILE-`) }) timers.writeFile({ some: 'data' }) const data = JSON.parse(fs.readFileSync(resolve(dir, 'TIMING_FILE-timing.json'))) t.match(data, { @@ -88,7 +88,7 @@ t.test('fails to write file', async (t) => { const { logs, timers } = mockTimers(t) const dir = t.testdir() - timers.load({ dir: join(dir, 'does', 'not', 'exist') }) + timers.load({ path: join(dir, 'does', 'not', 'exist') }) timers.writeFile() t.match(logs.warn, [['timing', 'could not write timing file']]) From 7e6b8244b0eef9ee60e293859f4147e577cb2f3b Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Thu, 22 Sep 2022 23:01:59 -0700 Subject: [PATCH 3/9] chore(make): docs should come before test during publish docs needs to rebuild cmark-gfm which needs to be present in order to run tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 484c5152f0292..5f66ee7174f16 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ prune: deps node bin/npm-cli.js prune --omit=dev --no-save --no-audit --no-fund node scripts/git-dirty.js -publish: gitclean ls-ok link lint-all test-all docs prune +publish: gitclean ls-ok link docs lint-all test-all prune node bin/npm-cli.js publish --tag=$(PUBLISHTAG) release: gitclean ls-ok docs prune From bc2155247d00b7a868c414f4bc86993069b035f9 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 28 Sep 2022 15:14:34 -0700 Subject: [PATCH 4/9] deps: npm-package-arg@9.1.2 Fixes #4994 --- node_modules/npm-package-arg/lib/npa.js | 6 ++++-- node_modules/npm-package-arg/package.json | 26 +++++++++++++++++------ package-lock.json | 8 +++---- package.json | 2 +- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/node_modules/npm-package-arg/lib/npa.js b/node_modules/npm-package-arg/lib/npa.js index 61fee0783ecc7..cd1932b0bc78b 100644 --- a/node_modules/npm-package-arg/lib/npa.js +++ b/node_modules/npm-package-arg/lib/npa.js @@ -241,8 +241,10 @@ function fromFile (res, where) { rawNoPrefix = rawSpec.replace(/^file:/, '') } // turn file:/../foo into file:../foo - if (/^\/\.\.?(\/|$)/.test(rawNoPrefix)) { - const rawSpec = res.rawSpec.replace(/^file:\//, 'file:') + // for 1, 2 or 3 leading slashes since we attempted + // in the previous step to make it a file protocol url with a leading slash + if (/^\/{1,3}\.\.?(\/|$)/.test(rawNoPrefix)) { + const rawSpec = res.rawSpec.replace(/^file:\/{1,3}/, 'file:') resolvedUrl = new url.URL(rawSpec, `file://${path.resolve(where)}/`) specUrl = new url.URL(rawSpec) rawNoPrefix = rawSpec.replace(/^file:/, '') diff --git a/node_modules/npm-package-arg/package.json b/node_modules/npm-package-arg/package.json index b9729db5d4f14..eddc6bbbbb8f8 100644 --- a/node_modules/npm-package-arg/package.json +++ b/node_modules/npm-package-arg/package.json @@ -1,6 +1,6 @@ { "name": "npm-package-arg", - "version": "9.1.0", + "version": "9.1.2", "description": "Parse the things that can be arguments to `npm install`", "main": "./lib/npa.js", "directories": { @@ -18,13 +18,10 @@ }, "devDependencies": { "@npmcli/eslint-config": "^3.0.1", - "@npmcli/template-oss": "3.5.0", + "@npmcli/template-oss": "4.3.2", "tap": "^16.0.1" }, "scripts": { - "preversion": "npm test", - "postversion": "npm publish", - "prepublishOnly": "git push origin --follow-tags", "test": "tap", "snap": "tap", "npmclilint": "npmcli-lint", @@ -49,10 +46,25 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" }, "tap": { - "branches": 97 + "branches": 97, + "nyc-arg": [ + "--exclude", + "tap-snapshots/**" + ] }, "templateOSS": { "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "3.5.0" + "version": "4.3.2", + "releaseBranches": [ + "v9" + ], + "ciVersions": [ + "12.13.0", + "12.x", + "14.15.0", + "14.x", + "16.0.0", + "16.x" + ] } } diff --git a/package-lock.json b/package-lock.json index 43f364e27eb8c..2441009d92e20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -135,7 +135,7 @@ "nopt": "^6.0.0", "npm-audit-report": "^3.0.0", "npm-install-checks": "^5.0.0", - "npm-package-arg": "^9.1.0", + "npm-package-arg": "^9.1.2", "npm-pick-manifest": "^7.0.2", "npm-profile": "^6.2.0", "npm-registry-fetch": "^13.3.1", @@ -8010,9 +8010,9 @@ "license": "ISC" }, "node_modules/npm-package-arg": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-9.1.0.tgz", - "integrity": "sha512-4J0GL+u2Nh6OnhvUKXRr2ZMG4lR8qtLp+kv7UiV00Y+nGiSxtttCyIRHCt5L5BNkXQld/RceYItau3MDOoGiBw==", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-9.1.2.tgz", + "integrity": "sha512-pzd9rLEx4TfNJkovvlBSLGhq31gGu2QDexFPWT19yCDh0JgnRhlBLNo5759N0AJmBk+kQ9Y/hXoLnlgFD+ukmg==", "inBundle": true, "dependencies": { "hosted-git-info": "^5.0.0", diff --git a/package.json b/package.json index 7d0be98833116..250f519e11994 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,7 @@ "nopt": "^6.0.0", "npm-audit-report": "^3.0.0", "npm-install-checks": "^5.0.0", - "npm-package-arg": "^9.1.0", + "npm-package-arg": "^9.1.2", "npm-pick-manifest": "^7.0.2", "npm-profile": "^6.2.0", "npm-registry-fetch": "^13.3.1", From 525654e957a80c7f47472e18240e3c8d94e0568f Mon Sep 17 00:00:00 2001 From: Gar Date: Tue, 27 Sep 2022 09:13:10 -0700 Subject: [PATCH 5/9] feat: default access to `public` BREAKING CHANGE: The default value of `access` is now `public` --- workspaces/libnpmpublish/README.md | 4 +- workspaces/libnpmpublish/lib/publish.js | 5 +- workspaces/libnpmpublish/test/publish.js | 62 +++++++++++++++++++++++- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/workspaces/libnpmpublish/README.md b/workspaces/libnpmpublish/README.md index 85fb73e52fdbe..9c9c61d4b5965 100644 --- a/workspaces/libnpmpublish/README.md +++ b/workspaces/libnpmpublish/README.md @@ -44,8 +44,8 @@ A couple of options of note: defaults to `latest`. * `opts.access` - tells the registry whether this package should be - published as public or restricted. Only applies to scoped packages, which - default to restricted. + published as `public` or `restricted`. Only applies to scoped + packages. Defaults to `public`. * `opts.token` - can be passed in and will be used as the authentication token for the registry. For other ways to pass in auth details, see the diff --git a/workspaces/libnpmpublish/lib/publish.js b/workspaces/libnpmpublish/lib/publish.js index 75b764c98963f..7d01fabf1f2b4 100644 --- a/workspaces/libnpmpublish/lib/publish.js +++ b/workspaces/libnpmpublish/lib/publish.js @@ -17,10 +17,9 @@ Remove the 'private' field from the package.json to publish it.`), // spec is used to pick the appropriate registry/auth combo const spec = npa.resolve(manifest.name, manifest.version) opts = { - defaultTag: 'latest', - // if scoped, restricted by default - access: spec.scope ? 'restricted' : 'public', + access: 'public', algorithms: ['sha512'], + defaultTag: 'latest', ...opts, spec, } diff --git a/workspaces/libnpmpublish/test/publish.js b/workspaces/libnpmpublish/test/publish.js index fdd20f899430e..c696e82b22544 100644 --- a/workspaces/libnpmpublish/test/publish.js +++ b/workspaces/libnpmpublish/test/publish.js @@ -79,7 +79,66 @@ t.test('basic publish', async t => { t.ok(ret, 'publish succeeded') }) -t.test('scoped publish', async t => { +t.test('scoped publish - default access', async t => { + const manifest = { + name: '@claudiahdz/libnpmpublish', + version: '1.0.0', + description: 'some stuff', + } + + const tarData = await pack(`file:${testDir}`, { ...OPTS }) + const shasum = crypto.createHash('sha1').update(tarData).digest('hex') + const integrity = ssri.fromData(tarData, { algorithms: ['sha512'] }) + const packument = { + _id: '@claudiahdz/libnpmpublish', + name: '@claudiahdz/libnpmpublish', + description: 'some stuff', + 'dist-tags': { + latest: '1.0.0', + }, + versions: { + '1.0.0': { + _id: '@claudiahdz/libnpmpublish@1.0.0', + _nodeVersion: process.versions.node, + _npmVersion: '6.13.7', + name: '@claudiahdz/libnpmpublish', + version: '1.0.0', + description: 'some stuff', + dist: { + shasum, + integrity: integrity.toString(), + tarball: 'http://mock.reg/@claudiahdz/libnpmpublish/' + + '-/@claudiahdz/libnpmpublish-1.0.0.tgz', + }, + }, + }, + access: 'public', + _attachments: { + '@claudiahdz/libnpmpublish-1.0.0.tgz': { + content_type: 'application/octet-stream', + data: tarData.toString('base64'), + length: tarData.length, + }, + }, + } + + const srv = tnock(t, REG) + srv.put('/@claudiahdz%2flibnpmpublish', body => { + t.same(body, packument, 'posted packument matches expectations') + return true + }, { + authorization: 'Bearer deadbeef', + }).reply(201, {}) + + const ret = await publish(manifest, tarData, { + ...OPTS, + npmVersion: '6.13.7', + token: 'deadbeef', + }) + t.ok(ret, 'publish succeeded') +}) + +t.test('scoped publish - restricted access', async t => { const manifest = { name: '@claudiahdz/libnpmpublish', version: '1.0.0', @@ -132,6 +191,7 @@ t.test('scoped publish', async t => { const ret = await publish(manifest, tarData, { ...OPTS, + access: 'restricted', npmVersion: '6.13.7', token: 'deadbeef', }) From f0e758494698d9dd8a58d07bf71c87608c36869e Mon Sep 17 00:00:00 2001 From: Gar Date: Wed, 28 Sep 2022 10:56:40 -0700 Subject: [PATCH 6/9] docs: update docs/logging for new --access default --- docs/content/commands/npm-publish.md | 20 ++-- docs/content/using-npm/config.md | 20 ++-- lib/commands/publish.js | 11 +- lib/utils/config/definitions.js | 22 ++-- .../test/lib/commands/publish.js.test.cjs | 100 +++++++++++++++++- .../lib/utils/config/definitions.js.test.cjs | 20 ++-- .../lib/utils/config/describe-all.js.test.cjs | 20 ++-- test/lib/commands/publish.js | 64 ++++++++++- 8 files changed, 217 insertions(+), 60 deletions(-) diff --git a/docs/content/commands/npm-publish.md b/docs/content/commands/npm-publish.md index 536d04988e684..0c1b777c881bb 100644 --- a/docs/content/commands/npm-publish.md +++ b/docs/content/commands/npm-publish.md @@ -118,19 +118,19 @@ tarball that will be compared with the local files by default. #### `access` -* Default: 'restricted' for scoped packages, 'public' for unscoped packages +* Default: 'public' for new packages, existing packages it will not change the + current level * Type: null, "restricted", or "public" -When publishing scoped packages, the access level defaults to `restricted`. -If you want your scoped package to be publicly viewable (and installable) -set `--access=public`. The only valid values for `access` are `public` and -`restricted`. Unscoped packages _always_ have an access level of `public`. +If do not want your scoped package to be publicly viewable (and installable) +set `--access=restricted`. -Note: Using the `--access` flag on the `npm publish` command will only set -the package access level on the initial publish of the package. Any -subsequent `npm publish` commands using the `--access` flag will not have an -effect to the access level. To make changes to the access level after the -initial publish use `npm access`. +Unscoped packages can not be set to `restricted`. + +Note: This defaults to not changing the current access level for existing +packages. Specifying a value of `restricted` or `public` during publish will +change the access for an existing package the same way that `npm access set +status` would. diff --git a/docs/content/using-npm/config.md b/docs/content/using-npm/config.md index 2de35fa2a46c3..e5d9d081feb4a 100644 --- a/docs/content/using-npm/config.md +++ b/docs/content/using-npm/config.md @@ -151,19 +151,19 @@ safer to use a registry-provided authentication bearer token stored in the #### `access` -* Default: 'restricted' for scoped packages, 'public' for unscoped packages +* Default: 'public' for new packages, existing packages it will not change the + current level * Type: null, "restricted", or "public" -When publishing scoped packages, the access level defaults to `restricted`. -If you want your scoped package to be publicly viewable (and installable) -set `--access=public`. The only valid values for `access` are `public` and -`restricted`. Unscoped packages _always_ have an access level of `public`. +If do not want your scoped package to be publicly viewable (and installable) +set `--access=restricted`. -Note: Using the `--access` flag on the `npm publish` command will only set -the package access level on the initial publish of the package. Any -subsequent `npm publish` commands using the `--access` flag will not have an -effect to the access level. To make changes to the access level after the -initial publish use `npm access`. +Unscoped packages can not be set to `restricted`. + +Note: This defaults to not changing the current access level for existing +packages. Specifying a value of `restricted` or `public` during publish will +change the access for an existing package the same way that `npm access set +status` would. diff --git a/lib/commands/publish.js b/lib/commands/publish.js index 64b6dfc513c95..1f281a0bb1d4f 100644 --- a/lib/commands/publish.js +++ b/lib/commands/publish.js @@ -114,10 +114,13 @@ class Publish extends BaseCommand { } } - log.notice( - '', - `Publishing to ${outputRegistry} with tag ${defaultTag}${dryRun ? ' (dry-run)' : ''}` - ) + const access = opts.access === null ? 'default' : opts.access + let msg = `Publishing to ${outputRegistry} with tag ${defaultTag} and ${access} access` + if (dryRun) { + msg = `${msg} (dry-run)` + } + + log.notice('', msg) if (!dryRun) { await otplease(this.npm, opts, opts => libpub(manifest, tarballData, opts)) diff --git a/lib/utils/config/definitions.js b/lib/utils/config/definitions.js index ae38efc32b5ae..5f74eb03b14c8 100644 --- a/lib/utils/config/definitions.js +++ b/lib/utils/config/definitions.js @@ -160,21 +160,19 @@ define('_auth', { define('access', { default: null, defaultDescription: ` - 'restricted' for scoped packages, 'public' for unscoped packages + 'public' for new packages, existing packages it will not change the current level `, type: [null, 'restricted', 'public'], description: ` - When publishing scoped packages, the access level defaults to - \`restricted\`. If you want your scoped package to be publicly viewable - (and installable) set \`--access=public\`. The only valid values for - \`access\` are \`public\` and \`restricted\`. Unscoped packages _always_ - have an access level of \`public\`. - - Note: Using the \`--access\` flag on the \`npm publish\` command will only - set the package access level on the initial publish of the package. Any - subsequent \`npm publish\` commands using the \`--access\` flag will not - have an effect to the access level. To make changes to the access level - after the initial publish use \`npm access\`. + If do not want your scoped package to be publicly viewable (and + installable) set \`--access=restricted\`. + + Unscoped packages can not be set to \`restricted\`. + + Note: This defaults to not changing the current access level for existing + packages. Specifying a value of \`restricted\` or \`public\` during + publish will change the access for an existing package the same way that + \`npm access set status\` would. `, flatten, }) diff --git a/tap-snapshots/test/lib/commands/publish.js.test.cjs b/tap-snapshots/test/lib/commands/publish.js.test.cjs index 3b215960fa37e..28211f7794cba 100644 --- a/tap-snapshots/test/lib/commands/publish.js.test.cjs +++ b/tap-snapshots/test/lib/commands/publish.js.test.cjs @@ -51,7 +51,7 @@ Array [ ], Array [ "", - "Publishing to https://registry.npmjs.org/ with tag latest (dry-run)", + "Publishing to https://registry.npmjs.org/ with tag latest and default access (dry-run)", ], ] ` @@ -72,7 +72,7 @@ exports[`test/lib/commands/publish.js TAP json > must match snapshot 1`] = ` Array [ Array [ "", - "Publishing to https://registry.npmjs.org/ with tag latest", + "Publishing to https://registry.npmjs.org/ with tag latest and default access", ], ] ` @@ -112,6 +112,53 @@ Array [ ] ` +exports[`test/lib/commands/publish.js TAP public access > must match snapshot 1`] = ` +Array [ + Array [ + "", + ], + Array [ + "", + "package: @npm/test-package@1.0.0", + ], + Array [ + "=== Tarball Contents ===", + ], + Array [ + "", + "55B package.json", + ], + Array [ + "=== Tarball Details ===", + ], + Array [ + "", + String( + name: @npm/test-package + version: 1.0.0 + filename: @npm/test-package-1.0.0.tgz + package size: 147 B + unpacked size: 55 B + shasum:{sha} + integrity:{sha} + total files: 1 + ), + ], + Array [ + "", + "", + ], + Array [ + "", + "Publishing to https://registry.npmjs.org/ with tag latest and public access", + ], +] +` + +exports[`test/lib/commands/publish.js TAP public access > new package version 1`] = ` ++ @npm/test-package@1.0.0 +` + exports[`test/lib/commands/publish.js TAP re-loads publishConfig.registry if added during script process > new package version 1`] = ` + test-package@1.0.0 ` @@ -120,6 +167,53 @@ exports[`test/lib/commands/publish.js TAP respects publishConfig.registry, runs ` +exports[`test/lib/commands/publish.js TAP restricted access > must match snapshot 1`] = ` +Array [ + Array [ + "", + ], + Array [ + "", + "package: @npm/test-package@1.0.0", + ], + Array [ + "=== Tarball Contents ===", + ], + Array [ + "", + "55B package.json", + ], + Array [ + "=== Tarball Details ===", + ], + Array [ + "", + String( + name: @npm/test-package + version: 1.0.0 + filename: @npm/test-package-1.0.0.tgz + package size: 147 B + unpacked size: 55 B + shasum:{sha} + integrity:{sha} + total files: 1 + ), + ], + Array [ + "", + "", + ], + Array [ + "", + "Publishing to https://registry.npmjs.org/ with tag latest and restricted access", + ], +] +` + +exports[`test/lib/commands/publish.js TAP restricted access > new package version 1`] = ` ++ @npm/test-package@1.0.0 +` + exports[`test/lib/commands/publish.js TAP scoped _auth config scoped registry > new package version 1`] = ` + @npm/test-package@1.0.0 ` @@ -165,7 +259,7 @@ Array [ ], Array [ "", - "Publishing to https://registry.npmjs.org/ with tag latest", + "Publishing to https://registry.npmjs.org/ with tag latest and default access", ], ] ` diff --git a/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs b/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs index df9313270d056..1005b9e4df112 100644 --- a/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs +++ b/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs @@ -180,19 +180,19 @@ safer to use a registry-provided authentication bearer token stored in the exports[`test/lib/utils/config/definitions.js TAP > config description for access 1`] = ` #### \`access\` -* Default: 'restricted' for scoped packages, 'public' for unscoped packages +* Default: 'public' for new packages, existing packages it will not change the + current level * Type: null, "restricted", or "public" -When publishing scoped packages, the access level defaults to \`restricted\`. -If you want your scoped package to be publicly viewable (and installable) -set \`--access=public\`. The only valid values for \`access\` are \`public\` and -\`restricted\`. Unscoped packages _always_ have an access level of \`public\`. +If do not want your scoped package to be publicly viewable (and installable) +set \`--access=restricted\`. -Note: Using the \`--access\` flag on the \`npm publish\` command will only set -the package access level on the initial publish of the package. Any -subsequent \`npm publish\` commands using the \`--access\` flag will not have an -effect to the access level. To make changes to the access level after the -initial publish use \`npm access\`. +Unscoped packages can not be set to \`restricted\`. + +Note: This defaults to not changing the current access level for existing +packages. Specifying a value of \`restricted\` or \`public\` during publish will +change the access for an existing package the same way that \`npm access set +status\` would. ` exports[`test/lib/utils/config/definitions.js TAP > config description for all 1`] = ` diff --git a/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs b/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs index 89ca7c952b182..e14b88464237c 100644 --- a/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs +++ b/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs @@ -24,19 +24,19 @@ safer to use a registry-provided authentication bearer token stored in the #### \`access\` -* Default: 'restricted' for scoped packages, 'public' for unscoped packages +* Default: 'public' for new packages, existing packages it will not change the + current level * Type: null, "restricted", or "public" -When publishing scoped packages, the access level defaults to \`restricted\`. -If you want your scoped package to be publicly viewable (and installable) -set \`--access=public\`. The only valid values for \`access\` are \`public\` and -\`restricted\`. Unscoped packages _always_ have an access level of \`public\`. +If do not want your scoped package to be publicly viewable (and installable) +set \`--access=restricted\`. -Note: Using the \`--access\` flag on the \`npm publish\` command will only set -the package access level on the initial publish of the package. Any -subsequent \`npm publish\` commands using the \`--access\` flag will not have an -effect to the access level. To make changes to the access level after the -initial publish use \`npm access\`. +Unscoped packages can not be set to \`restricted\`. + +Note: This defaults to not changing the current access level for existing +packages. Specifying a value of \`restricted\` or \`public\` during publish will +change the access for an existing package the same way that \`npm access set +status\` would. diff --git a/test/lib/commands/publish.js b/test/lib/commands/publish.js index 16b79df532d82..995abff88c2c1 100644 --- a/test/lib/commands/publish.js +++ b/test/lib/commands/publish.js @@ -28,7 +28,7 @@ t.cleanSnapshot = data => { t.test('respects publishConfig.registry, runs appropriate scripts', async t => { const { npm, joinedOutput, prefix } = await loadMockNpm(t, { config: { - loglevel: 'silent', // prevent scripts from leaking to stdout during the test + loglevel: 'silent', [`${alternateRegistry.slice(6)}/:_authToken`]: 'test-other-token', }, prefixDir: { @@ -730,3 +730,65 @@ t.test('scoped _auth config scoped registry', async t => { await npm.exec('publish', []) t.matchSnapshot(joinedOutput(), 'new package version') }) + +t.test('restricted access', async t => { + const spec = npa('@npm/test-package') + const { npm, joinedOutput, logs } = await loadMockNpm(t, { + config: { + ...auth, + access: 'restricted', + }, + prefixDir: { + 'package.json': JSON.stringify({ + name: '@npm/test-package', + version: '1.0.0', + }, null, 2), + }, + globals: ({ prefix }) => ({ + 'process.cwd': () => prefix, + }), + }) + const registry = new MockRegistry({ + tap: t, + registry: npm.config.get('registry'), + authorization: token, + }) + registry.nock.put(`/${spec.escapedName}`, body => { + t.equal(body.access, 'restricted', 'access is explicitly set to restricted') + return true + }).reply(200, {}) + await npm.exec('publish', []) + t.matchSnapshot(joinedOutput(), 'new package version') + t.matchSnapshot(logs.notice) +}) + +t.test('public access', async t => { + const spec = npa('@npm/test-package') + const { npm, joinedOutput, logs } = await loadMockNpm(t, { + config: { + ...auth, + access: 'public', + }, + prefixDir: { + 'package.json': JSON.stringify({ + name: '@npm/test-package', + version: '1.0.0', + }, null, 2), + }, + globals: ({ prefix }) => ({ + 'process.cwd': () => prefix, + }), + }) + const registry = new MockRegistry({ + tap: t, + registry: npm.config.get('registry'), + authorization: token, + }) + registry.nock.put(`/${spec.escapedName}`, body => { + t.equal(body.access, 'public', 'access is explicitly set to public') + return true + }).reply(200, {}) + await npm.exec('publish', []) + t.matchSnapshot(joinedOutput(), 'new package version') + t.matchSnapshot(logs.notice) +}) From 3ae796d937bd36a5b1b9fd6e9e8473b4f2ddc32d Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Wed, 28 Sep 2022 12:27:58 -0700 Subject: [PATCH 7/9] feat: implement new `npm-packlist` behavior This also lands the latest `pacote` which now requires passing in an `Arborist` constructor for use in loading the package tree that gets passed to `npm-packlist`. BREAKING CHANGE: `npm pack` now follows a strict order of operations when applying ignore rules. If a files array is present in the package.json, then rules in .gitignore and .npmignore files from the root will be ignored. --- DEPENDENCIES.md | 20 +- lib/commands/cache.js | 3 +- lib/package-url-cmd.js | 2 + .../@npmcli/metavuln-calculator/package.json | 19 +- node_modules/npm-bundled/LICENSE | 15 - node_modules/npm-bundled/lib/index.js | 254 ------ .../npm-normalize-package-bin/LICENSE | 15 - .../npm-normalize-package-bin/lib/index.js | 64 -- .../npm-normalize-package-bin/package.json | 41 - node_modules/npm-bundled/package.json | 47 -- node_modules/npm-packlist/bin/index.js | 39 - node_modules/npm-packlist/lib/index.js | 766 ++++++++---------- .../npm-normalize-package-bin/LICENSE | 15 - .../npm-normalize-package-bin/lib/index.js | 64 -- .../npm-normalize-package-bin/package.json | 41 - node_modules/npm-packlist/package.json | 29 +- node_modules/pacote/lib/dir.js | 15 +- node_modules/pacote/lib/git.js | 6 + node_modules/pacote/lib/registry.js | 1 - node_modules/pacote/package.json | 22 +- package-lock.json | 80 +- package.json | 2 +- tap-snapshots/test/lib/utils/tar.js.test.cjs | 16 +- test/fixtures/mock-registry.js | 3 +- test/lib/commands/publish.js | 3 +- test/lib/utils/tar.js | 9 +- .../arborist/lib/arborist/build-ideal-tree.js | 1 + workspaces/arborist/lib/arborist/reify.js | 1 + workspaces/arborist/package.json | 4 +- .../registry-mocks/fetch-lock-contents.js | 3 +- workspaces/libnpmdiff/lib/tarball.js | 6 +- workspaces/libnpmdiff/package.json | 3 +- workspaces/libnpmexec/package.json | 2 +- workspaces/libnpmpack/lib/index.js | 2 + workspaces/libnpmpack/package.json | 3 +- 35 files changed, 481 insertions(+), 1135 deletions(-) delete mode 100644 node_modules/npm-bundled/LICENSE delete mode 100644 node_modules/npm-bundled/lib/index.js delete mode 100644 node_modules/npm-bundled/node_modules/npm-normalize-package-bin/LICENSE delete mode 100644 node_modules/npm-bundled/node_modules/npm-normalize-package-bin/lib/index.js delete mode 100644 node_modules/npm-bundled/node_modules/npm-normalize-package-bin/package.json delete mode 100644 node_modules/npm-bundled/package.json delete mode 100755 node_modules/npm-packlist/bin/index.js delete mode 100644 node_modules/npm-packlist/node_modules/npm-normalize-package-bin/LICENSE delete mode 100644 node_modules/npm-packlist/node_modules/npm-normalize-package-bin/lib/index.js delete mode 100644 node_modules/npm-packlist/node_modules/npm-normalize-package-bin/package.json diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index e14cecf0dc7fe..0ce063561c744 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -25,6 +25,7 @@ graph LR; libnpmaccess-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmaccess-->npmcli-template-oss["@npmcli/template-oss"]; libnpmdiff-->npm-package-arg; + libnpmdiff-->npmcli-arborist["@npmcli/arborist"]; libnpmdiff-->npmcli-disparity-colors["@npmcli/disparity-colors"]; libnpmdiff-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmdiff-->npmcli-installed-package-contents["@npmcli/installed-package-contents"]; @@ -54,6 +55,7 @@ graph LR; libnpmorg-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmorg-->npmcli-template-oss["@npmcli/template-oss"]; libnpmpack-->npm-package-arg; + libnpmpack-->npmcli-arborist["@npmcli/arborist"]; libnpmpack-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmpack-->npmcli-run-script["@npmcli/run-script"]; libnpmpack-->npmcli-template-oss["@npmcli/template-oss"]; @@ -139,8 +141,6 @@ graph LR; npm-package-arg-->semver; npm-package-arg-->validate-npm-package-name; npm-packlist-->ignore-walk; - npm-packlist-->npm-bundled; - npm-packlist-->npm-normalize-package-bin; npm-profile-->npm-registry-fetch; npm-profile-->proc-log; npm-registry-fetch-->make-fetch-happen; @@ -333,6 +333,7 @@ graph LR; libnpmdiff-->diff; libnpmdiff-->minimatch; libnpmdiff-->npm-package-arg; + libnpmdiff-->npmcli-arborist["@npmcli/arborist"]; libnpmdiff-->npmcli-disparity-colors["@npmcli/disparity-colors"]; libnpmdiff-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmdiff-->npmcli-installed-package-contents["@npmcli/installed-package-contents"]; @@ -379,6 +380,7 @@ graph LR; libnpmorg-->tap; libnpmpack-->nock; libnpmpack-->npm-package-arg; + libnpmpack-->npmcli-arborist["@npmcli/arborist"]; libnpmpack-->npmcli-eslint-config["@npmcli/eslint-config"]; libnpmpack-->npmcli-run-script["@npmcli/run-script"]; libnpmpack-->npmcli-template-oss["@npmcli/template-oss"]; @@ -554,10 +556,7 @@ graph LR; npm-package-arg-->proc-log; npm-package-arg-->semver; npm-package-arg-->validate-npm-package-name; - npm-packlist-->glob; npm-packlist-->ignore-walk; - npm-packlist-->npm-bundled; - npm-packlist-->npm-normalize-package-bin; npm-pick-manifest-->npm-install-checks; npm-pick-manifest-->npm-normalize-package-bin; npm-pick-manifest-->npm-package-arg; @@ -756,12 +755,13 @@ Each group depends on packages lower down the chain, nothing depends on packages higher up the chain. - npm - - libnpmexec, libnpmfund - - @npmcli/arborist, libnpmpublish - - @npmcli/metavuln-calculator, libnpmdiff, libnpmpack + - libnpmpublish + - libnpmdiff, libnpmexec, libnpmfund, libnpmpack + - @npmcli/arborist + - @npmcli/metavuln-calculator - pacote, libnpmaccess, libnpmhook, libnpmorg, libnpmsearch, libnpmteam, npm-profile - npm-registry-fetch - make-fetch-happen, libnpmversion, @npmcli/config, init-package-json - - @npmcli/installed-package-contents, @npmcli/map-workspaces, cacache, @npmcli/git, @npmcli/run-script, npm-packlist, read-package-json, @npmcli/query, readdir-scoped-modules, promzard - - npm-bundled, read-package-json-fast, @npmcli/fs, unique-filename, @npmcli/promise-spawn, npm-package-arg, normalize-package-data, bin-links, nopt, npm-install-checks, npmlog, dezalgo, read + - @npmcli/installed-package-contents, @npmcli/map-workspaces, cacache, @npmcli/git, @npmcli/run-script, read-package-json, @npmcli/query, readdir-scoped-modules, promzard + - npm-bundled, read-package-json-fast, @npmcli/fs, unique-filename, @npmcli/promise-spawn, npm-package-arg, npm-packlist, normalize-package-data, bin-links, nopt, npm-install-checks, npmlog, dezalgo, read - npm-normalize-package-bin, @npmcli/name-from-folder, semver, @npmcli/move-file, fs-minipass, infer-owner, ssri, unique-slug, proc-log, @npmcli/node-gyp, hosted-git-info, validate-npm-package-name, ignore-walk, minipass-fetch, @npmcli/package-json, cmd-shim, read-cmd-shim, write-file-atomic, abbrev, are-we-there-yet, gauge, parse-conflict-json, wrappy, treeverse, @npmcli/eslint-config, @npmcli/template-oss, @npmcli/disparity-colors, @npmcli/ci-detect, mute-stream, ini, npm-audit-report, npm-user-validate \ No newline at end of file diff --git a/lib/commands/cache.js b/lib/commands/cache.js index bc52889c0006f..a2e6434b34cab 100644 --- a/lib/commands/cache.js +++ b/lib/commands/cache.js @@ -1,4 +1,5 @@ const cacache = require('cacache') +const Arborist = require('@npmcli/arborist') const { promisify } = require('util') const pacote = require('pacote') const path = require('path') @@ -164,7 +165,7 @@ class Cache extends BaseCommand { return pacote.tarball.stream(spec, stream => { stream.resume() return stream.promise() - }, this.npm.flatOptions) + }, { ...this.npm.flatOptions, Arborist }) })) } diff --git a/lib/package-url-cmd.js b/lib/package-url-cmd.js index 4254dde4517ba..eac2bbe1b6d51 100644 --- a/lib/package-url-cmd.js +++ b/lib/package-url-cmd.js @@ -2,6 +2,7 @@ const pacote = require('pacote') const hostedGitInfo = require('hosted-git-info') +const Arborist = require('@npmcli/arborist') const openUrl = require('./utils/open-url.js') const log = require('./utils/log-shim') @@ -31,6 +32,7 @@ class PackageUrlCommand extends BaseCommand { ...this.npm.flatOptions, where: this.npm.localPrefix, fullMetadata: true, + Arborist, } const mani = await pacote.manifest(arg, opts) const url = this.getUrl(arg, mani) diff --git a/node_modules/@npmcli/metavuln-calculator/package.json b/node_modules/@npmcli/metavuln-calculator/package.json index 2e7209ffc7da0..90b4d2ecddce4 100644 --- a/node_modules/@npmcli/metavuln-calculator/package.json +++ b/node_modules/@npmcli/metavuln-calculator/package.json @@ -1,6 +1,6 @@ { "name": "@npmcli/metavuln-calculator", - "version": "3.1.1", + "version": "4.0.0-pre.0", "main": "lib/index.js", "files": [ "bin/", @@ -18,9 +18,6 @@ "posttest": "npm run lint", "snap": "tap", "postsnap": "npm run lint", - "preversion": "npm test", - "postversion": "npm publish", - "prepublishOnly": "git push origin --follow-tags", "eslint": "eslint", "lint": "eslint \"**/*.js\"", "lintfix": "npm run lint -- --fix", @@ -29,25 +26,29 @@ }, "tap": { "check-coverage": true, - "coverage-map": "map.js" + "coverage-map": "map.js", + "nyc-arg": [ + "--exclude", + "tap-snapshots/**" + ] }, "devDependencies": { "@npmcli/eslint-config": "^3.0.1", - "@npmcli/template-oss": "3.5.0", + "@npmcli/template-oss": "4.4.2", "require-inject": "^1.4.4", "tap": "^16.0.1" }, "dependencies": { "cacache": "^16.0.0", "json-parse-even-better-errors": "^2.3.1", - "pacote": "^13.0.3", + "pacote": "^14.0.0 || ^14.0.0-pre.0", "semver": "^7.3.5" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" }, "templateOSS": { "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "3.5.0" + "version": "4.4.2" } } diff --git a/node_modules/npm-bundled/LICENSE b/node_modules/npm-bundled/LICENSE deleted file mode 100644 index 20a4762540923..0000000000000 --- a/node_modules/npm-bundled/LICENSE +++ /dev/null @@ -1,15 +0,0 @@ -The ISC License - -Copyright (c) npm, Inc. and Contributors - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR -IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/node_modules/npm-bundled/lib/index.js b/node_modules/npm-bundled/lib/index.js deleted file mode 100644 index 4f54ca647c087..0000000000000 --- a/node_modules/npm-bundled/lib/index.js +++ /dev/null @@ -1,254 +0,0 @@ -'use strict' - -// walk the tree of deps starting from the top level list of bundled deps -// Any deps at the top level that are depended on by a bundled dep that -// does not have that dep in its own node_modules folder are considered -// bundled deps as well. This list of names can be passed to npm-packlist -// as the "bundled" argument. Additionally, packageJsonCache is shared so -// packlist doesn't have to re-read files already consumed in this pass - -const fs = require('fs') -const path = require('path') -const EE = require('events').EventEmitter -// we don't care about the package bins, but we share a pj cache -// with other modules that DO care about it, so keep it nice. -const normalizePackageBin = require('npm-normalize-package-bin') - -class BundleWalker extends EE { - constructor (opt) { - opt = opt || {} - super(opt) - this.path = path.resolve(opt.path || process.cwd()) - - this.parent = opt.parent || null - if (this.parent) { - this.result = this.parent.result - // only collect results in node_modules folders at the top level - // since the node_modules in a bundled dep is included always - if (!this.parent.parent) { - const base = path.basename(this.path) - const scope = path.basename(path.dirname(this.path)) - this.result.add(/^@/.test(scope) ? scope + '/' + base : base) - } - this.root = this.parent.root - this.packageJsonCache = this.parent.packageJsonCache - } else { - this.result = new Set() - this.root = this.path - this.packageJsonCache = opt.packageJsonCache || new Map() - } - - this.seen = new Set() - this.didDone = false - this.children = 0 - this.node_modules = [] - this.package = null - this.bundle = null - } - - addListener (ev, fn) { - return this.on(ev, fn) - } - - on (ev, fn) { - const ret = super.on(ev, fn) - if (ev === 'done' && this.didDone) { - this.emit('done', this.result) - } - return ret - } - - done () { - if (!this.didDone) { - this.didDone = true - if (!this.parent) { - const res = Array.from(this.result) - this.result = res - this.emit('done', res) - } else { - this.emit('done') - } - } - } - - start () { - const pj = path.resolve(this.path, 'package.json') - if (this.packageJsonCache.has(pj)) { - this.onPackage(this.packageJsonCache.get(pj)) - } else { - this.readPackageJson(pj) - } - return this - } - - readPackageJson (pj) { - fs.readFile(pj, (er, data) => - er ? this.done() : this.onPackageJson(pj, data)) - } - - onPackageJson (pj, data) { - try { - this.package = normalizePackageBin(JSON.parse(data + '')) - } catch (er) { - return this.done() - } - this.packageJsonCache.set(pj, this.package) - this.onPackage(this.package) - } - - allDepsBundled (pkg) { - return Object.keys(pkg.dependencies || {}).concat( - Object.keys(pkg.optionalDependencies || {})) - } - - onPackage (pkg) { - // all deps are bundled if we got here as a child. - // otherwise, only bundle bundledDeps - // Get a unique-ified array with a short-lived Set - const bdRaw = this.parent ? this.allDepsBundled(pkg) - : pkg.bundleDependencies || pkg.bundledDependencies || [] - - const bd = Array.from(new Set( - Array.isArray(bdRaw) ? bdRaw - : bdRaw === true ? this.allDepsBundled(pkg) - : Object.keys(bdRaw))) - - if (!bd.length) { - return this.done() - } - - this.bundle = bd - this.readModules() - } - - readModules () { - readdirNodeModules(this.path + '/node_modules', (er, nm) => - er ? this.onReaddir([]) : this.onReaddir(nm)) - } - - onReaddir (nm) { - // keep track of what we have, in case children need it - this.node_modules = nm - - this.bundle.forEach(dep => this.childDep(dep)) - if (this.children === 0) { - this.done() - } - } - - childDep (dep) { - if (this.node_modules.indexOf(dep) !== -1) { - if (!this.seen.has(dep)) { - this.seen.add(dep) - this.child(dep) - } - } else if (this.parent) { - this.parent.childDep(dep) - } - } - - child (dep) { - const p = this.path + '/node_modules/' + dep - this.children += 1 - const child = new BundleWalker({ - path: p, - parent: this, - }) - child.on('done', _ => { - if (--this.children === 0) { - this.done() - } - }) - child.start() - } -} - -class BundleWalkerSync extends BundleWalker { - start () { - super.start() - this.done() - return this - } - - readPackageJson (pj) { - try { - this.onPackageJson(pj, fs.readFileSync(pj)) - } catch { - // empty catch - } - return this - } - - readModules () { - try { - this.onReaddir(readdirNodeModulesSync(this.path + '/node_modules')) - } catch { - this.onReaddir([]) - } - } - - child (dep) { - new BundleWalkerSync({ - path: this.path + '/node_modules/' + dep, - parent: this, - }).start() - } -} - -const readdirNodeModules = (nm, cb) => { - fs.readdir(nm, (er, set) => { - if (er) { - cb(er) - } else { - const scopes = set.filter(f => /^@/.test(f)) - if (!scopes.length) { - cb(null, set) - } else { - const unscoped = set.filter(f => !/^@/.test(f)) - let count = scopes.length - scopes.forEach(scope => { - fs.readdir(nm + '/' + scope, (readdirEr, pkgs) => { - if (readdirEr || !pkgs.length) { - unscoped.push(scope) - } else { - unscoped.push.apply(unscoped, pkgs.map(p => scope + '/' + p)) - } - if (--count === 0) { - cb(null, unscoped) - } - }) - }) - } - } - }) -} - -const readdirNodeModulesSync = nm => { - const set = fs.readdirSync(nm) - const unscoped = set.filter(f => !/^@/.test(f)) - const scopes = set.filter(f => /^@/.test(f)).map(scope => { - try { - const pkgs = fs.readdirSync(nm + '/' + scope) - return pkgs.length ? pkgs.map(p => scope + '/' + p) : [scope] - } catch (er) { - return [scope] - } - }).reduce((a, b) => a.concat(b), []) - return unscoped.concat(scopes) -} - -const walk = (options, callback) => { - const p = new Promise((resolve, reject) => { - new BundleWalker(options).on('done', resolve).on('error', reject).start() - }) - return callback ? p.then(res => callback(null, res), callback) : p -} - -const walkSync = options => { - return new BundleWalkerSync(options).start().result -} - -module.exports = walk -walk.sync = walkSync -walk.BundleWalker = BundleWalker -walk.BundleWalkerSync = BundleWalkerSync diff --git a/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/LICENSE b/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/LICENSE deleted file mode 100644 index 19cec97b18468..0000000000000 --- a/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/LICENSE +++ /dev/null @@ -1,15 +0,0 @@ -The ISC License - -Copyright (c) npm, Inc. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR -IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/lib/index.js b/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/lib/index.js deleted file mode 100644 index d6f0a581b9e66..0000000000000 --- a/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/lib/index.js +++ /dev/null @@ -1,64 +0,0 @@ -// pass in a manifest with a 'bin' field here, and it'll turn it -// into a properly santized bin object -const { join, basename } = require('path') - -const normalize = pkg => - !pkg.bin ? removeBin(pkg) - : typeof pkg.bin === 'string' ? normalizeString(pkg) - : Array.isArray(pkg.bin) ? normalizeArray(pkg) - : typeof pkg.bin === 'object' ? normalizeObject(pkg) - : removeBin(pkg) - -const normalizeString = pkg => { - if (!pkg.name) { - return removeBin(pkg) - } - pkg.bin = { [pkg.name]: pkg.bin } - return normalizeObject(pkg) -} - -const normalizeArray = pkg => { - pkg.bin = pkg.bin.reduce((acc, k) => { - acc[basename(k)] = k - return acc - }, {}) - return normalizeObject(pkg) -} - -const removeBin = pkg => { - delete pkg.bin - return pkg -} - -const normalizeObject = pkg => { - const orig = pkg.bin - const clean = {} - let hasBins = false - Object.keys(orig).forEach(binKey => { - const base = join('/', basename(binKey.replace(/\\|:/g, '/'))).slice(1) - - if (typeof orig[binKey] !== 'string' || !base) { - return - } - - const binTarget = join('/', orig[binKey]) - .replace(/\\/g, '/').slice(1) - - if (!binTarget) { - return - } - - clean[base] = binTarget - hasBins = true - }) - - if (hasBins) { - pkg.bin = clean - } else { - delete pkg.bin - } - - return pkg -} - -module.exports = normalize diff --git a/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/package.json b/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/package.json deleted file mode 100644 index 02de808d9b702..0000000000000 --- a/node_modules/npm-bundled/node_modules/npm-normalize-package-bin/package.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "npm-normalize-package-bin", - "version": "2.0.0", - "description": "Turn any flavor of allowable package.json bin into a normalized object", - "main": "lib/index.js", - "repository": { - "type": "git", - "url": "https://github.com/npm/npm-normalize-package-bin.git" - }, - "author": "GitHub Inc.", - "license": "ISC", - "scripts": { - "test": "tap", - "snap": "tap", - "preversion": "npm test", - "postversion": "npm publish", - "postpublish": "git push origin --follow-tags", - "lint": "eslint \"**/*.js\"", - "postlint": "template-oss-check", - "template-oss-apply": "template-oss-apply --force", - "lintfix": "npm run lint -- --fix", - "prepublishOnly": "git push origin --follow-tags", - "posttest": "npm run lint" - }, - "devDependencies": { - "@npmcli/eslint-config": "^3.1.0", - "@npmcli/template-oss": "3.5.0", - "tap": "^16.3.0" - }, - "files": [ - "bin/", - "lib/" - ], - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - }, - "templateOSS": { - "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "3.5.0" - } -} diff --git a/node_modules/npm-bundled/package.json b/node_modules/npm-bundled/package.json deleted file mode 100644 index e4c0106c2d504..0000000000000 --- a/node_modules/npm-bundled/package.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "npm-bundled", - "version": "2.0.1", - "description": "list things in node_modules that are bundledDependencies, or transitive dependencies thereof", - "main": "lib/index.js", - "repository": { - "type": "git", - "url": "https://github.com/npm/npm-bundled.git" - }, - "author": "GitHub Inc.", - "license": "ISC", - "devDependencies": { - "@npmcli/eslint-config": "^3.1.0", - "@npmcli/template-oss": "3.5.0", - "mkdirp": "^1.0.4", - "mutate-fs": "^2.1.1", - "rimraf": "^3.0.2", - "tap": "^16.3.0" - }, - "scripts": { - "test": "tap", - "preversion": "npm test", - "postversion": "npm publish", - "postpublish": "git push origin --all; git push origin --tags", - "lint": "eslint \"**/*.js\"", - "postlint": "template-oss-check", - "template-oss-apply": "template-oss-apply --force", - "lintfix": "npm run lint -- --fix", - "prepublishOnly": "git push origin --follow-tags", - "snap": "tap", - "posttest": "npm run lint" - }, - "files": [ - "bin/", - "lib/" - ], - "dependencies": { - "npm-normalize-package-bin": "^2.0.0" - }, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - }, - "templateOSS": { - "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "3.5.0" - } -} diff --git a/node_modules/npm-packlist/bin/index.js b/node_modules/npm-packlist/bin/index.js deleted file mode 100755 index 48a6b879aa823..0000000000000 --- a/node_modules/npm-packlist/bin/index.js +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env node -'use strict' - -const packlist = require('../') - -const dirs = [] -let doSort = false -process.argv.slice(2).forEach(arg => { - if (arg === '-h' || arg === '--help') { - console.log('usage: npm-packlist [-s --sort] [directory, directory, ...]') - process.exit(0) - } else if (arg === '-s' || arg === '--sort') { - doSort = true - } else { - dirs.push(arg) - } -}) - -const sort = list => doSort ? list.sort((a, b) => a.localeCompare(b, 'en')) : list - -const main = async () => { - if (!dirs.length) { - const results = await packlist({ path: process.cwd() }) - console.log(sort(results).join('\n')) - } else { - for (const dir of dirs) { - console.group(`> ${dir}`) - const results = await packlist({ path: dir }) - console.log(sort(results).join('\n')) - console.groupEnd() - } - } -} - -// coverage disabled for catch handler because we don't need to test that -main().catch(/* istanbul ignore next */(err) => { - process.exitCode = 1 - console.error(err.stack) -}) diff --git a/node_modules/npm-packlist/lib/index.js b/node_modules/npm-packlist/lib/index.js index bd72329f027e6..1b2cdbb8df641 100644 --- a/node_modules/npm-packlist/lib/index.js +++ b/node_modules/npm-packlist/lib/index.js @@ -1,87 +1,18 @@ 'use strict' -// Do a two-pass walk, first to get the list of packages that need to be -// bundled, then again to get the actual files and folders. -// Keep a cache of node_modules content and package.json data, so that the -// second walk doesn't have to re-do all the same work. +const { Walker: IgnoreWalker } = require('ignore-walk') +const { lstatSync: lstat, readFileSync: readFile } = require('fs') +const { basename, dirname, extname, join, relative, resolve, sep } = require('path') -const bundleWalk = require('npm-bundled') -const BundleWalker = bundleWalk.BundleWalker +// symbols used to represent synthetic rule sets +const defaultRules = Symbol('npm-packlist.rules.default') +const strictRules = Symbol('npm-packlist.rules.strict') -const ignoreWalk = require('ignore-walk') -const IgnoreWalker = ignoreWalk.Walker - -const rootBuiltinRules = Symbol('root-builtin-rules') -const packageNecessaryRules = Symbol('package-necessary-rules') -const path = require('path') - -const normalizePackageBin = require('npm-normalize-package-bin') - -// Weird side-effect of this: a readme (etc) file will be included -// if it exists anywhere within a folder with a package.json file. -// The original intent was only to include these files in the root, -// but now users in the wild are dependent on that behavior for -// localized documentation and other use cases. Adding a `/` to -// these rules, while tempting and arguably more "correct", is a -// significant change that will break existing use cases. -const packageMustHaveFileNames = 'readme|copying|license|licence' - -const packageMustHaves = `@(${packageMustHaveFileNames}){,.*[^~$]}` -const packageMustHavesRE = new RegExp(`^(${packageMustHaveFileNames})(\\..*[^~$])?$`, 'i') - -const fs = require('fs') -const glob = require('glob') -const globify = pattern => pattern.split('\\').join('/') - -const readOutOfTreeIgnoreFiles = (root, rel, result = '') => { - for (const file of ['.npmignore', '.gitignore']) { - try { - const ignoreContent = fs.readFileSync(path.join(root, file), { encoding: 'utf8' }) - result += ignoreContent + '\n' - // break the loop immediately after concatting, this allows us to prioritize the - // .npmignore and discard the .gitignore if one exists - break - } catch (err) { - // we ignore ENOENT errors completely because we don't care if the file doesn't exist - // but we throw everything else because failing to read a file that does exist is - // something that the user likely wants to know about. we don't need to test this. - /* istanbul ignore next */ - if (err.code !== 'ENOENT') { - throw err - } - } - } - - if (!rel) { - return result - } - - const firstRel = rel.split(path.sep)[0] - const newRoot = path.join(root, firstRel) - const newRel = path.relative(newRoot, path.join(root, rel)) - - return readOutOfTreeIgnoreFiles(newRoot, newRel, result) -} - -const pathHasPkg = (input) => { - if (!input.startsWith('node_modules/')) { - return false - } - - const segments = input.slice('node_modules/'.length).split('/', 2) - return segments[0].startsWith('@') - ? segments.length === 2 - : true -} - -const pkgFromPath = (input) => { - const segments = input.slice('node_modules/'.length).split('/', 2) - return segments[0].startsWith('@') - ? segments.join('/') - : segments[0] -} +// There may be others, but :?|<> are handled by node-tar +const nameIsBadForWindows = file => /\*/.test(file) -const defaultRules = [ +// these are the default rules that are applied to everything except for non-link bundled deps +const defaults = [ '.npmignore', '.gitignore', '**/.git', @@ -103,410 +34,413 @@ const defaultRules = [ '._*', '**/._*/**', '*.orig', - '/package-lock.json', - '/yarn.lock', - '/pnpm-lock.yaml', '/archived-packages/**', ] -// There may be others, but :?|<> are handled by node-tar -const nameIsBadForWindows = file => /\*/.test(file) +const strictDefaults = [ + // these are forcibly included at all levels + '!/readme{,.*[^~$]}', + '!/copying{,.*[^~$]}', + '!/license{,.*[^~$]}', + '!/licence{,.*[^~$]}', + // these are forcibly excluded + '/.git', +] -class Walker extends IgnoreWalker { - constructor (opt) { - opt = opt || {} - - // the order in which rules are applied. - opt.ignoreFiles = [ - rootBuiltinRules, - 'package.json', - '.npmignore', - '.gitignore', - packageNecessaryRules, - ] +const normalizePath = (path) => path.split('\\').join('/') - opt.includeEmpty = false - opt.path = opt.path || process.cwd() - - // only follow links in the root node_modules folder, because if those - // folders are included, it's because they're bundled, and bundles - // should include the contents, not the symlinks themselves. - // This regexp tests to see that we're either a node_modules folder, - // or a @scope within a node_modules folder, in the root's node_modules - // hierarchy (ie, not in test/foo/node_modules/ or something). - const followRe = /^(?:\/node_modules\/(?:@[^/]+\/[^/]+|[^/]+)\/)*\/node_modules(?:\/@[^/]+)?$/ - const rootPath = opt.parent ? opt.parent.root : opt.path - const followTestPath = opt.path.replace(/\\/g, '/').slice(rootPath.length) - opt.follow = followRe.test(followTestPath) - - super(opt) - - // ignore a bunch of things by default at the root level. - // also ignore anything in the main project node_modules hierarchy, - // except bundled dependencies - if (this.isProject) { - this.bundled = opt.bundled || [] - this.bundledScopes = Array.from(new Set( - this.bundled.filter(f => /^@/.test(f)) - .map(f => f.split('/')[0]))) - this.packageJsonCache = this.parent ? this.parent.packageJsonCache - : (opt.packageJsonCache || new Map()) - let rules = defaultRules.join('\n') + '\n' - - if (opt.prefix && opt.workspaces) { - const gPath = globify(opt.path) - const gPrefix = globify(opt.prefix) - const gWorkspaces = opt.workspaces.map((ws) => globify(ws)) - // if opt.path and opt.prefix are not the same directory, and opt.workspaces has opt.path - // in it, then we know that opt.path is a workspace directory. in order to not drop ignore - // rules from directories between the workspace root (opt.prefix) and the workspace itself - // (opt.path), we need to find and read those now - /* istanbul ignore else */ - if (gPath !== gPrefix && gWorkspaces.includes(gPath)) { - // relpath is the relative path between the prefix and the parent of opt.path - // we use the parent because ignore-walk will read the files in opt.path already - const relpath = path.relative(opt.prefix, path.dirname(opt.path)) - rules += readOutOfTreeIgnoreFiles(opt.prefix, relpath) - } else if (gPath === gPrefix) { - // on the other hand, if the path and the prefix are the same, then we ignore workspaces - // so that we don't pack workspaces inside of a root project - rules += opt.workspaces.map((ws) => globify(path.relative(opt.path, ws))).join('\n') - } +const readOutOfTreeIgnoreFiles = (root, rel, result = []) => { + for (const file of ['.npmignore', '.gitignore']) { + try { + const ignoreContent = readFile(join(root, file), { encoding: 'utf8' }) + result.push(ignoreContent) + // break the loop immediately after reading, this allows us to prioritize + // the .npmignore and discard the .gitignore if one is present + break + } catch (err) { + // we ignore ENOENT errors completely because we don't care if the file doesn't exist + // but we throw everything else because failing to read a file that does exist is + // something that the user likely wants to know about + // istanbul ignore next -- we do not need to test a thrown error + if (err.code !== 'ENOENT') { + throw err } - - super.onReadIgnoreFile(rootBuiltinRules, rules, _ => _) - } else { - this.bundled = [] - this.bundledScopes = [] - this.packageJsonCache = this.parent.packageJsonCache } } - get isProject () { - return !this.parent || this.parent.follow && this.isSymbolicLink + if (!rel) { + return result } - onReaddir (entries) { - if (this.isProject) { - entries = entries.filter(e => - e !== '.git' && - !(e === 'node_modules' && this.bundled.length === 0) - ) - } - - // if we have a package.json, then look in it for 'files' - // we _only_ do this in the root project, not bundled deps - // or other random folders. Bundled deps are always assumed - // to be in the state the user wants to include them, and - // a package.json somewhere else might be a template or - // test or something else entirely. - if (!this.isProject || !entries.includes('package.json')) { - return super.onReaddir(entries) - } + const firstRel = rel.split(sep, 1)[0] + const newRoot = join(root, firstRel) + const newRel = relative(newRoot, join(root, rel)) - // when the cache has been seeded with the root manifest, - // we must respect that (it may differ from the filesystem) - const ig = path.resolve(this.path, 'package.json') + return readOutOfTreeIgnoreFiles(newRoot, newRel, result) +} - if (this.packageJsonCache.has(ig)) { - const pkg = this.packageJsonCache.get(ig) +class PackWalker extends IgnoreWalker { + constructor (tree, opts) { + const options = { + ...opts, + includeEmpty: false, + follow: false, + // we path.resolve() here because ignore-walk doesn't do it and we want full paths + path: resolve(opts?.path || tree.path).replace(/\\/g, '/'), + ignoreFiles: opts?.ignoreFiles || [ + defaultRules, + 'package.json', + '.npmignore', + '.gitignore', + strictRules, + ], + } - // fall back to filesystem when seeded manifest is invalid - if (!pkg || typeof pkg !== 'object') { - return this.readPackageJson(entries) + super(options) + this.isPackage = options.isPackage + this.seen = options.seen || new Set() + this.tree = tree + this.requiredFiles = options.requiredFiles || [] + + const additionalDefaults = [] + if (options.prefix && options.workspaces) { + const path = normalizePath(options.path) + const prefix = normalizePath(options.prefix) + const workspaces = options.workspaces.map((ws) => normalizePath(ws)) + + // istanbul ignore else - this does nothing unless we need it to + if (path !== prefix && workspaces.includes(path)) { + // if path and prefix are not the same directory, and workspaces has path in it + // then we know path is a workspace directory. in order to not drop ignore rules + // from directories between the workspaces root (prefix) and the workspace itself + // (path) we need to find and read those now + const relpath = relative(options.prefix, dirname(options.path)) + additionalDefaults.push(...readOutOfTreeIgnoreFiles(options.prefix, relpath)) + } else if (path === prefix) { + // on the other hand, if the path and prefix are the same, then we ignore workspaces + // so that we don't pack a workspace as part of the root project. append them as + // normalized relative paths from the root + additionalDefaults.push(...workspaces.map((w) => normalizePath(relative(options.path, w)))) } - - // feels wonky, but this ensures package bin is _always_ - // normalized, as well as guarding against invalid JSON - return this.getPackageFiles(entries, JSON.stringify(pkg)) } - this.readPackageJson(entries) - } + // go ahead and inject the default rules now + this.injectRules(defaultRules, [...defaults, ...additionalDefaults]) - onReadPackageJson (entries, er, pkg) { - if (er) { - this.emit('error', er) - } else { - this.getPackageFiles(entries, pkg) + if (!this.isPackage) { + // if this instance is not a package, then place some strict default rules, and append + // known required files for this directory + this.injectRules(strictRules, [ + ...strictDefaults, + ...this.requiredFiles.map((file) => `!${file}`), + ]) } } - mustHaveFilesFromPackage (pkg) { - const files = [] - if (pkg.browser) { - files.push('/' + pkg.browser) - } - if (pkg.main) { - files.push('/' + pkg.main) + // overridden method: we intercept the reading of the package.json file here so that we can + // process it into both the package.json file rules as well as the strictRules synthetic rule set + addIgnoreFile (file, callback) { + // if we're adding anything other than package.json, then let ignore-walk handle it + if (file !== 'package.json' || !this.isPackage) { + return super.addIgnoreFile(file, callback) } - if (pkg.bin) { - // always an object because normalized already - for (const key in pkg.bin) { - files.push('/' + pkg.bin[key]) - } + + return this.processPackage(callback) + } + + // overridden method: if we're done, but we're a package, then we also need to evaluate bundles + // before we actually emit our done event + emit (ev, data) { + if (ev !== 'done' || !this.isPackage) { + return super.emit(ev, data) } - files.push( - '/package.json', - '/npm-shrinkwrap.json', - '!/package-lock.json', - packageMustHaves - ) - return files + + // we intentionally delay the done event while keeping the function sync here + // eslint-disable-next-line promise/catch-or-return, promise/always-return + this.gatherBundles().then(() => { + super.emit('done', this.result) + }) + return true } - getPackageFiles (entries, pkg) { - try { - // XXX this could be changed to use read-package-json-fast - // which handles the normalizing of bins for us, and simplifies - // the test for bundleDependencies and bundledDependencies later. - // HOWEVER if we do this, we need to be sure that we're careful - // about what we write back out since rpj-fast removes some fields - // that the user likely wants to keep. it also would add a second - // file read that we would want to optimize away. - pkg = normalizePackageBin(JSON.parse(pkg.toString())) - } catch (er) { - // not actually a valid package.json - return super.onReaddir(entries) + // overridden method: before actually filtering, we make sure that we've removed the rules for + // files that should no longer take effect due to our order of precedence + filterEntries () { + if (this.ignoreRules['package.json']) { + // package.json means no .npmignore or .gitignore + this.ignoreRules['.npmignore'] = null + this.ignoreRules['.gitignore'] = null + } else if (this.ignoreRules['.npmignore']) { + // .npmignore means no .gitignore + this.ignoreRules['.gitignore'] = null } - const ig = path.resolve(this.path, 'package.json') - this.packageJsonCache.set(ig, pkg) + return super.filterEntries() + } - // no files list, just return the normal readdir() result - if (!Array.isArray(pkg.files)) { - return super.onReaddir(entries) + // overridden method: we never want to include anything that isn't a file or directory + onstat (opts, callback) { + if (!opts.st.isFile() && !opts.st.isDirectory()) { + return callback() } - pkg.files.push(...this.mustHaveFilesFromPackage(pkg)) + return super.onstat(opts, callback) + } - // If the package has a files list, then it's unlikely to include - // node_modules, because why would you do that? but since we use - // the files list as the effective readdir result, that means it - // looks like we don't have a node_modules folder at all unless we - // include it here. - if ((pkg.bundleDependencies || pkg.bundledDependencies) && entries.includes('node_modules')) { - pkg.files.push('node_modules') + // overridden method: we want to refuse to pack files that are invalid, node-tar protects us from + // a lot of them but not all + stat (opts, callback) { + if (nameIsBadForWindows(opts.entry)) { + return callback() } - const patterns = Array.from(new Set(pkg.files)).reduce((set, pattern) => { - const excl = pattern.match(/^!+/) - if (excl) { - pattern = pattern.slice(excl[0].length) - } - // strip off any / or ./ from the start of the pattern. /foo => foo, ./foo => foo - pattern = pattern.replace(/^\.?\/+/, '') - // an odd number of ! means a negated pattern. !!foo ==> foo - const negate = excl && excl[0].length % 2 === 1 - set.push({ pattern, negate }) - return set - }, []) - - let n = patterns.length - const set = new Set() - const negates = new Set() - const results = [] - const then = (pattern, negate, er, fileList, i) => { - if (er) { - return this.emit('error', er) - } + return super.stat(opts, callback) + } - results[i] = { negate, fileList } - if (--n === 0) { - processResults(results) + // overridden method: this is called to create options for a child walker when we step + // in to a normal child directory (this will never be a bundle). the default method here + // copies the root's `ignoreFiles` value, but we don't want to respect package.json for + // subdirectories, so we override it with a list that intentionally omits package.json + walkerOpt (entry, opts) { + let ignoreFiles = null + + // however, if we have a tree, and we have workspaces, and the directory we're about + // to step into is a workspace, then we _do_ want to respect its package.json + if (this.tree.workspaces) { + const workspaceDirs = [...this.tree.workspaces.values()] + .map((dir) => dir.replace(/\\/g, '/')) + + const entryPath = join(this.path, entry).replace(/\\/g, '/') + if (workspaceDirs.includes(entryPath)) { + ignoreFiles = [ + defaultRules, + 'package.json', + '.npmignore', + '.gitignore', + strictRules, + ] } + } else { + ignoreFiles = [ + defaultRules, + '.npmignore', + '.gitignore', + strictRules, + ] } - const processResults = processed => { - for (const { negate, fileList } of processed) { - if (negate) { - fileList.forEach(f => { - f = f.replace(/\/+$/, '') - set.delete(f) - negates.add(f) - }) - } else { - fileList.forEach(f => { - f = f.replace(/\/+$/, '') - set.add(f) - negates.delete(f) - }) - } - } - const list = Array.from(set) - // replace the files array with our computed explicit set - pkg.files = list.concat(Array.from(negates).map(f => '!' + f)) - const rdResult = Array.from(new Set( - list.map(f => f.replace(/^\/+/, '')) - )) - super.onReaddir(rdResult) + return { + ...super.walkerOpt(entry, opts), + ignoreFiles, + // we map over our own requiredFiles and pass ones that are within this entry + requiredFiles: this.requiredFiles + .map((file) => { + if (relative(file, entry) === '..') { + return relative(entry, file).replace(/\\/g, '/') + } + return false + }) + .filter(Boolean), } + } - // maintain the index so that we process them in-order only once all - // are completed, otherwise the parallelism messes things up, since a - // glob like **/*.js will always be slower than a subsequent !foo.js - patterns.forEach(({ pattern, negate }, i) => - this.globFiles(pattern, (er, res) => then(pattern, negate, er, res, i))) + // overridden method: we want child walkers to be instances of this class, not ignore-walk + walker (entry, opts, callback) { + new PackWalker(this.tree, this.walkerOpt(entry, opts)).on('done', callback).start() } - filterEntry (entry, partial) { - // get the partial path from the root of the walk - const p = this.path.slice(this.root.length + 1) - const { isProject } = this - const pkg = isProject && pathHasPkg(entry) - ? pkgFromPath(entry) - : null - const rootNM = isProject && entry === 'node_modules' - const rootPJ = isProject && entry === 'package.json' - - return ( - // if we're in a bundled package, check with the parent. - /^node_modules($|\/)/i.test(p) && !this.isProject ? this.parent.filterEntry( - this.basename + '/' + entry, partial) - - // if package is bundled, all files included - // also include @scope dirs for bundled scoped deps - // they'll be ignored if no files end up in them. - // However, this only matters if we're in the root. - // node_modules folders elsewhere, like lib/node_modules, - // should be included normally unless ignored. - : pkg ? this.bundled.indexOf(pkg) !== -1 || - this.bundledScopes.indexOf(pkg) !== -1 - - // only walk top node_modules if we want to bundle something - : rootNM ? !!this.bundled.length - - // always include package.json at the root. - : rootPJ ? true - - // always include readmes etc in any included dir - : packageMustHavesRE.test(entry) ? true - - // npm-shrinkwrap and package.json always included in the root pkg - : isProject && (entry === 'npm-shrinkwrap.json' || entry === 'package.json') - ? true - - // package-lock never included - : isProject && entry === 'package-lock.json' ? false - - // otherwise, follow ignore-walk's logic - : super.filterEntry(entry, partial) - ) + // overridden method: we use a custom sort method to help compressibility + sort (a, b) { + // optimize for compressibility + // extname, then basename, then locale alphabetically + // https://twitter.com/isntitvacant/status/1131094910923231232 + const exta = extname(a).toLowerCase() + const extb = extname(b).toLowerCase() + const basea = basename(a).toLowerCase() + const baseb = basename(b).toLowerCase() + + return exta.localeCompare(extb, 'en') || + basea.localeCompare(baseb, 'en') || + a.localeCompare(b, 'en') } - filterEntries () { - if (this.ignoreRules['.npmignore']) { - this.ignoreRules['.gitignore'] = null - } - this.filterEntries = super.filterEntries - super.filterEntries() + // convenience method: this joins the given rules with newlines, appends a trailing newline, + // and calls the internal onReadIgnoreFile method + injectRules (filename, rules, callback = () => {}) { + this.onReadIgnoreFile(filename, `${rules.join('\n')}\n`, callback) } - addIgnoreFile (file, then) { - const ig = path.resolve(this.path, file) - if (file === 'package.json' && !this.isProject) { - then() - } else if (this.packageJsonCache.has(ig)) { - this.onPackageJson(ig, this.packageJsonCache.get(ig), then) - } else { - super.addIgnoreFile(file, then) + // custom method: this is called by addIgnoreFile when we find a package.json, it uses the + // arborist tree to pull both default rules and strict rules for the package + processPackage (callback) { + const { + bin, + browser, + files, + main, + } = this.tree.package + + // rules in these arrays are inverted since they are patterns we want to _not_ ignore + const ignores = [] + const strict = [ + ...strictDefaults, + '!/package.json', + '!/npm-shrinkwrap.json', + '/.git', + '/node_modules', + '/package-lock.json', + '/yarn.lock', + '/pnpm-lock.yaml', + ] + + // if we have a files array in our package, we need to pull rules from it + if (files) { + for (const file of files) { + // invert the rule because these are things we want to include + const inverse = `!${file}` + try { + // if an entry in the files array is a specific file, then we need to include it as a + // strict requirement for this package. if it's a directory or a pattern, it's a default + // pattern instead. this is ugly, but we have to stat to find out if it's a file + const stat = lstat(join(this.path, file.replace(/^!+/, '')).replace(/\\/g, '/')) + // if we have a file and we know that, it's strictly required + if (stat.isFile()) { + strict.unshift(inverse) + this.requiredFiles.push(file.startsWith('/') ? file.slice(1) : file) + } else if (stat.isDirectory()) { + // otherwise, it's a default ignore, and since we got here we know it's not a pattern + // so we include the directory contents + ignores.push(inverse) + ignores.push(`${inverse}/**`) + } + // if the thing exists, but is neither a file or a directory, we don't want it at all + } catch (err) { + // if lstat throws, then we assume we're looking at a pattern and treat it as a default + ignores.push(inverse) + } + } + + // we prepend a '*' to exclude everything, followed by our inverted file rules + // which now mean to include those + this.injectRules('package.json', ['*', ...ignores]) } - } - onPackageJson (ig, pkg, then) { - this.packageJsonCache.set(ig, pkg) + // browser is required + if (browser) { + strict.push(`!/${browser}`) + } - if (Array.isArray(pkg.files)) { - // in this case we already included all the must-haves - super.onReadIgnoreFile('package.json', pkg.files.map( - f => '!' + f - ).join('\n') + '\n', then) - } else { - // if there's a bin, browser or main, make sure we don't ignore it - // also, don't ignore the package.json itself, or any files that - // must be included in the package. - const rules = this.mustHaveFilesFromPackage(pkg).map(f => `!${f}`) - const data = rules.join('\n') + '\n' - super.onReadIgnoreFile(packageNecessaryRules, data, then) + // main is required + if (main) { + strict.push(`!/${main}`) } - } - // override parent stat function to completely skip any filenames - // that will break windows entirely. - // XXX(isaacs) Next major version should make this an error instead. - stat ({ entry, file, dir }, then) { - if (nameIsBadForWindows(entry)) { - then() - } else { - super.stat({ entry, file, dir }, then) + // each bin is required + if (bin) { + for (const key in bin) { + strict.push(`!/${bin[key]}`) + } } + + // and now we add all of the strict rules to our synthetic file + this.injectRules(strictRules, strict, callback) } - // override parent onstat function to nix all symlinks, other than - // those coming out of the followed bundled symlink deps - onstat ({ st, entry, file, dir, isSymbolicLink }, then) { - if (st.isSymbolicLink()) { - then() - } else { - super.onstat({ st, entry, file, dir, isSymbolicLink }, then) + // custom method: after we've finished gathering the files for the root package, we call this + // before emitting the 'done' event in order to gather all of the files for bundled deps + async gatherBundles () { + if (this.seen.has(this.tree)) { + return } - } - onReadIgnoreFile (file, data, then) { - if (file === 'package.json') { - try { - const ig = path.resolve(this.path, file) - this.onPackageJson(ig, JSON.parse(data), then) - } catch (er) { - // ignore package.json files that are not json - then() - } + // add this node to our seen tracker + this.seen.add(this.tree) + + // if we're the project root, then we look at our bundleDependencies, otherwise we got here + // because we're a bundled dependency of the root, which means we need to include all prod + // and optional dependencies in the bundle + let toBundle + if (this.tree.isProjectRoot) { + const { bundleDependencies } = this.tree.package + toBundle = bundleDependencies || [] } else { - super.onReadIgnoreFile(file, data, then) + const { dependencies, optionalDependencies } = this.tree.package + toBundle = Object.keys(dependencies || {}).concat(Object.keys(optionalDependencies || {})) } - } - sort (a, b) { - // optimize for compressibility - // extname, then basename, then locale alphabetically - // https://twitter.com/isntitvacant/status/1131094910923231232 - const exta = path.extname(a).toLowerCase() - const extb = path.extname(b).toLowerCase() - const basea = path.basename(a).toLowerCase() - const baseb = path.basename(b).toLowerCase() + for (const dep of toBundle) { + const edge = this.tree.edgesOut.get(dep) + // no edgeOut = missing node, so skip it. we can't pack it if it's not here + // we also refuse to pack peer dependencies and dev dependencies + if (!edge || edge.peer || edge.dev) { + continue + } - return exta.localeCompare(extb, 'en') || - basea.localeCompare(baseb, 'en') || - a.localeCompare(b, 'en') - } + // get a reference to the node we're bundling + const node = this.tree.edgesOut.get(dep).to + // we use node.path for the path because we want the location the node was linked to, + // not where it actually lives on disk + const path = node.path + // but link nodes don't have edgesOut, so we need to pass in the target of the node + // in order to make sure we correctly traverse its dependencies + const tree = node.target + + // and start building options to be passed to the walker for this package + const walkerOpts = { + path, + isPackage: true, + ignoreFiles: [], + seen: this.seen, // pass through seen so we can prevent infinite circular loops + } - globFiles (pattern, cb) { - glob(globify(pattern), { dot: true, cwd: this.path, nocase: true }, cb) - } + // if our node is a link, we apply defaultRules. we don't do this for regular bundled + // deps because their .npmignore and .gitignore files are excluded by default and may + // override defaults + if (node.isLink) { + walkerOpts.ignoreFiles.push(defaultRules) + } - readPackageJson (entries) { - fs.readFile(this.path + '/package.json', (er, pkg) => - this.onReadPackageJson(entries, er, pkg)) - } + // _all_ nodes will follow package.json rules from their package root + walkerOpts.ignoreFiles.push('package.json') + + // only link nodes will obey .npmignore or .gitignore + if (node.isLink) { + walkerOpts.ignoreFiles.push('.npmignore') + walkerOpts.ignoreFiles.push('.gitignore') + } - walker (entry, opt, then) { - new Walker(this.walkerOpt(entry, opt)).on('done', then).start() + // _all_ nodes follow strict rules + walkerOpts.ignoreFiles.push(strictRules) + + // create a walker for this dependency and gather its results + const walker = new PackWalker(tree, walkerOpts) + const bundled = await new Promise((pResolve, pReject) => { + walker.on('error', pReject) + walker.on('done', pResolve) + walker.start() + }) + + // now we make sure we have our paths correct from the root, and accumulate everything into + // our own result set to deduplicate + const relativeFrom = relative(this.root, walker.path) + for (const file of bundled) { + this.result.add(join(relativeFrom, file).replace(/\\/g, '/')) + } + } } } -const walk = (options, callback) => { - options = options || {} - const p = new Promise((resolve, reject) => { - const bw = new BundleWalker(options) - bw.on('done', bundled => { - options.bundled = bundled - options.packageJsonCache = bw.packageJsonCache - new Walker(options).on('done', resolve).on('error', reject).start() - }) - bw.start() +const walk = (tree, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + const p = new Promise((pResolve, pReject) => { + new PackWalker(tree, { ...options, isPackage: true }) + .on('done', pResolve).on('error', pReject).start() }) return callback ? p.then(res => callback(null, res), callback) : p } module.exports = walk -walk.Walker = Walker +walk.Walker = PackWalker diff --git a/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/LICENSE b/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/LICENSE deleted file mode 100644 index 19cec97b18468..0000000000000 --- a/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/LICENSE +++ /dev/null @@ -1,15 +0,0 @@ -The ISC License - -Copyright (c) npm, Inc. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR -IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/lib/index.js b/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/lib/index.js deleted file mode 100644 index d6f0a581b9e66..0000000000000 --- a/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/lib/index.js +++ /dev/null @@ -1,64 +0,0 @@ -// pass in a manifest with a 'bin' field here, and it'll turn it -// into a properly santized bin object -const { join, basename } = require('path') - -const normalize = pkg => - !pkg.bin ? removeBin(pkg) - : typeof pkg.bin === 'string' ? normalizeString(pkg) - : Array.isArray(pkg.bin) ? normalizeArray(pkg) - : typeof pkg.bin === 'object' ? normalizeObject(pkg) - : removeBin(pkg) - -const normalizeString = pkg => { - if (!pkg.name) { - return removeBin(pkg) - } - pkg.bin = { [pkg.name]: pkg.bin } - return normalizeObject(pkg) -} - -const normalizeArray = pkg => { - pkg.bin = pkg.bin.reduce((acc, k) => { - acc[basename(k)] = k - return acc - }, {}) - return normalizeObject(pkg) -} - -const removeBin = pkg => { - delete pkg.bin - return pkg -} - -const normalizeObject = pkg => { - const orig = pkg.bin - const clean = {} - let hasBins = false - Object.keys(orig).forEach(binKey => { - const base = join('/', basename(binKey.replace(/\\|:/g, '/'))).slice(1) - - if (typeof orig[binKey] !== 'string' || !base) { - return - } - - const binTarget = join('/', orig[binKey]) - .replace(/\\/g, '/').slice(1) - - if (!binTarget) { - return - } - - clean[base] = binTarget - hasBins = true - }) - - if (hasBins) { - pkg.bin = clean - } else { - delete pkg.bin - } - - return pkg -} - -module.exports = normalize diff --git a/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/package.json b/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/package.json deleted file mode 100644 index 02de808d9b702..0000000000000 --- a/node_modules/npm-packlist/node_modules/npm-normalize-package-bin/package.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "npm-normalize-package-bin", - "version": "2.0.0", - "description": "Turn any flavor of allowable package.json bin into a normalized object", - "main": "lib/index.js", - "repository": { - "type": "git", - "url": "https://github.com/npm/npm-normalize-package-bin.git" - }, - "author": "GitHub Inc.", - "license": "ISC", - "scripts": { - "test": "tap", - "snap": "tap", - "preversion": "npm test", - "postversion": "npm publish", - "postpublish": "git push origin --follow-tags", - "lint": "eslint \"**/*.js\"", - "postlint": "template-oss-check", - "template-oss-apply": "template-oss-apply --force", - "lintfix": "npm run lint -- --fix", - "prepublishOnly": "git push origin --follow-tags", - "posttest": "npm run lint" - }, - "devDependencies": { - "@npmcli/eslint-config": "^3.1.0", - "@npmcli/template-oss": "3.5.0", - "tap": "^16.3.0" - }, - "files": [ - "bin/", - "lib/" - ], - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - }, - "templateOSS": { - "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "3.5.0" - } -} diff --git a/node_modules/npm-packlist/package.json b/node_modules/npm-packlist/package.json index c3c8817202a39..4aaa524bf76a7 100644 --- a/node_modules/npm-packlist/package.json +++ b/node_modules/npm-packlist/package.json @@ -1,16 +1,13 @@ { "name": "npm-packlist", - "version": "5.1.3", + "version": "7.0.0-pre.0", "description": "Get a list of the files to add from a folder into an npm package", "directories": { "test": "test" }, - "main": "lib", + "main": "lib/index.js", "dependencies": { - "glob": "^8.0.1", - "ignore-walk": "^5.0.1", - "npm-bundled": "^2.0.0", - "npm-normalize-package-bin": "^2.0.0" + "ignore-walk": "^5.0.1" }, "author": "GitHub Inc.", "license": "ISC", @@ -19,8 +16,9 @@ "lib/" ], "devDependencies": { + "@npmcli/arborist": "^6.0.0 || ^6.0.0-pre.0", "@npmcli/eslint-config": "^3.0.1", - "@npmcli/template-oss": "3.6.0", + "@npmcli/template-oss": "4.4.2", "mutate-fs": "^2.1.1", "tap": "^16.0.1" }, @@ -29,9 +27,6 @@ "posttest": "npm run lint", "snap": "tap", "postsnap": "npm run lintfix --", - "preversion": "npm test", - "postversion": "npm publish", - "prepublishOnly": "git push origin --follow-tags", "eslint": "eslint", "lint": "eslint \"**/*.js\"", "lintfix": "npm run lint -- --fix", @@ -46,16 +41,20 @@ "tap": { "test-env": [ "LC_ALL=sk" + ], + "nyc-arg": [ + "--exclude", + "tap-snapshots/**" + ], + "files": [ + "test/*.js" ] }, - "bin": { - "npm-packlist": "bin/index.js" - }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" }, "templateOSS": { "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "3.6.0" + "version": "4.4.2" } } diff --git a/node_modules/pacote/lib/dir.js b/node_modules/pacote/lib/dir.js index 502379810a006..df04cd08c51e5 100644 --- a/node_modules/pacote/lib/dir.js +++ b/node_modules/pacote/lib/dir.js @@ -16,6 +16,9 @@ class DirFetcher extends Fetcher { super(spec, opts) // just the fully resolved filename this.resolved = this.spec.fetchSpec + + this.tree = opts.tree || null + this.Arborist = opts.Arborist || null } // exposes tarCreateOptions as public API @@ -59,6 +62,10 @@ class DirFetcher extends Fetcher { } [_tarballFromResolved] () { + if (!this.tree && !this.Arborist) { + throw new Error('DirFetcher requires either a tree or an Arborist constructor to pack') + } + const stream = new Minipass() stream.resolved = this.resolved stream.integrity = this.integrity @@ -68,7 +75,13 @@ class DirFetcher extends Fetcher { // run the prepare script, get the list of files, and tar it up // pipe to the stream, and proxy errors the chain. this[_prepareDir]() - .then(() => packlist({ path: this.resolved, prefix, workspaces })) + .then(async () => { + if (!this.tree) { + const arb = new this.Arborist({ path: this.resolved }) + this.tree = await arb.loadActual() + } + return packlist(this.tree, { path: this.resolved, prefix, workspaces }) + }) .then(files => tar.c(tarCreateOptions(this.package), files) .on('error', er => stream.emit('error', er)).pipe(stream)) .catch(er => stream.emit('error', er)) diff --git a/node_modules/pacote/lib/git.js b/node_modules/pacote/lib/git.js index c4819b4fdf49c..1fa8b1f966334 100644 --- a/node_modules/pacote/lib/git.js +++ b/node_modules/pacote/lib/git.js @@ -61,6 +61,8 @@ class GitFetcher extends Fetcher { } else { this.resolvedSha = '' } + + this.Arborist = opts.Arborist || null } // just exposed to make it easier to test all the combinations @@ -206,8 +208,12 @@ class GitFetcher extends Fetcher { // check it out and then shell out to the DirFetcher tarball packer this[_clone](dir => this[_prepareDir](dir) .then(() => new Promise((res, rej) => { + if (!this.Arborist) { + throw new Error('GitFetcher requires an Arborist constructor to pack a tarball') + } const df = new DirFetcher(`file:${dir}`, { ...this.opts, + Arborist: this.Arborist, resolved: null, integrity: null, }) diff --git a/node_modules/pacote/lib/registry.js b/node_modules/pacote/lib/registry.js index c8eb6b0290702..eeb22e93c33d6 100644 --- a/node_modules/pacote/lib/registry.js +++ b/node_modules/pacote/lib/registry.js @@ -97,7 +97,6 @@ class RegistryFetcher extends Fetcher { integrity: null, }) const packument = await res.json() - packument._cached = res.headers.has('x-local-cache') packument._contentLength = +res.headers.get('content-length') if (this.packumentCache) { this.packumentCache.set(this.packumentUrl, packument) diff --git a/node_modules/pacote/package.json b/node_modules/pacote/package.json index 960530ec0b33d..f9b796d86cd20 100644 --- a/node_modules/pacote/package.json +++ b/node_modules/pacote/package.json @@ -1,6 +1,6 @@ { "name": "pacote", - "version": "13.6.2", + "version": "14.0.0-pre.3", "description": "JavaScript package downloader", "author": "GitHub Inc.", "bin": { @@ -11,9 +11,6 @@ "scripts": { "test": "tap", "snap": "tap", - "preversion": "npm test", - "postversion": "npm publish", - "prepublishOnly": "git push origin --follow-tags", "lint": "eslint \"**/*.js\"", "postlint": "template-oss-check", "lintfix": "npm run lint -- --fix", @@ -21,11 +18,16 @@ "template-oss-apply": "template-oss-apply --force" }, "tap": { - "timeout": 300 + "timeout": 300, + "nyc-arg": [ + "--exclude", + "tap-snapshots/**" + ] }, "devDependencies": { - "@npmcli/eslint-config": "^3.0.1", - "@npmcli/template-oss": "3.5.0", + "@npmcli/arborist": "^6.0.0 || ^6.0.0-pre.0", + "@npmcli/eslint-config": "^3.1.0", + "@npmcli/template-oss": "4.4.2", "hosted-git-info": "^5.0.0", "mutate-fs": "^2.1.1", "nock": "^13.2.4", @@ -53,7 +55,7 @@ "minipass": "^3.1.6", "mkdirp": "^1.0.4", "npm-package-arg": "^9.0.0", - "npm-packlist": "^5.1.0", + "npm-packlist": "^7.0.0 || ^7.0.0-pre.0", "npm-pick-manifest": "^7.0.0", "npm-registry-fetch": "^13.0.1", "proc-log": "^2.0.0", @@ -65,7 +67,7 @@ "tar": "^6.1.11" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" }, "repository": { "type": "git", @@ -73,7 +75,7 @@ }, "templateOSS": { "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.", - "version": "3.5.0", + "version": "4.4.2", "windowsCI": false } } diff --git a/package-lock.json b/package-lock.json index 2441009d92e20..e455502bd7466 100644 --- a/package-lock.json +++ b/package-lock.json @@ -143,7 +143,7 @@ "npmlog": "^6.0.2", "opener": "^1.5.2", "p-map": "^4.0.0", - "pacote": "^13.6.2", + "pacote": "^14.0.0-pre.3", "parse-conflict-json": "^2.0.2", "proc-log": "^2.0.1", "qrcode-terminal": "^0.12.0", @@ -2269,17 +2269,17 @@ } }, "node_modules/@npmcli/metavuln-calculator": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@npmcli/metavuln-calculator/-/metavuln-calculator-3.1.1.tgz", - "integrity": "sha512-n69ygIaqAedecLeVH3KnO39M6ZHiJ2dEv5A7DGvcqCB8q17BGUgW8QaanIkbWUo2aYGZqJaOORTLAlIvKjNDKA==", + "version": "4.0.0-pre.0", + "resolved": "https://registry.npmjs.org/@npmcli/metavuln-calculator/-/metavuln-calculator-4.0.0-pre.0.tgz", + "integrity": "sha512-2rJ7hovlcZMkqKm2cOWuZ0YsXIcP3iARsm+aYn/SLXK9aWRMVTW1f4fpDjtSvkZkaQVr48ofSG3YLYwlersSQA==", "dependencies": { "cacache": "^16.0.0", "json-parse-even-better-errors": "^2.3.1", - "pacote": "^13.0.3", + "pacote": "^14.0.0 || ^14.0.0-pre.0", "semver": "^7.3.5" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, "node_modules/@npmcli/move-file": { @@ -7966,27 +7966,6 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, - "node_modules/npm-bundled": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-2.0.1.tgz", - "integrity": "sha512-gZLxXdjEzE/+mOstGDqR6b0EkhJ+kM6fxM6vUuckuctuVPh80Q6pw/rSZj9s4Gex9GxWtIicO1pc8DB9KZWudw==", - "inBundle": true, - "dependencies": { - "npm-normalize-package-bin": "^2.0.0" - }, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - } - }, - "node_modules/npm-bundled/node_modules/npm-normalize-package-bin": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", - "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", - "inBundle": true, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - } - }, "node_modules/npm-install-checks": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/npm-install-checks/-/npm-install-checks-5.0.0.tgz", @@ -8025,30 +8004,15 @@ } }, "node_modules/npm-packlist": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-5.1.3.tgz", - "integrity": "sha512-263/0NGrn32YFYi4J533qzrQ/krmmrWwhKkzwTuM4f/07ug51odoaNjUexxO4vxlzURHcmYMH1QjvHjsNDKLVg==", + "version": "7.0.0-pre.0", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-7.0.0-pre.0.tgz", + "integrity": "sha512-m98nCdY9RDDSJAODMf9afFwUyyuSO7rl1b8vvKRJD6s/isdTlOEgfdxJP4Pj31l973GNDe1n41e07QGPSIBQSw==", "inBundle": true, "dependencies": { - "glob": "^8.0.1", - "ignore-walk": "^5.0.1", - "npm-bundled": "^2.0.0", - "npm-normalize-package-bin": "^2.0.0" - }, - "bin": { - "npm-packlist": "bin/index.js" + "ignore-walk": "^5.0.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - } - }, - "node_modules/npm-packlist/node_modules/npm-normalize-package-bin": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-2.0.0.tgz", - "integrity": "sha512-awzfKUO7v0FscrSpRoogyNm0sajikhBWpU0QMrW09AMi9n1PoKU6WaIqUzuJSQnpciZZmJ/jMZ2Egfmb/9LiWQ==", - "inBundle": true, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, "node_modules/npm-pick-manifest": { @@ -8587,9 +8551,9 @@ } }, "node_modules/pacote": { - "version": "13.6.2", - "resolved": "https://registry.npmjs.org/pacote/-/pacote-13.6.2.tgz", - "integrity": "sha512-Gu8fU3GsvOPkak2CkbojR7vjs3k3P9cA6uazKTHdsdV0gpCEQq2opelnEv30KRQWgVzP5Vd/5umjcedma3MKtg==", + "version": "14.0.0-pre.3", + "resolved": "https://registry.npmjs.org/pacote/-/pacote-14.0.0-pre.3.tgz", + "integrity": "sha512-WS8jos9mKpG6yRdMacwBc5WPEE4Z4xyJqyYiBoEU/0ayFlEPL8M8LUXlg86zjMWVpPobWIOIHvDO2i5oxOpIgQ==", "inBundle": true, "dependencies": { "@npmcli/git": "^3.0.0", @@ -8603,7 +8567,7 @@ "minipass": "^3.1.6", "mkdirp": "^1.0.4", "npm-package-arg": "^9.0.0", - "npm-packlist": "^5.1.0", + "npm-packlist": "^7.0.0 || ^7.0.0-pre.0", "npm-pick-manifest": "^7.0.0", "npm-registry-fetch": "^13.0.1", "proc-log": "^2.0.0", @@ -8618,7 +8582,7 @@ "pacote": "lib/bin.js" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, "node_modules/parent-module": { @@ -13870,7 +13834,7 @@ "@isaacs/string-locale-compare": "^1.1.0", "@npmcli/installed-package-contents": "^1.0.7", "@npmcli/map-workspaces": "^2.0.3", - "@npmcli/metavuln-calculator": "^3.0.1", + "@npmcli/metavuln-calculator": "^4.0.0-pre.0", "@npmcli/move-file": "^2.0.0", "@npmcli/name-from-folder": "^1.0.1", "@npmcli/node-gyp": "^2.0.0", @@ -13891,7 +13855,7 @@ "npm-pick-manifest": "^7.0.2", "npm-registry-fetch": "^13.0.0", "npmlog": "^6.0.2", - "pacote": "^13.6.1", + "pacote": "^14.0.0-pre.3", "parse-conflict-json": "^2.0.1", "proc-log": "^2.0.0", "promise-all-reject-late": "^1.0.0", @@ -13942,13 +13906,14 @@ "version": "5.0.0-pre.0", "license": "ISC", "dependencies": { + "@npmcli/arborist": "^6.0.0-pre.2", "@npmcli/disparity-colors": "^2.0.0", "@npmcli/installed-package-contents": "^1.0.7", "binary-extensions": "^2.2.0", "diff": "^5.1.0", "minimatch": "^5.0.1", "npm-package-arg": "^9.0.1", - "pacote": "^13.6.1", + "pacote": "^14.0.0-pre.3", "tar": "^6.1.0" }, "devDependencies": { @@ -13972,7 +13937,7 @@ "mkdirp-infer-owner": "^2.0.0", "npm-package-arg": "^9.0.1", "npmlog": "^6.0.2", - "pacote": "^13.6.1", + "pacote": "^14.0.0-pre.3", "proc-log": "^2.0.0", "read": "^1.0.7", "read-package-json-fast": "^2.0.2", @@ -14045,9 +14010,10 @@ "version": "5.0.0-pre.0", "license": "ISC", "dependencies": { + "@npmcli/arborist": "^6.0.0-pre.2", "@npmcli/run-script": "^4.1.3", "npm-package-arg": "^9.0.1", - "pacote": "^13.6.1" + "pacote": "^14.0.0-pre.3" }, "devDependencies": { "@npmcli/eslint-config": "^3.1.0", diff --git a/package.json b/package.json index 250f519e11994..a457bf5ed3aa2 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "npmlog": "^6.0.2", "opener": "^1.5.2", "p-map": "^4.0.0", - "pacote": "^13.6.2", + "pacote": "^14.0.0-pre.3", "parse-conflict-json": "^2.0.2", "proc-log": "^2.0.1", "qrcode-terminal": "^0.12.0", diff --git a/tap-snapshots/test/lib/utils/tar.js.test.cjs b/tap-snapshots/test/lib/utils/tar.js.test.cjs index d132d7af6e6f9..e4af36aeae0b6 100644 --- a/tap-snapshots/test/lib/utils/tar.js.test.cjs +++ b/tap-snapshots/test/lib/utils/tar.js.test.cjs @@ -11,10 +11,10 @@ exports[`test/lib/utils/tar.js TAP should log tarball contents > must match snap package: my-cool-pkg@1.0.0 === Tarball Contents === -4B cat -4B chai -4B dog -97B package.json +4B cat +4B chai +4B dog +114B package.json === Bundled Dependencies === bundle-dep @@ -23,10 +23,10 @@ bundle-dep name: my-cool-pkg version: 1.0.0 filename: my-cool-pkg-1.0.0.tgz -package size: 274 B -unpacked size: 113 B -shasum: cd0dfccff77dff944eb761854bc0b0497d974f67 -integrity: sha512-qeFip1jH05vkW[...]zHSdMdPpYogMA== +package size: 271 B +unpacked size: 126 B +shasum: 23e31c8ad422f96301c07730e61ff403b10306f1 +integrity: sha512-/Lg5tEGQv5A5y[...]gq8T9D5+Wat1A== bundled deps: 1 bundled files: 0 own files: 5 diff --git a/test/fixtures/mock-registry.js b/test/fixtures/mock-registry.js index d978929b6b0d8..a39532958b338 100644 --- a/test/fixtures/mock-registry.js +++ b/test/fixtures/mock-registry.js @@ -5,6 +5,7 @@ * for tests against any registry data. */ const pacote = require('pacote') +const Arborist = require('@npmcli/arborist') const npa = require('npm-package-arg') class MockRegistry { #tap @@ -250,7 +251,7 @@ class MockRegistry { async tarball ({ manifest, tarball }) { const nock = this.nock const dist = new URL(manifest.dist.tarball) - const tar = await pacote.tarball(tarball) + const tar = await pacote.tarball(tarball, { Arborist }) nock.get(dist.pathname).reply(200, tar) return nock } diff --git a/test/lib/commands/publish.js b/test/lib/commands/publish.js index 995abff88c2c1..00fba9ef218e0 100644 --- a/test/lib/commands/publish.js +++ b/test/lib/commands/publish.js @@ -2,6 +2,7 @@ const t = require('tap') const { load: loadMockNpm } = require('../../fixtures/mock-npm') const MockRegistry = require('../../fixtures/mock-registry.js') const pacote = require('pacote') +const Arborist = require('@npmcli/arborist') const path = require('path') const fs = require('@npmcli/fs') const npa = require('npm-package-arg') @@ -227,7 +228,7 @@ t.test('tarball', async t => { 'index.js': 'console.log("hello world"}', }, }) - const tarball = await pacote.tarball(home) + const tarball = await pacote.tarball(home, { Arborist }) const tarFilename = path.join(home, 'tarball.tgz') await fs.writeFile(tarFilename, tarball) const registry = new MockRegistry({ diff --git a/test/lib/utils/tar.js b/test/lib/utils/tar.js index adc5cb364997f..23f40703b5cf4 100644 --- a/test/lib/utils/tar.js +++ b/test/lib/utils/tar.js @@ -27,12 +27,17 @@ t.test('should log tarball contents', async (t) => { bundleDependencies: [ 'bundle-dep', ], - }, null, 2), + dependencies: { + 'bundle-dep': '1.0.0', + }, + }), cat: 'meow', chai: 'blub', dog: 'woof', node_modules: { - 'bundle-dep': 'toto', + 'bundle-dep': { + 'package.json': '', + }, }, }) diff --git a/workspaces/arborist/lib/arborist/build-ideal-tree.js b/workspaces/arborist/lib/arborist/build-ideal-tree.js index e9a8720d7322d..0260bd563ab2f 100644 --- a/workspaces/arborist/lib/arborist/build-ideal-tree.js +++ b/workspaces/arborist/lib/arborist/build-ideal-tree.js @@ -833,6 +833,7 @@ This is a one-time fix-up, please be patient... await cacache.tmp.withTmp(this.cache, opt, async path => { await pacote.extract(node.resolved, path, { ...opt, + Arborist, resolved: node.resolved, integrity: node.integrity, }) diff --git a/workspaces/arborist/lib/arborist/reify.js b/workspaces/arborist/lib/arborist/reify.js index 0c9026f5e4d1e..4f9db7575d79b 100644 --- a/workspaces/arborist/lib/arborist/reify.js +++ b/workspaces/arborist/lib/arborist/reify.js @@ -676,6 +676,7 @@ module.exports = cls => class Reifier extends cls { }) await pacote.extract(res, node.path, { ...this.options, + Arborist: this.constructor, resolved: node.resolved, integrity: node.integrity, }) diff --git a/workspaces/arborist/package.json b/workspaces/arborist/package.json index 8082cf820d2f2..243b9d4675017 100644 --- a/workspaces/arborist/package.json +++ b/workspaces/arborist/package.json @@ -6,7 +6,7 @@ "@isaacs/string-locale-compare": "^1.1.0", "@npmcli/installed-package-contents": "^1.0.7", "@npmcli/map-workspaces": "^2.0.3", - "@npmcli/metavuln-calculator": "^3.0.1", + "@npmcli/metavuln-calculator": "^4.0.0-pre.0", "@npmcli/move-file": "^2.0.0", "@npmcli/name-from-folder": "^1.0.1", "@npmcli/node-gyp": "^2.0.0", @@ -27,7 +27,7 @@ "npm-pick-manifest": "^7.0.2", "npm-registry-fetch": "^13.0.0", "npmlog": "^6.0.2", - "pacote": "^13.6.1", + "pacote": "^14.0.0-pre.3", "parse-conflict-json": "^2.0.1", "proc-log": "^2.0.0", "promise-all-reject-late": "^1.0.0", diff --git a/workspaces/arborist/test/fixtures/registry-mocks/fetch-lock-contents.js b/workspaces/arborist/test/fixtures/registry-mocks/fetch-lock-contents.js index 0f756f00bd657..5fb010c13bea2 100644 --- a/workspaces/arborist/test/fixtures/registry-mocks/fetch-lock-contents.js +++ b/workspaces/arborist/test/fixtures/registry-mocks/fetch-lock-contents.js @@ -1,5 +1,6 @@ // fetch all the deps and tarballs in a v2 lockfile const pacote = require('pacote') +const Arborist = require('../../index.js') const url = require('url') const mkdirp = require('mkdirp') const {dirname, resolve} = require('path') @@ -29,7 +30,7 @@ const main = async lock => { continue const path = url.parse(meta.resolved).pathname.replace(/^\/@?/, '') const tgzFile = resolve(dir, path) - await pacote.tarball.file(meta.resolved, tgzFile) + await pacote.tarball.file(meta.resolved, tgzFile, { Arborist }) } console.log('OK!') } diff --git a/workspaces/libnpmdiff/lib/tarball.js b/workspaces/libnpmdiff/lib/tarball.js index 4d01d69c9c413..930d624f2d5b6 100644 --- a/workspaces/libnpmdiff/lib/tarball.js +++ b/workspaces/libnpmdiff/lib/tarball.js @@ -1,5 +1,6 @@ const { relative } = require('path') +const Arborist = require('@npmcli/arborist') const npa = require('npm-package-arg') const pkgContents = require('@npmcli/installed-package-contents') const pacote = require('pacote') @@ -28,7 +29,10 @@ const tarball = (manifest, opts) => { return nodeModulesTarball(manifest, opts) } - return pacote.tarball(manifest._resolved, opts) + return pacote.tarball(manifest._resolved, { + ...opts, + Arborist, + }) } module.exports = tarball diff --git a/workspaces/libnpmdiff/package.json b/workspaces/libnpmdiff/package.json index 6b3e12e4fc3fb..cb14dfa3116db 100644 --- a/workspaces/libnpmdiff/package.json +++ b/workspaces/libnpmdiff/package.json @@ -47,13 +47,14 @@ "tap": "^16.0.1" }, "dependencies": { + "@npmcli/arborist": "^6.0.0-pre.2", "@npmcli/disparity-colors": "^2.0.0", "@npmcli/installed-package-contents": "^1.0.7", "binary-extensions": "^2.2.0", "diff": "^5.1.0", "minimatch": "^5.0.1", "npm-package-arg": "^9.0.1", - "pacote": "^13.6.1", + "pacote": "^14.0.0-pre.3", "tar": "^6.1.0" }, "templateOSS": { diff --git a/workspaces/libnpmexec/package.json b/workspaces/libnpmexec/package.json index 0f2dde78fcc23..2d4df8579047f 100644 --- a/workspaces/libnpmexec/package.json +++ b/workspaces/libnpmexec/package.json @@ -66,7 +66,7 @@ "mkdirp-infer-owner": "^2.0.0", "npm-package-arg": "^9.0.1", "npmlog": "^6.0.2", - "pacote": "^13.6.1", + "pacote": "^14.0.0-pre.3", "proc-log": "^2.0.0", "read": "^1.0.7", "read-package-json-fast": "^2.0.2", diff --git a/workspaces/libnpmpack/lib/index.js b/workspaces/libnpmpack/lib/index.js index dc9cfd2c4e781..93428b37cb269 100644 --- a/workspaces/libnpmpack/lib/index.js +++ b/workspaces/libnpmpack/lib/index.js @@ -5,6 +5,7 @@ const npa = require('npm-package-arg') const runScript = require('@npmcli/run-script') const path = require('path') const util = require('util') +const Arborist = require('@npmcli/arborist') const writeFile = util.promisify(require('fs').writeFile) module.exports = pack @@ -33,6 +34,7 @@ async function pack (spec = 'file:.', opts = {}) { // packs tarball const tarball = await pacote.tarball(manifest._resolved, { ...opts, + Arborist, integrity: manifest._integrity, }) diff --git a/workspaces/libnpmpack/package.json b/workspaces/libnpmpack/package.json index 96c9589a8cad3..20dc024ae1e13 100644 --- a/workspaces/libnpmpack/package.json +++ b/workspaces/libnpmpack/package.json @@ -35,9 +35,10 @@ "bugs": "https://github.com/npm/libnpmpack/issues", "homepage": "https://npmjs.com/package/libnpmpack", "dependencies": { + "@npmcli/arborist": "^6.0.0-pre.2", "@npmcli/run-script": "^4.1.3", "npm-package-arg": "^9.0.1", - "pacote": "^13.6.1" + "pacote": "^14.0.0-pre.3" }, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" From 45e43f1e87b8811a382c153488c5ad626fc59fc0 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Thu, 29 Sep 2022 10:02:02 -0700 Subject: [PATCH 8/9] chore(libnpmpack): add sleep to tests to reduce flakiness --- workspaces/libnpmpack/test/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workspaces/libnpmpack/test/index.js b/workspaces/libnpmpack/test/index.js index b8007efe0b69e..d9ec1d12a6628 100644 --- a/workspaces/libnpmpack/test/index.js +++ b/workspaces/libnpmpack/test/index.js @@ -37,8 +37,8 @@ t.test('writes tarball to file when dryRun === false', async t => { name: 'my-cool-pkg', version: '1.0.0', scripts: { - prepack: 'touch prepack', - postpack: 'touch postpack', + prepack: 'touch prepack && sleep 1', + postpack: 'sleep 1 && touch postpack', }, }, null, 2), }) From b3977743be36b49f43c13cf116044731ed16d960 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 00:29:40 +0000 Subject: [PATCH 9/9] chore: release 9.0.0-pre.3 --- .release-please-manifest.json | 14 +++++----- CHANGELOG.md | 37 ++++++++++++++++++++++++++ package-lock.json | 38 +++++++++++++-------------- package.json | 14 +++++----- workspaces/arborist/CHANGELOG.md | 10 +++++++ workspaces/arborist/package.json | 2 +- workspaces/libnpmdiff/CHANGELOG.md | 14 ++++++++++ workspaces/libnpmdiff/package.json | 4 +-- workspaces/libnpmexec/CHANGELOG.md | 14 ++++++++++ workspaces/libnpmexec/package.json | 4 +-- workspaces/libnpmfund/CHANGELOG.md | 6 +++++ workspaces/libnpmfund/package.json | 4 +-- workspaces/libnpmpack/CHANGELOG.md | 14 ++++++++++ workspaces/libnpmpack/package.json | 4 +-- workspaces/libnpmpublish/CHANGELOG.md | 14 ++++++++++ workspaces/libnpmpublish/package.json | 4 +-- 16 files changed, 153 insertions(+), 44 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 6a8d9b04e0f32..b36c4d29f6a2d 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,14 +1,14 @@ { - ".": "9.0.0-pre.2", - "workspaces/arborist": "6.0.0-pre.2", + ".": "9.0.0-pre.3", + "workspaces/arborist": "6.0.0-pre.3", "workspaces/libnpmaccess": "7.0.0-pre.1", - "workspaces/libnpmdiff": "5.0.0-pre.0", - "workspaces/libnpmexec": "5.0.0-pre.2", - "workspaces/libnpmfund": "4.0.0-pre.2", + "workspaces/libnpmdiff": "5.0.0-pre.1", + "workspaces/libnpmexec": "5.0.0-pre.3", + "workspaces/libnpmfund": "4.0.0-pre.3", "workspaces/libnpmhook": "9.0.0-pre.0", "workspaces/libnpmorg": "5.0.0-pre.0", - "workspaces/libnpmpack": "5.0.0-pre.0", - "workspaces/libnpmpublish": "7.0.0-pre.0", + "workspaces/libnpmpack": "5.0.0-pre.1", + "workspaces/libnpmpublish": "7.0.0-pre.1", "workspaces/libnpmsearch": "6.0.0-pre.0", "workspaces/libnpmteam": "5.0.0-pre.0", "workspaces/libnpmversion": "4.0.0-pre.0" diff --git a/CHANGELOG.md b/CHANGELOG.md index 689f7f19f8832..56800101dc402 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ # Changelog +## [9.0.0-pre.3](https://github.com/npm/cli/compare/v9.0.0-pre.2...v9.0.0-pre.3) (2022-09-30) + +### ⚠️ BREAKING CHANGES + +* `npm pack` now follows a strict order of operations when applying ignore rules. If a files array is present in the package.json, then rules in .gitignore and .npmignore files from the root will be ignored. +* `--timing` file changes: + - When run with the `--timing` flag, `npm` now writes timing data to a + file alongside the debug log data, respecting the `logs-dir` option and + falling back to `/_logs/` dir, instead of directly inside the + cache directory. + - The timing file data is no longer newline delimited JSON, and instead + each run will create a uniquely named `-timing.json` file, with the + `` portion being the same as the debug log. + - Finally, the data inside the file now has three top level keys, + `metadata`, `timers, and `unfinishedTimers` instead of everything being + a top level key. + +### Features + +* [`3ae796d`](https://github.com/npm/cli/commit/3ae796d937bd36a5b1b9fd6e9e8473b4f2ddc32d) implement new `npm-packlist` behavior (@lukekarrys) +* [`e64d69a`](https://github.com/npm/cli/commit/e64d69aedecc0943425605b3a6dc68aec3ad93aa) [#5581](https://github.com/npm/cli/pull/5581) write eresolve error files to the logs directory (@lukekarrys) +* [`3445da0`](https://github.com/npm/cli/commit/3445da0138f9eed9d73d2b3f5f451fcc1fa2e3fe) timings are now written alongside debug log files (@lukekarrys) + +### Documentation + +* [`f0e7584`](https://github.com/npm/cli/commit/f0e758494698d9dd8a58d07bf71c87608c36869e) [#5601](https://github.com/npm/cli/pull/5601) update docs/logging for new --access default (@wraithgar) + +### Dependencies + +* [`bc21552`](https://github.com/npm/cli/commit/bc2155247d00b7a868c414f4bc86993069b035f9) [#5603](https://github.com/npm/cli/pull/5603) `npm-package-arg@9.1.2` +* [Workspace](https://github.com/npm/cli/compare/arborist-v6.0.0-pre.2...arborist-v6.0.0-pre.3): `@npmcli/arborist@6.0.0-pre.3` +* [Workspace](https://github.com/npm/cli/compare/libnpmdiff-v5.0.0-pre.0...libnpmdiff-v5.0.0-pre.1): `libnpmdiff@5.0.0-pre.1` +* [Workspace](https://github.com/npm/cli/compare/libnpmexec-v5.0.0-pre.2...libnpmexec-v5.0.0-pre.3): `libnpmexec@5.0.0-pre.3` +* [Workspace](https://github.com/npm/cli/compare/libnpmfund-v4.0.0-pre.2...libnpmfund-v4.0.0-pre.3): `libnpmfund@4.0.0-pre.3` +* [Workspace](https://github.com/npm/cli/compare/libnpmpack-v5.0.0-pre.0...libnpmpack-v5.0.0-pre.1): `libnpmpack@5.0.0-pre.1` +* [Workspace](https://github.com/npm/cli/compare/libnpmpublish-v7.0.0-pre.0...libnpmpublish-v7.0.0-pre.1): `libnpmpublish@7.0.0-pre.1` + ## [9.0.0-pre.2](https://github.com/npm/cli/compare/v9.0.0-pre.1...v9.0.0-pre.2) (2022-09-23) ### ⚠️ BREAKING CHANGES diff --git a/package-lock.json b/package-lock.json index e455502bd7466..4c8acfb869429 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "npm", - "version": "9.0.0-pre.2", + "version": "9.0.0-pre.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "npm", - "version": "9.0.0-pre.2", + "version": "9.0.0-pre.3", "bundleDependencies": [ "@isaacs/string-locale-compare", "@npmcli/arborist", @@ -91,7 +91,7 @@ ], "dependencies": { "@isaacs/string-locale-compare": "^1.1.0", - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/ci-detect": "^2.0.0", "@npmcli/config": "^4.2.1", "@npmcli/fs": "^2.1.0", @@ -115,13 +115,13 @@ "is-cidr": "^4.0.2", "json-parse-even-better-errors": "^2.3.1", "libnpmaccess": "^7.0.0-pre.1", - "libnpmdiff": "^5.0.0-pre.0", - "libnpmexec": "^5.0.0-pre.2", - "libnpmfund": "^4.0.0-pre.2", + "libnpmdiff": "^5.0.0-pre.1", + "libnpmexec": "^5.0.0-pre.3", + "libnpmfund": "^4.0.0-pre.3", "libnpmhook": "^9.0.0-pre.0", "libnpmorg": "^5.0.0-pre.0", - "libnpmpack": "^5.0.0-pre.0", - "libnpmpublish": "^7.0.0-pre.0", + "libnpmpack": "^5.0.0-pre.1", + "libnpmpublish": "^7.0.0-pre.1", "libnpmsearch": "^6.0.0-pre.0", "libnpmteam": "^5.0.0-pre.0", "libnpmversion": "^4.0.0-pre.0", @@ -13828,7 +13828,7 @@ }, "workspaces/arborist": { "name": "@npmcli/arborist", - "version": "6.0.0-pre.2", + "version": "6.0.0-pre.3", "license": "ISC", "dependencies": { "@isaacs/string-locale-compare": "^1.1.0", @@ -13903,10 +13903,10 @@ } }, "workspaces/libnpmdiff": { - "version": "5.0.0-pre.0", + "version": "5.0.0-pre.1", "license": "ISC", "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/disparity-colors": "^2.0.0", "@npmcli/installed-package-contents": "^1.0.7", "binary-extensions": "^2.2.0", @@ -13926,10 +13926,10 @@ } }, "workspaces/libnpmexec": { - "version": "5.0.0-pre.2", + "version": "5.0.0-pre.3", "license": "ISC", "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/ci-detect": "^2.0.0", "@npmcli/fs": "^2.1.1", "@npmcli/run-script": "^4.2.0", @@ -13957,10 +13957,10 @@ } }, "workspaces/libnpmfund": { - "version": "4.0.0-pre.2", + "version": "4.0.0-pre.3", "license": "ISC", "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2" + "@npmcli/arborist": "^6.0.0-pre.3" }, "devDependencies": { "@npmcli/eslint-config": "^3.1.0", @@ -14007,10 +14007,10 @@ } }, "workspaces/libnpmpack": { - "version": "5.0.0-pre.0", + "version": "5.0.0-pre.1", "license": "ISC", "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/run-script": "^4.1.3", "npm-package-arg": "^9.0.1", "pacote": "^14.0.0-pre.3" @@ -14026,7 +14026,7 @@ } }, "workspaces/libnpmpublish": { - "version": "7.0.0-pre.0", + "version": "7.0.0-pre.1", "license": "ISC", "dependencies": { "normalize-package-data": "^4.0.0", @@ -14038,7 +14038,7 @@ "devDependencies": { "@npmcli/eslint-config": "^3.1.0", "@npmcli/template-oss": "4.4.1", - "libnpmpack": "^5.0.0-pre.0", + "libnpmpack": "^5.0.0-pre.1", "lodash.clonedeep": "^4.5.0", "nock": "^13.2.4", "tap": "^16.0.1" diff --git a/package.json b/package.json index a457bf5ed3aa2..be4d7f26d00c5 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "9.0.0-pre.2", + "version": "9.0.0-pre.3", "name": "npm", "description": "a package manager for JavaScript", "workspaces": [ @@ -56,7 +56,7 @@ }, "dependencies": { "@isaacs/string-locale-compare": "^1.1.0", - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/ci-detect": "^2.0.0", "@npmcli/config": "^4.2.1", "@npmcli/fs": "^2.1.0", @@ -80,13 +80,13 @@ "is-cidr": "^4.0.2", "json-parse-even-better-errors": "^2.3.1", "libnpmaccess": "^7.0.0-pre.1", - "libnpmdiff": "^5.0.0-pre.0", - "libnpmexec": "^5.0.0-pre.2", - "libnpmfund": "^4.0.0-pre.2", + "libnpmdiff": "^5.0.0-pre.1", + "libnpmexec": "^5.0.0-pre.3", + "libnpmfund": "^4.0.0-pre.3", "libnpmhook": "^9.0.0-pre.0", "libnpmorg": "^5.0.0-pre.0", - "libnpmpack": "^5.0.0-pre.0", - "libnpmpublish": "^7.0.0-pre.0", + "libnpmpack": "^5.0.0-pre.1", + "libnpmpublish": "^7.0.0-pre.1", "libnpmsearch": "^6.0.0-pre.0", "libnpmteam": "^5.0.0-pre.0", "libnpmversion": "^4.0.0-pre.0", diff --git a/workspaces/arborist/CHANGELOG.md b/workspaces/arborist/CHANGELOG.md index cd8e412efb9e7..da71aab1a6487 100644 --- a/workspaces/arborist/CHANGELOG.md +++ b/workspaces/arborist/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [6.0.0-pre.3](https://github.com/npm/cli/compare/arborist-v6.0.0-pre.2...arborist-v6.0.0-pre.3) (2022-09-30) + +### ⚠️ BREAKING CHANGES + +* `npm pack` now follows a strict order of operations when applying ignore rules. If a files array is present in the package.json, then rules in .gitignore and .npmignore files from the root will be ignored. + +### Features + +* [`3ae796d`](https://github.com/npm/cli/commit/3ae796d937bd36a5b1b9fd6e9e8473b4f2ddc32d) implement new `npm-packlist` behavior (@lukekarrys) + ## [6.0.0-pre.2](https://github.com/npm/cli/compare/arborist-v6.0.0-pre.1...arborist-v6.0.0-pre.2) (2022-09-23) ### Features diff --git a/workspaces/arborist/package.json b/workspaces/arborist/package.json index 243b9d4675017..33b0794a21910 100644 --- a/workspaces/arborist/package.json +++ b/workspaces/arborist/package.json @@ -1,6 +1,6 @@ { "name": "@npmcli/arborist", - "version": "6.0.0-pre.2", + "version": "6.0.0-pre.3", "description": "Manage node_modules trees", "dependencies": { "@isaacs/string-locale-compare": "^1.1.0", diff --git a/workspaces/libnpmdiff/CHANGELOG.md b/workspaces/libnpmdiff/CHANGELOG.md index 96a402f9179a1..46c76e3489791 100644 --- a/workspaces/libnpmdiff/CHANGELOG.md +++ b/workspaces/libnpmdiff/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [5.0.0-pre.1](https://github.com/npm/cli/compare/libnpmdiff-v5.0.0-pre.0...libnpmdiff-v5.0.0-pre.1) (2022-09-30) + +### ⚠️ BREAKING CHANGES + +* `npm pack` now follows a strict order of operations when applying ignore rules. If a files array is present in the package.json, then rules in .gitignore and .npmignore files from the root will be ignored. + +### Features + +* [`3ae796d`](https://github.com/npm/cli/commit/3ae796d937bd36a5b1b9fd6e9e8473b4f2ddc32d) implement new `npm-packlist` behavior (@lukekarrys) + +### Dependencies + +* [Workspace](https://github.com/npm/cli/compare/arborist-v6.0.0-pre.2...arborist-v6.0.0-pre.3): `@npmcli/arborist@6.0.0-pre.3` + ## [5.0.0-pre.0](https://github.com/npm/cli/compare/libnpmdiff-v4.0.5...libnpmdiff-v5.0.0-pre.0) (2022-09-08) ### ⚠ BREAKING CHANGES diff --git a/workspaces/libnpmdiff/package.json b/workspaces/libnpmdiff/package.json index cb14dfa3116db..c933f590edfff 100644 --- a/workspaces/libnpmdiff/package.json +++ b/workspaces/libnpmdiff/package.json @@ -1,6 +1,6 @@ { "name": "libnpmdiff", - "version": "5.0.0-pre.0", + "version": "5.0.0-pre.1", "description": "The registry diff", "repository": { "type": "git", @@ -47,7 +47,7 @@ "tap": "^16.0.1" }, "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/disparity-colors": "^2.0.0", "@npmcli/installed-package-contents": "^1.0.7", "binary-extensions": "^2.2.0", diff --git a/workspaces/libnpmexec/CHANGELOG.md b/workspaces/libnpmexec/CHANGELOG.md index 972bf585f3cab..1bc75bde5d2e3 100644 --- a/workspaces/libnpmexec/CHANGELOG.md +++ b/workspaces/libnpmexec/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [5.0.0-pre.3](https://github.com/npm/cli/compare/libnpmexec-v5.0.0-pre.2...libnpmexec-v5.0.0-pre.3) (2022-09-30) + +### ⚠️ BREAKING CHANGES + +* `npm pack` now follows a strict order of operations when applying ignore rules. If a files array is present in the package.json, then rules in .gitignore and .npmignore files from the root will be ignored. + +### Features + +* [`3ae796d`](https://github.com/npm/cli/commit/3ae796d937bd36a5b1b9fd6e9e8473b4f2ddc32d) implement new `npm-packlist` behavior (@lukekarrys) + +### Dependencies + +* [Workspace](https://github.com/npm/cli/compare/arborist-v6.0.0-pre.2...arborist-v6.0.0-pre.3): `@npmcli/arborist@6.0.0-pre.3` + ## [5.0.0-pre.2](https://github.com/npm/cli/compare/libnpmexec-v5.0.0-pre.1...libnpmexec-v5.0.0-pre.2) (2022-09-23) ### Dependencies diff --git a/workspaces/libnpmexec/package.json b/workspaces/libnpmexec/package.json index 2d4df8579047f..48bd5e0075645 100644 --- a/workspaces/libnpmexec/package.json +++ b/workspaces/libnpmexec/package.json @@ -1,6 +1,6 @@ { "name": "libnpmexec", - "version": "5.0.0-pre.2", + "version": "5.0.0-pre.3", "files": [ "bin/", "lib/" @@ -58,7 +58,7 @@ "tap": "^16.0.1" }, "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/ci-detect": "^2.0.0", "@npmcli/fs": "^2.1.1", "@npmcli/run-script": "^4.2.0", diff --git a/workspaces/libnpmfund/CHANGELOG.md b/workspaces/libnpmfund/CHANGELOG.md index 440951a45b302..51f191203dcd7 100644 --- a/workspaces/libnpmfund/CHANGELOG.md +++ b/workspaces/libnpmfund/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [4.0.0-pre.3](https://github.com/npm/cli/compare/libnpmfund-v4.0.0-pre.2...libnpmfund-v4.0.0-pre.3) (2022-09-30) + +### Dependencies + +* [Workspace](https://github.com/npm/cli/compare/arborist-v6.0.0-pre.2...arborist-v6.0.0-pre.3): `@npmcli/arborist@6.0.0-pre.3` + ## [4.0.0-pre.2](https://github.com/npm/cli/compare/libnpmfund-v4.0.0-pre.1...libnpmfund-v4.0.0-pre.2) (2022-09-23) ### Dependencies diff --git a/workspaces/libnpmfund/package.json b/workspaces/libnpmfund/package.json index 43a90df172d53..81deb03ac9188 100644 --- a/workspaces/libnpmfund/package.json +++ b/workspaces/libnpmfund/package.json @@ -1,6 +1,6 @@ { "name": "libnpmfund", - "version": "4.0.0-pre.2", + "version": "4.0.0-pre.3", "main": "lib/index.js", "files": [ "bin/", @@ -46,7 +46,7 @@ "tap": "^16.0.1" }, "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2" + "@npmcli/arborist": "^6.0.0-pre.3" }, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" diff --git a/workspaces/libnpmpack/CHANGELOG.md b/workspaces/libnpmpack/CHANGELOG.md index b38e01146ddc8..72d52f802b852 100644 --- a/workspaces/libnpmpack/CHANGELOG.md +++ b/workspaces/libnpmpack/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [5.0.0-pre.1](https://github.com/npm/cli/compare/libnpmpack-v5.0.0-pre.0...libnpmpack-v5.0.0-pre.1) (2022-09-30) + +### ⚠️ BREAKING CHANGES + +* `npm pack` now follows a strict order of operations when applying ignore rules. If a files array is present in the package.json, then rules in .gitignore and .npmignore files from the root will be ignored. + +### Features + +* [`3ae796d`](https://github.com/npm/cli/commit/3ae796d937bd36a5b1b9fd6e9e8473b4f2ddc32d) implement new `npm-packlist` behavior (@lukekarrys) + +### Dependencies + +* [Workspace](https://github.com/npm/cli/compare/arborist-v6.0.0-pre.2...arborist-v6.0.0-pre.3): `@npmcli/arborist@6.0.0-pre.3` + ## [5.0.0-pre.0](https://github.com/npm/cli/compare/libnpmpack-v4.1.3...libnpmpack-v5.0.0-pre.0) (2022-09-08) ### ⚠ BREAKING CHANGES diff --git a/workspaces/libnpmpack/package.json b/workspaces/libnpmpack/package.json index 20dc024ae1e13..aa0b27036bddd 100644 --- a/workspaces/libnpmpack/package.json +++ b/workspaces/libnpmpack/package.json @@ -1,6 +1,6 @@ { "name": "libnpmpack", - "version": "5.0.0-pre.0", + "version": "5.0.0-pre.1", "description": "Programmatic API for the bits behind npm pack", "author": "GitHub Inc.", "main": "lib/index.js", @@ -35,7 +35,7 @@ "bugs": "https://github.com/npm/libnpmpack/issues", "homepage": "https://npmjs.com/package/libnpmpack", "dependencies": { - "@npmcli/arborist": "^6.0.0-pre.2", + "@npmcli/arborist": "^6.0.0-pre.3", "@npmcli/run-script": "^4.1.3", "npm-package-arg": "^9.0.1", "pacote": "^14.0.0-pre.3" diff --git a/workspaces/libnpmpublish/CHANGELOG.md b/workspaces/libnpmpublish/CHANGELOG.md index 2c69ced553573..cfc1f93dd8f85 100644 --- a/workspaces/libnpmpublish/CHANGELOG.md +++ b/workspaces/libnpmpublish/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [7.0.0-pre.1](https://github.com/npm/cli/compare/libnpmpublish-v7.0.0-pre.0...libnpmpublish-v7.0.0-pre.1) (2022-09-30) + +### ⚠️ BREAKING CHANGES + +* The default value of `access` is now `public` + +### Features + +* [`525654e`](https://github.com/npm/cli/commit/525654e957a80c7f47472e18240e3c8d94e0568f) default access to `public` (@wraithgar) + +### Documentation + +* [`f0e7584`](https://github.com/npm/cli/commit/f0e758494698d9dd8a58d07bf71c87608c36869e) [#5601](https://github.com/npm/cli/pull/5601) update docs/logging for new --access default (@wraithgar) + ## [7.0.0-pre.0](https://github.com/npm/cli/compare/libnpmpublish-v6.0.5...libnpmpublish-v7.0.0-pre.0) (2022-09-08) ### ⚠ BREAKING CHANGES diff --git a/workspaces/libnpmpublish/package.json b/workspaces/libnpmpublish/package.json index ea2fdc181d0c0..8bfa3baf69f60 100644 --- a/workspaces/libnpmpublish/package.json +++ b/workspaces/libnpmpublish/package.json @@ -1,6 +1,6 @@ { "name": "libnpmpublish", - "version": "7.0.0-pre.0", + "version": "7.0.0-pre.1", "description": "Programmatic API for the bits behind npm publish and unpublish", "author": "GitHub Inc.", "main": "lib/index.js", @@ -26,7 +26,7 @@ "devDependencies": { "@npmcli/eslint-config": "^3.1.0", "@npmcli/template-oss": "4.4.1", - "libnpmpack": "^5.0.0-pre.0", + "libnpmpack": "^5.0.0-pre.1", "lodash.clonedeep": "^4.5.0", "nock": "^13.2.4", "tap": "^16.0.1"