From 70f95927077f0820f01fe841da62fba88c4164bc Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Thu, 13 Aug 2026 23:35:25 +0900 Subject: [PATCH] feat(cli): auto-detect embedded private packages [RED-862] Automatically detect lockfile packages that Checkly runners cannot fetch from the public npm registry and embed their tarballs into Playwright code bundles, alongside the explicit checks.embeddedPackages list. Enabled by default (checks.detectEmbeddedPackages, per-run --no-detect-embedded-packages). Private package names never leave the machine unless explicitly opted in: detection uses zero-network public proofs, @scope:registry mappings, and the project's own Sonatype Nexus REST API, with an opt-in public-registry integrity diff (checks.detectEmbeddedPackagesFallback) for anything left undecided. Results are cached (summary keyed by lockfile + registry config + credentials + explicit specs, plus immutable per-entry integrity-proof verdicts); degraded runs are never summary-cached. All detection failures fail soft with warnings naming the actual cause; explicit entries remain strictly fatal. Co-Authored-By: Claude Fable 5 --- .../references/configure-playwright-checks.md | 3 +- .../cli/src/commands/debug/parse-project.ts | 2 + packages/cli/src/commands/deploy.ts | 7 + packages/cli/src/commands/pw-test.ts | 7 + packages/cli/src/commands/test.ts | 7 + packages/cli/src/commands/validate.ts | 2 + .../test-embedded-packages-detect/.npmrc | 2 + .../checkly.config.ts | 19 + .../checkly.detect-off.config.ts | 20 + .../package.json | 7 + .../playwright.config.ts | 6 + .../pnpm-lock.yaml | 57 ++ .../tests/example.spec.ts | 6 + .../__tests__/playwright-check.spec.ts | 75 +- .../project-embedded-packages.spec.ts | 13 +- packages/cli/src/constructs/session.ts | 11 +- .../__tests__/checkly-config-loader.spec.ts | 12 + .../detect-embedded-packages-bad-type.js | 11 + .../configs/detect-fallback-bad-value.js | 11 + .../cli/src/services/checkly-config-loader.ts | 58 ++ .../__tests__/detection-cache.spec.ts | 163 ++++ .../__tests__/detection.spec.ts | 422 +++++++++ .../__tests__/integrity.spec.ts | 31 +- .../__tests__/materializer.spec.ts | 819 ++++++++++++++++++ .../embedded-packages/__tests__/npmrc.spec.ts | 36 +- .../embedded-packages/detection-cache.ts | 267 ++++++ .../services/embedded-packages/detection.ts | 511 +++++++++++ .../services/embedded-packages/integrity.ts | 20 + .../embedded-packages/lockfile-packages.ts | 4 +- .../embedded-packages/materializer.ts | 628 +++++++++++++- .../src/services/embedded-packages/npmrc.ts | 78 +- packages/cli/src/services/project-parser.ts | 6 + 32 files changed, 3294 insertions(+), 27 deletions(-) create mode 100644 packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/.npmrc create mode 100644 packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.config.ts create mode 100644 packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.detect-off.config.ts create mode 100644 packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/package.json create mode 100644 packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/playwright.config.ts create mode 100644 packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/pnpm-lock.yaml create mode 100644 packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/tests/example.spec.ts create mode 100644 packages/cli/src/services/__tests__/fixtures/configs/detect-embedded-packages-bad-type.js create mode 100644 packages/cli/src/services/__tests__/fixtures/configs/detect-fallback-bad-value.js create mode 100644 packages/cli/src/services/embedded-packages/__tests__/detection-cache.spec.ts create mode 100644 packages/cli/src/services/embedded-packages/__tests__/detection.spec.ts create mode 100644 packages/cli/src/services/embedded-packages/detection-cache.ts create mode 100644 packages/cli/src/services/embedded-packages/detection.ts diff --git a/packages/cli/src/ai-context/references/configure-playwright-checks.md b/packages/cli/src/ai-context/references/configure-playwright-checks.md index 2430d5577..f7b9a41b7 100644 --- a/packages/cli/src/ai-context/references/configure-playwright-checks.md +++ b/packages/cli/src/ai-context/references/configure-playwright-checks.md @@ -14,7 +14,8 @@ - Use `installCommand` only when the default package-manager install command is not enough. - Checkly caches installed dependencies between runs, keyed off the lock file, `package.json` and `.npmrc` contents. To force a reinstall declaratively, set `caching.dependencyCache.version` (a string or a safe integer) at the top level of `checkly.config.ts` (not per check — one code bundle serves all Playwright Check Suites) and change its value whenever the cache should be invalidated; scheduled checks pick up the change on the next `checkly deploy`. Unset or empty-string values leave the cache key unchanged, so a dynamic value such as `version: process.env.DEPENDENCY_CACHE_VERSION` is safe when the variable is not always set. For a one-off reinstall during an ad-hoc run, use the `--refresh-cache` flag available on the run/test commands (`checkly test`, `checkly pw-test`, `checkly trigger`, `checkly checks run`) instead; the config value is the persistent knob that also applies to deployed, scheduled checks. - In Checkly CLI v8.0.0 and later, `include` patterns resolve relative to the Playwright config directory, not the project root. If `playwrightConfigPath` points to a subdirectory, adjust `include` globs. Example: `playwrightConfigPath: "./e2e/playwright.config.ts"` with a root fixture at `fixtures/data.json` needs `include: ["../fixtures/data.json"]`. -- If dependencies come from a private registry that Checkly's infrastructure cannot reach (for example an intranet-only Nexus mirror), list them in `checks.embeddedPackages` in `checkly.config.ts`. Each entry is a package name (embeds every version found in the lockfile) or an exact `name@version` pin; names may contain `*` wildcards (`@acme/*`, `acme-*`, `@acme/*-utils`) where each `*` matches any run of characters except `/` (never crossing the scope separator); as long as a spec matches at least one registry package, matches that cannot be embedded are skipped (workspace members silently, git/file/URL dependencies and integrity-less entries with a warning since the runner must fetch those itself), while a spec whose only matches cannot be embedded — or that matches nothing at all — is an error; a pattern embeds every lockfile version of every package it matches, so scope it to the packages the runner genuinely cannot fetch. List every unreachable package by name, including private packages that only appear as transitive dependencies of other private packages — dependencies of listed packages are not embedded automatically. The CLI resolves entries against the workspace-root lockfile (`pnpm-lock.yaml` or `package-lock.json`), reuses tarballs from local caches (its own, then npm's) or downloads them from the registry configured in `.npmrc`, verifies each against the lockfile's integrity hash, and ships them inside the code bundle at `.checkly/embedded-packages/*.tgz`, where the runner serves them through a local registry during install. Downloads are cached under the workspace root's `node_modules/.cache/checkly` (in a monorepo that is the repo root, not the member package; override with `CHECKLY_CACHE_DIR`; a per-user cache dir is the fallback when the project location isn't writable), so nothing lands in the project outside `node_modules`. CI setups that cache `node_modules` — or platforms that preserve `node_modules/.cache` — persist the tarballs automatically; otherwise persist `CHECKLY_CACHE_DIR` in CI to avoid re-downloading (note `npm ci` deletes `node_modules` wholesale, unlike incremental pnpm installs). The machine running `checkly deploy`/`test` needs registry access on a cold cache. Applies to Playwright Check Suites only, not browser or multistep checks. +- Dependencies that Checkly's infrastructure cannot fetch from the public npm registry (for example packages from an intranet-only Nexus mirror) are **detected and embedded into the code bundle automatically** during `deploy`/`test`/`pw-test` (`checks.detectEmbeddedPackages`, default `true`; per-run override `--no-detect-embedded-packages`). Private package names never leave the machine by default: detection needs no network when the effective registry is the public one, packages from a scoped registry (`@scope:registry` in `.npmrc`) are embedded without any lookup, and remaining undecided packages are resolved by asking the private registry which packages it hosts (Sonatype Nexus REST API, using the `.npmrc` credentials; the credentials must be able to browse every npm hosted repository on the instance) — each package is checked against the instance its lockfile-recorded `resolved` URL points at; packages whose lockfile records no source URL (`pnpm-lock.yaml` records none) are classified by the configured registry's inventory, where hosted means embed and absent means public; a recorded source that isn't a Nexus content URL is checked against the configured registry only when it shares that registry's host, and only to confirm the package is hosted there (a package the instance doesn't host stays undecided rather than being assumed public). Results are cached keyed by the lockfile, so repeat runs are free. Whatever detection leaves undecided — because the registry API is unavailable (no REST access, or a non-Nexus registry), because a same-origin recorded source is not hosted on the instance even though the API works, or because the registry configuration itself cannot be resolved (e.g. an unset environment variable referenced in `.npmrc`, which is also warned about separately) — is skipped with a warning; set `checks.detectEmbeddedPackagesFallback: "public-registry"` to instead allow integrity lookups against public npm for those undecided packages (accurate for any registry product, but it transmits the undecided package names, potentially private ones, to the public registry — the exact versions listed in `checks.embeddedPackages` are exempt, though *other* lockfile versions of a pinned name still count as undecided and are transmitted; verdicts obtained this way are cached as immutable proofs and continue to apply after the option is set back to `"skip"` — clear the CLI cache to discard them), or list the packages explicitly. Detection assumes proxy repositories front public npm; packages proxied from *another private* registry are not detected and must be listed explicitly. Detection state lives in the CLI cache — delete `node_modules/.cache/checkly`, plus the per-user cache directory used when the project location isn't writable (`~/Library/Caches/checkly` on macOS, `~/.cache/checkly` on Linux, `%LOCALAPPDATA%\checkly\Cache` on Windows), or point `CHECKLY_CACHE_DIR` elsewhere, to reset it. +- To embed packages explicitly — pinning a version, forcing a public package in, or working with detection off — list them in `checks.embeddedPackages` in `checkly.config.ts`. An explicit entry takes over its package name: detection never adds other versions of an explicitly listed name. Each entry is a package name (embeds every version found in the lockfile) or an exact `name@version` pin; names may contain `*` wildcards (`@acme/*`, `acme-*`, `@acme/*-utils`) where each `*` matches any run of characters except `/` (never crossing the scope separator); as long as a spec matches at least one registry package, matches that cannot be embedded are skipped (workspace members silently, git/file/URL dependencies and integrity-less entries with a warning since the runner must fetch those itself), while a spec whose only matches cannot be embedded — or that matches nothing at all — is an error; a pattern embeds every lockfile version of every package it matches, so scope it to the packages the runner genuinely cannot fetch. With detection disabled, list every unreachable package by name, including private packages that only appear as transitive dependencies of other private packages — dependencies of listed packages are not embedded implicitly. The CLI resolves entries against the workspace-root lockfile (`pnpm-lock.yaml` or `package-lock.json`), reuses tarballs from local caches (its own, then npm's) or downloads them from the registry configured in `.npmrc`, verifies each against the lockfile's integrity hash, and ships them inside the code bundle at `.checkly/embedded-packages/*.tgz`, where the runner serves them through a local registry during install. Downloads are cached under the workspace root's `node_modules/.cache/checkly` (in a monorepo that is the repo root, not the member package; override with `CHECKLY_CACHE_DIR`; a per-user cache dir is the fallback when the project location isn't writable), so nothing lands in the project outside `node_modules`. CI setups that cache `node_modules` — or platforms that preserve `node_modules/.cache` — persist the tarballs automatically; otherwise persist `CHECKLY_CACHE_DIR` in CI to avoid re-downloading (note `npm ci` deletes `node_modules` wholesale, unlike incremental pnpm installs). The machine running `checkly deploy`/`test` needs registry access on a cold cache. Applies to Playwright Check Suites only, not browser or multistep checks. ## Install troubleshooting diff --git a/packages/cli/src/commands/debug/parse-project.ts b/packages/cli/src/commands/debug/parse-project.ts index c357dfb8c..588a06567 100644 --- a/packages/cli/src/commands/debug/parse-project.ts +++ b/packages/cli/src/commands/debug/parse-project.ts @@ -150,6 +150,8 @@ export default class ParseProjectCommand extends Command { playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath, include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, embeddedPackages: checklyConfig.checks?.embeddedPackages, + detectEmbeddedPackages: checklyConfig.checks?.detectEmbeddedPackages, + detectEmbeddedPackagesFallback: checklyConfig.checks?.detectEmbeddedPackagesFallback, playwrightChecks: checklyConfig.checks?.playwrightChecks, loadPlaywrightChecksOnly: emulatePwTest, warnOnWebServerConfig: emulatePwTest && !(includeFlag.length > 0), diff --git a/packages/cli/src/commands/deploy.ts b/packages/cli/src/commands/deploy.ts index 9b5521d5d..c4d4f4547 100644 --- a/packages/cli/src/commands/deploy.ts +++ b/packages/cli/src/commands/deploy.ts @@ -97,6 +97,11 @@ export default class Deploy extends AuthCommand { allowNo: true, env: 'CHECKLY_VERIFY_RUNTIME_DEPENDENCIES', }), + 'detect-embedded-packages': Flags.boolean({ + description: '[default: true] Automatically embed dependencies that Checkly cannot fetch from the public npm registry (see checks.detectEmbeddedPackages).', + allowNo: true, + env: 'CHECKLY_DETECT_EMBEDDED_PACKAGES', + }), 'debug-bundle': Flags.boolean({ description: 'Output the project bundle to a file without deploying any resources.', default: false, @@ -179,6 +184,8 @@ export default class Deploy extends AuthCommand { playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath, include: checklyConfig.checks?.include, embeddedPackages: checklyConfig.checks?.embeddedPackages, + detectEmbeddedPackages: flags['detect-embedded-packages'] ?? checklyConfig.checks?.detectEmbeddedPackages, + detectEmbeddedPackagesFallback: checklyConfig.checks?.detectEmbeddedPackagesFallback, playwrightChecks: checklyConfig.checks?.playwrightChecks, }) const repoInfo = getGitInformation(project.repoUrl) diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts index 96c88fccd..875a63594 100644 --- a/packages/cli/src/commands/pw-test.ts +++ b/packages/cli/src/commands/pw-test.ts @@ -112,6 +112,11 @@ export default class PwTestCommand extends AuthCommand { multiple: true, default: [], }), + 'detect-embedded-packages': Flags.boolean({ + description: '[default: true] Automatically embed dependencies that Checkly cannot fetch from the public npm registry (see checks.detectEmbeddedPackages).', + allowNo: true, + env: 'CHECKLY_DETECT_EMBEDDED_PACKAGES', + }), 'install-command': Flags.string({ description: 'Command to install dependencies before running tests.', }), @@ -215,6 +220,8 @@ export default class PwTestCommand extends AuthCommand { playwrightConfigPath, include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, embeddedPackages: checklyConfig.checks?.embeddedPackages, + detectEmbeddedPackages: flags['detect-embedded-packages'] ?? checklyConfig.checks?.detectEmbeddedPackages, + detectEmbeddedPackagesFallback: checklyConfig.checks?.detectEmbeddedPackagesFallback, playwrightChecks: [playwrightCheck], loadPlaywrightChecksOnly: true, warnOnWebServerConfig: !(includeFlag.length > 0), diff --git a/packages/cli/src/commands/test.ts b/packages/cli/src/commands/test.ts index 5d5265419..59e119890 100644 --- a/packages/cli/src/commands/test.ts +++ b/packages/cli/src/commands/test.ts @@ -116,6 +116,11 @@ export default class Test extends AuthCommand { allowNo: true, env: 'CHECKLY_VERIFY_RUNTIME_DEPENDENCIES', }), + 'detect-embedded-packages': Flags.boolean({ + description: '[default: true] Automatically embed dependencies that Checkly cannot fetch from the public npm registry (see checks.detectEmbeddedPackages).', + allowNo: true, + env: 'CHECKLY_DETECT_EMBEDDED_PACKAGES', + }), 'refresh-cache': Flags.boolean({ description: 'Force a fresh install of dependencies and update the cached version.', default: false, @@ -207,6 +212,8 @@ export default class Test extends AuthCommand { playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath, include: checklyConfig.checks?.include, embeddedPackages: checklyConfig.checks?.embeddedPackages, + detectEmbeddedPackages: flags['detect-embedded-packages'] ?? checklyConfig.checks?.detectEmbeddedPackages, + detectEmbeddedPackagesFallback: checklyConfig.checks?.detectEmbeddedPackagesFallback, playwrightChecks: checklyConfig.checks?.playwrightChecks, checkFilter: check => { if (check instanceof HeartbeatMonitor) { diff --git a/packages/cli/src/commands/validate.ts b/packages/cli/src/commands/validate.ts index 24a84adc0..f238f7648 100644 --- a/packages/cli/src/commands/validate.ts +++ b/packages/cli/src/commands/validate.ts @@ -63,6 +63,8 @@ export default class Validate extends AuthCommand { playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath, include: checklyConfig.checks?.include, embeddedPackages: checklyConfig.checks?.embeddedPackages, + detectEmbeddedPackages: checklyConfig.checks?.detectEmbeddedPackages, + detectEmbeddedPackagesFallback: checklyConfig.checks?.detectEmbeddedPackagesFallback, playwrightChecks: checklyConfig.checks?.playwrightChecks, }) diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/.npmrc b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/.npmrc new file mode 100644 index 000000000..1bef5cc0d --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/.npmrc @@ -0,0 +1,2 @@ +registry=https://registry.npmjs.org/ +@acme:registry=https://nexus.local/repository/npm-private/ diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.config.ts new file mode 100644 index 000000000..3dede8ef2 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Check Fixture', + logicalId: 'check-fixture', + checks: { + checkMatch: '**/*.check.ts', + ignoreDirectoriesMatch: [], + playwrightConfigPath: './playwright.config.ts', + playwrightChecks: [ + { + logicalId: 'playwright-check-suite', + name: 'Playwright Check Suite', + } + ], + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.detect-off.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.detect-off.config.ts new file mode 100644 index 000000000..0418dc128 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/checkly.detect-off.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Check Fixture', + logicalId: 'check-fixture', + checks: { + checkMatch: '**/*.check.ts', + ignoreDirectoriesMatch: [], + playwrightConfigPath: './playwright.config.ts', + detectEmbeddedPackages: false, + playwrightChecks: [ + { + logicalId: 'playwright-check-suite', + name: 'Playwright Check Suite', + } + ], + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/package.json new file mode 100644 index 000000000..b12adbc29 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/package.json @@ -0,0 +1,7 @@ +{ + "name": "playwright-bundle-test", + "version": "1.0.0", + "dependencies": { + "@playwright/test": "^1.55.1" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/playwright.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/playwright.config.ts new file mode 100644 index 000000000..eed093e29 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/playwright.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + testDir: './tests', + timeout: 30000, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/pnpm-lock.yaml b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/pnpm-lock.yaml new file mode 100644 index 000000000..f59fa575e --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/pnpm-lock.yaml @@ -0,0 +1,57 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@playwright/test': + specifier: ^1.55.1 + version: 1.59.1 + +packages: + + '@acme/private-utils@1.2.3': + resolution: {integrity: sha512-dnkm3WedrIfH8+nRoHESfj0/DDeZdBTCpP2B5ZUSR/6YsMiOtYmauw1FRb2hDNC00ZLWu8Ya8sZfR2D/s1VhTQ==} + + '@playwright/test@1.59.1': + resolution: {integrity: sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==} + engines: {node: '>=18'} + hasBin: true + + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + playwright-core@1.59.1: + resolution: {integrity: sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.59.1: + resolution: {integrity: sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==} + engines: {node: '>=18'} + hasBin: true + +snapshots: + + '@acme/private-utils@1.2.3': {} + + '@playwright/test@1.59.1': + dependencies: + playwright: 1.59.1 + + fsevents@2.3.2: + optional: true + + playwright-core@1.59.1: {} + + playwright@1.59.1: + dependencies: + playwright-core: 1.59.1 + optionalDependencies: + fsevents: 2.3.2 diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/tests/example.spec.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/tests/example.spec.ts new file mode 100644 index 000000000..4cbbbc71e --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-embedded-packages-detect/tests/example.spec.ts @@ -0,0 +1,6 @@ +import { test, expect } from '@playwright/test' + +test('basic test', async ({ page }) => { + await page.goto('https://playwright.dev/') + expect(await page.title()).toContain('Playwright') +}) diff --git a/packages/cli/src/constructs/__tests__/playwright-check.spec.ts b/packages/cli/src/constructs/__tests__/playwright-check.spec.ts index 9dd5cc8f5..ecf4a43e4 100644 --- a/packages/cli/src/constructs/__tests__/playwright-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/playwright-check.spec.ts @@ -14,6 +14,13 @@ async function parseProject (fixt: FixtureSandbox, ...args: string[]): Promise

{ }, DEFAULT_TEST_TIMEOUT) }) + describe('bundling with auto-detected embedded packages', () => { + let fixt: FixtureSandbox + let cacheDir: string + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'playwright-check', 'test-cases', 'test-embedded-packages-detect'), + }) + cacheDir = await seedTarballCache('@acme+private-utils@1.2.3.tgz') + }, DEFAULT_TEST_TIMEOUT) + + afterAll(async () => { + await fixt?.destroy() + if (cacheDir) { + await fs.rm(cacheDir, { recursive: true, force: true }) + } + }) + + it('should embed a scope-mapped package without configuration', async () => { + // The fixture's .npmrc maps @acme to a private registry, so detection + // embeds @acme/private-utils with zero network traffic; the tarball + // comes from the pre-seeded CLI cache. + const output = await parseProjectWithOptions(fixt, { env: { CHECKLY_CACHE_DIR: cacheDir } }) + + expect(output.diagnostics.fatal).toBe(false) + + const { + codeBundlePath, + } = output.payload.resources[0].payload as any + + const files = await listTarFiles(codeBundlePath) + + expect(files).toContain('.checkly/embedded-packages/@acme+private-utils@1.2.3.tgz') + }, DEFAULT_TEST_TIMEOUT) + + it('should not embed anything when detection is disabled', async () => { + const output = await parseProjectWithOptions( + fixt, + { env: { CHECKLY_CACHE_DIR: cacheDir } }, + '--config', 'checkly.detect-off.config.ts', + ) + + expect(output.diagnostics.fatal).toBe(false) + + const { + codeBundlePath, + } = output.payload.resources[0].payload as any + + const files = await listTarFiles(codeBundlePath) + + expect(files.some(file => file.startsWith('.checkly/'))).toBe(false) + }, DEFAULT_TEST_TIMEOUT) + }) + describe('embedded packages validation', () => { let fixt: FixtureSandbox diff --git a/packages/cli/src/constructs/__tests__/project-embedded-packages.spec.ts b/packages/cli/src/constructs/__tests__/project-embedded-packages.spec.ts index d859f217c..fcbbd6272 100644 --- a/packages/cli/src/constructs/__tests__/project-embedded-packages.spec.ts +++ b/packages/cli/src/constructs/__tests__/project-embedded-packages.spec.ts @@ -146,12 +146,23 @@ describe('Session.getEmbeddedPackagesMaterializer()', () => { Session.reset() }) - it('returns undefined without configuration', () => { + it('exists by default because detection defaults to on', () => { + expect(Session.getEmbeddedPackagesMaterializer()).toBeDefined() + }) + + it('returns undefined when detection is off and nothing is configured', () => { + Session.detectEmbeddedPackages = false expect(Session.getEmbeddedPackagesMaterializer()).toBeUndefined() Session.embeddedPackages = [] expect(Session.getEmbeddedPackagesMaterializer()).toBeUndefined() }) + it('exists with explicit packages even when detection is off', () => { + Session.detectEmbeddedPackages = false + Session.embeddedPackages = ['some-pkg'] + expect(Session.getEmbeddedPackagesMaterializer()).toBeDefined() + }) + it('memoizes the instance and reset() clears it', () => { Session.embeddedPackages = ['some-pkg'] const first = Session.getEmbeddedPackagesMaterializer() diff --git a/packages/cli/src/constructs/session.ts b/packages/cli/src/constructs/session.ts index 742042409..ef4ed19ed 100644 --- a/packages/cli/src/constructs/session.ts +++ b/packages/cli/src/constructs/session.ts @@ -75,6 +75,8 @@ export class Session { static constructExports: ConstructExport[] = [] static ignoreDirectoriesMatch: string[] = [] static embeddedPackages?: string[] + static detectEmbeddedPackages?: boolean + static detectEmbeddedPackagesFallback?: 'skip' | 'public-registry' static warnOnWebServerConfig?: boolean static packageManager: PackageManager = npmPackageManager static workspace: Result = Err(new Error(`Workspace support not initialized`)) @@ -103,6 +105,8 @@ export class Session { this.constructExports = [] this.ignoreDirectoriesMatch = [] this.embeddedPackages = undefined + this.detectEmbeddedPackages = undefined + this.detectEmbeddedPackagesFallback = undefined this.warnOnWebServerConfig = false this.packageManager = npmPackageManager this.workspace = Err(new Error(`Workspace support not initialized`)) @@ -239,13 +243,16 @@ export class Session { * every concurrently bundling check share one plan and one download run. */ static getEmbeddedPackagesMaterializer (): EmbeddedPackagesMaterializer | undefined { - const specs = this.embeddedPackages - if (specs === undefined || specs.length === 0) { + const specs = this.embeddedPackages ?? [] + const detect = this.detectEmbeddedPackages ?? true + if (specs.length === 0 && !detect) { return undefined } if (this.embeddedPackagesMaterializer === undefined) { this.embeddedPackagesMaterializer = new EmbeddedPackagesMaterializer({ specs, + detect, + detectionFallback: this.detectEmbeddedPackagesFallback, lockfilePath: this.workspace.ok()?.lockfile.ok(), workspaceRoot: this.basePath, contextDir: this.contextPath, diff --git a/packages/cli/src/services/__tests__/checkly-config-loader.spec.ts b/packages/cli/src/services/__tests__/checkly-config-loader.spec.ts index c7b84ab63..f4dadd2ae 100644 --- a/packages/cli/src/services/__tests__/checkly-config-loader.spec.ts +++ b/packages/cli/src/services/__tests__/checkly-config-loader.spec.ts @@ -125,6 +125,18 @@ describe('loadChecklyConfig()', () => { ['embedded-packages-bad-name.js'], )).rejects.toThrow(`is not a valid npm package name`) }) + it('rejects a non-boolean checks.detectEmbeddedPackages', async () => { + await expect(loadChecklyConfig( + path.join(__dirname, 'fixtures', 'configs'), + ['detect-embedded-packages-bad-type.js'], + )).rejects.toThrow(`Config field 'checks.detectEmbeddedPackages' must be a boolean if set`) + }) + it('rejects an invalid checks.detectEmbeddedPackagesFallback', async () => { + await expect(loadChecklyConfig( + path.join(__dirname, 'fixtures', 'configs'), + ['detect-fallback-bad-value.js'], + )).rejects.toThrow(`Config field 'checks.detectEmbeddedPackagesFallback' must be 'skip' or 'public-registry' if set`) + }) it('rejects a checks.embeddedPackages entry with a version range', async () => { await expect(loadChecklyConfig( path.join(__dirname, 'fixtures', 'configs'), diff --git a/packages/cli/src/services/__tests__/fixtures/configs/detect-embedded-packages-bad-type.js b/packages/cli/src/services/__tests__/fixtures/configs/detect-embedded-packages-bad-type.js new file mode 100644 index 000000000..5de0dab64 --- /dev/null +++ b/packages/cli/src/services/__tests__/fixtures/configs/detect-embedded-packages-bad-type.js @@ -0,0 +1,11 @@ +// Plain JS on purpose: bypasses the TypeScript type of ChecklyConfig so the +// runtime validation of checks.detectEmbeddedPackages is what rejects it. +const config = { + projectName: 'test-config-project', + logicalId: 'test-config-project', + checks: { + detectEmbeddedPackages: 'yes', + }, +} + +export default config diff --git a/packages/cli/src/services/__tests__/fixtures/configs/detect-fallback-bad-value.js b/packages/cli/src/services/__tests__/fixtures/configs/detect-fallback-bad-value.js new file mode 100644 index 000000000..bf2a91f26 --- /dev/null +++ b/packages/cli/src/services/__tests__/fixtures/configs/detect-fallback-bad-value.js @@ -0,0 +1,11 @@ +// Plain JS on purpose: bypasses the TypeScript type of ChecklyConfig so the +// runtime validation of checks.detectEmbeddedPackagesFallback rejects it. +const config = { + projectName: 'test-config-project', + logicalId: 'test-config-project', + checks: { + detectEmbeddedPackagesFallback: 'ask-nicely', + }, +} + +export default config diff --git a/packages/cli/src/services/checkly-config-loader.ts b/packages/cli/src/services/checkly-config-loader.ts index d25144321..198572ef2 100644 --- a/packages/cli/src/services/checkly-config-loader.ts +++ b/packages/cli/src/services/checkly-config-loader.ts @@ -134,6 +134,54 @@ export type ChecklyConfig = { * through a local registry during dependency installation. */ embeddedPackages?: string[] + /** + * Whether to automatically detect and embed dependencies that Checkly + * runners cannot fetch from the public npm registry, in addition to + * any explicit `embeddedPackages` entries. Defaults to `true`; the + * `--no-detect-embedded-packages` flag overrides per run. + * + * Detection is free of network traffic when the effective registry is + * the public one; with a private registry, undecided packages are + * resolved by asking that registry which packages it hosts, and the + * result is cached (keyed by the lockfile), so repeat runs cost + * nothing. A detected package never overrides an explicit + * `embeddedPackages` entry for the same name. + * + * Private package names never leave your machine by default: the + * registry interrogation uses the Sonatype Nexus REST API of the + * instance each package's lockfile-recorded source points at (the npm + * credentials must be able to browse every npm hosted repository on + * it). Packages whose lockfile records no source URL at all — + * `pnpm-lock.yaml` records none — are classified by the configured + * registry's inventory: hosted means embed, absent means public. A + * recorded source that doesn't look like a Nexus content URL is + * checked against the configured registry too, but only when it shares + * that registry's host, and only to confirm a package is hosted there + * — a package the instance doesn't host stays unclassified rather + * than being assumed public. See `detectEmbeddedPackagesFallback` for + * what happens to unclassified packages. Detection assumes proxy repositories on your + * registry front the public npm registry — packages served through a + * proxy of *another private* registry are not detected and should be + * listed in `embeddedPackages` explicitly. + */ + detectEmbeddedPackages?: boolean + /** + * What detection does with packages it cannot classify without + * querying the public npm registry — because the registry's REST API + * is not accessible with the configured npm credentials, because a + * package's recorded source shares the configured registry's host but + * is not hosted on it, or because the registry configuration itself + * cannot be resolved (for example an unset environment variable + * referenced in `.npmrc`, which is also warned about separately). + * `'skip'` (the default) leaves them un-embedded and prints a warning; + * `'public-registry'` allows integrity lookups against the public npm + * registry — accurate for any registry product, but it transmits the + * undecided package names (potentially private ones) to the public + * registry. Verdicts obtained from the public registry are cached as + * immutable proofs and continue to apply after the option is set back + * to `'skip'`; clear the CLI cache to discard them. + */ + detectEmbeddedPackagesFallback?: 'skip' | 'public-registry' /** * List of playwright checks that use the defined playwright config path */ @@ -319,6 +367,16 @@ function validateDependencyCacheVersion (config: ChecklyConfig): void { } function validateEmbeddedPackages (config: ChecklyConfig): void { + const detect = config.checks?.detectEmbeddedPackages + if (detect !== undefined && typeof detect !== 'boolean') { + throw new Error(`Config field 'checks.detectEmbeddedPackages' must be a boolean if set`) + } + + const fallback = config.checks?.detectEmbeddedPackagesFallback + if (fallback !== undefined && fallback !== 'skip' && fallback !== 'public-registry') { + throw new Error(`Config field 'checks.detectEmbeddedPackagesFallback' must be 'skip' or 'public-registry' if set`) + } + const embeddedPackages = config.checks?.embeddedPackages if (embeddedPackages === undefined) { return diff --git a/packages/cli/src/services/embedded-packages/__tests__/detection-cache.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/detection-cache.spec.ts new file mode 100644 index 000000000..fd402cc71 --- /dev/null +++ b/packages/cli/src/services/embedded-packages/__tests__/detection-cache.spec.ts @@ -0,0 +1,163 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { DetectionCache, detectionInputDigest, verdictKey } from '../detection-cache.js' +import { parseNpmrc } from '../npmrc.js' + +describe('detectionInputDigest()', () => { + const lockfile = `lockfileVersion: '9.0'\npackages: {}\n` + + it('is stable for identical inputs', () => { + const config = parseNpmrc('registry=https://nexus.local/npm/') + expect(detectionInputDigest(lockfile, config)).toBe(detectionInputDigest(lockfile, config)) + }) + + it('changes when the lockfile changes', () => { + const config = parseNpmrc('registry=https://nexus.local/npm/') + expect(detectionInputDigest(lockfile, config)).not.toBe(detectionInputDigest(`${lockfile}#`, config)) + }) + + it('changes when registry configuration changes', () => { + const a = parseNpmrc('registry=https://nexus.local/npm/') + const b = parseNpmrc('@acme:registry=https://nexus.local/npm-private/') + expect(detectionInputDigest(lockfile, a)).not.toBe(detectionInputDigest(lockfile, b)) + }) + + it('changes when a ${VAR}-referenced registry value changes', () => { + const config = parseNpmrc('registry=${MY_REGISTRY}') + expect(detectionInputDigest(lockfile, config, { MY_REGISTRY: 'https://a.example.com/' })) + .not.toBe(detectionInputDigest(lockfile, config, { MY_REGISTRY: 'https://b.example.com/' })) + }) + + it('changes when a credential rotated behind a ${VAR} reference changes', () => { + const config = parseNpmrc([ + 'registry=https://nexus.local/npm/', + '//nexus.local/npm/:_authToken=${NPM_TOKEN}', + ].join('\n')) + expect(detectionInputDigest(lockfile, config, { NPM_TOKEN: 'token-a' })) + .not.toBe(detectionInputDigest(lockfile, config, { NPM_TOKEN: 'token-b' })) + }) + + it('changes when registry credentials change', () => { + // The registry API filters what it shows by permission, so verdicts + // must not outlive a credentials change. + const a = parseNpmrc('registry=https://nexus.local/npm/') + const b = parseNpmrc([ + 'registry=https://nexus.local/npm/', + '//nexus.local/npm/:_authToken=secret', + ].join('\n')) + expect(detectionInputDigest(lockfile, a)).not.toBe(detectionInputDigest(lockfile, b)) + }) + + it('ignores npm configuration unrelated to registries or credentials', () => { + const a = parseNpmrc('registry=https://nexus.local/npm/') + const b = parseNpmrc([ + 'registry=https://nexus.local/npm/', + 'strict-ssl=false', + ].join('\n')) + expect(detectionInputDigest(lockfile, a)).toBe(detectionInputDigest(lockfile, b)) + }) +}) + +describe('DetectionCache', () => { + let dir: string + let cache: DetectionCache + + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-detection-cache-')) + cache = new DetectionCache(dir) + }) + + afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }) + }) + + it('round-trips a summary by input digest', async () => { + const embedKeys = [verdictKey({ name: '@acme/foo', version: '1.2.3', integrity: 'sha512-aaa' })] + await expect(cache.getSummary('digest-1')).resolves.toBeUndefined() + await cache.putSummary('digest-1', { embedKeys }) + await expect(cache.getSummary('digest-1')).resolves.toEqual({ embedKeys }) + await expect(cache.getSummary('digest-2')).resolves.toBeUndefined() + }) + + it('treats a structurally wrong summary as a miss', async () => { + await cache.putSummary('digest-1', { embedKeys: [] }) + const [file] = (await fs.readdir(dir)).filter(name => name.startsWith('summary-')) + await fs.writeFile(path.join(dir, file), JSON.stringify({ embedKeys: 'not-an-array' })) + await expect(cache.getSummary('digest-1')).resolves.toBeUndefined() + }) + + it('merges verdicts across writes', async () => { + const entryA = { name: 'a', version: '1.0.0', integrity: 'sha512-aaa' } + const entryB = { name: 'b', version: '2.0.0', integrity: 'sha512-bbb' } + await cache.putVerdicts({ [verdictKey(entryA)]: 'embed' }) + await cache.putVerdicts({ [verdictKey(entryB)]: 'public' }) + await expect(cache.getVerdicts()).resolves.toEqual({ + [verdictKey(entryA)]: 'embed', + [verdictKey(entryB)]: 'public', + }) + }) + + it('merges verdicts from every cache root, primary root winning', async () => { + const primary = path.join(dir, 'primary') + const fallback = path.join(dir, 'fallback') + const primaryCache = new DetectionCache(primary) + const fallbackCache = new DetectionCache(fallback) + const entryA = { name: 'a', version: '1.0.0', integrity: 'sha512-aaa' } + const entryB = { name: 'b', version: '2.0.0', integrity: 'sha512-bbb' } + await primaryCache.putVerdicts({ [verdictKey(entryA)]: 'embed' }) + // Overlapping key: the fallback disagrees about entryA — the primary + // root must win. + await fallbackCache.putVerdicts({ [verdictKey(entryA)]: 'public', [verdictKey(entryB)]: 'public' }) + + const multi = new DetectionCache([primary, fallback]) + await expect(multi.getVerdicts()).resolves.toEqual({ + [verdictKey(entryA)]: 'embed', + [verdictKey(entryB)]: 'public', + }) + }) + + it('bounds the verdict map, keeping the freshest entries beyond the cap', async () => { + const bulk = Object.fromEntries( + Array.from({ length: 10_001 }, (_, i) => [`pkg-${i}@1.0.0::sha512-x`, 'public' as const]), + ) + await cache.putVerdicts(bulk) + const fresh = { 'fresh@1.0.0::sha512-y': 'embed' as const } + await cache.putVerdicts(fresh) + await expect(cache.getVerdicts()).resolves.toEqual(fresh) + }) + + it('prunes summaries beyond the retention count', async () => { + for (let i = 0; i < 15; i++) { + // Hex digests, as detectionInputDigest produces. + await cache.putSummary(`abcdef${i.toString(16).padStart(2, '0')}`, { embedKeys: [] }) + } + const files = (await fs.readdir(dir)).filter(name => name.startsWith('summary-')) + expect(files.length).toBeLessThanOrEqual(10) + }) + + it('prunes only strictly older verdict files on write, keeping newer CLIs\' files', async () => { + await fs.writeFile(path.join(dir, 'verdicts-v1.json'), '{}') + await fs.writeFile(path.join(dir, 'verdicts-v99.json'), '{}') + await cache.putVerdicts({ 'a@1.0.0::sha512-aaa': 'embed' }) + const names = await fs.readdir(dir) + expect(names).not.toContain('verdicts-v1.json') + // A newer CLI sharing this cache root must not have its file deleted. + expect(names).toContain('verdicts-v99.json') + // The verdict file's version (2) is decoupled from DETECTOR_VERSION + // (3): a summary-semantics bump must not discard integrity proofs, + // which for opted-in users would mean re-sending private package + // names to the public registry. The literal filename pins that. + expect(names).toContain('verdicts-v2.json') + }) + + it('treats corrupt cache files as misses', async () => { + await cache.putSummary('digest-1', { embedKeys: [] }) + const [file] = (await fs.readdir(dir)).filter(name => name.startsWith('summary-')) + await fs.writeFile(path.join(dir, file), 'not json') + await expect(cache.getSummary('digest-1')).resolves.toBeUndefined() + }) +}) diff --git a/packages/cli/src/services/embedded-packages/__tests__/detection.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/detection.spec.ts new file mode 100644 index 000000000..9bf74077e --- /dev/null +++ b/packages/cli/src/services/embedded-packages/__tests__/detection.spec.ts @@ -0,0 +1,422 @@ +import { createHash } from 'node:crypto' +import http from 'node:http' +import { AddressInfo } from 'node:net' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { + DetectionUnavailableError, + NexusRegistryApi, + classifyEntries, + decideWithHostedInventory, + diffAgainstPublicRegistry, +} from '../detection.js' +import { LockfileRegistryPackage } from '../lockfile-packages.js' +import { parseNpmrc } from '../npmrc.js' + +const entry = (name: string, version: string, integrity: string, tarballUrl?: string): LockfileRegistryPackage => ({ + name, version, integrity, tarballUrl, +}) + +const sha512Of = (content: string) => `sha512-${createHash('sha512').update(content).digest('base64')}` + +describe('classifyEntries()', () => { + it('proves everything public under the default public registry', () => { + const result = classifyEntries([ + entry('foo', '1.0.0', 'sha512-aaa'), + entry('@acme/bar', '2.0.0', 'sha512-bbb'), + ], new Map(), {}) + expect(result.public).toHaveLength(2) + expect(result.embed).toHaveLength(0) + expect(result.undecided).toHaveLength(0) + }) + + it('recognizes the yarnpkg mirror as public', () => { + const config = parseNpmrc('registry=https://registry.yarnpkg.com/') + const result = classifyEntries([entry('foo', '1.0.0', 'sha512-aaa')], config, {}) + expect(result.public).toHaveLength(1) + }) + + it('embeds scope-mapped packages without a lookup', () => { + const config = parseNpmrc([ + 'registry=https://registry.npmjs.org/', + '@acme:registry=https://nexus.local/repository/npm-private/', + ].join('\n')) + const result = classifyEntries([ + entry('@acme/private-utils', '1.2.3', 'sha512-aaa'), + entry('public-pkg', '1.0.0', 'sha512-bbb'), + ], config, {}) + expect(result.embed.map(e => e.name)).toEqual(['@acme/private-utils']) + expect(result.public.map(e => e.name)).toEqual(['public-pkg']) + }) + + it('leaves everything undecided under a non-public default registry', () => { + const config = parseNpmrc('registry=https://nexus.local/repository/npm/') + const result = classifyEntries([ + entry('foo', '1.0.0', 'sha512-aaa'), + entry('@acme/bar', '2.0.0', 'sha512-bbb'), + ], config, {}) + expect(result.undecided).toHaveLength(2) + expect(result.embed).toHaveLength(0) + }) + + it('treats a lockfile-recorded public tarball URL as proof of publicness', () => { + const config = parseNpmrc('registry=https://nexus.local/repository/npm/') + const result = classifyEntries([ + entry('foo', '1.0.0', 'sha512-aaa', 'https://registry.npmjs.org/foo/-/foo-1.0.0.tgz'), + ], config, {}) + expect(result.public.map(e => e.name)).toEqual(['foo']) + }) + + it('lets a scope mapping mark a package private even with a non-public recorded source', () => { + // npm lockfiles record `resolved` for every entry; that must not + // defeat the zero-network scope tier. + const config = parseNpmrc([ + 'registry=https://registry.npmjs.org/', + '@acme:registry=https://nexus.local/repository/npm-private/', + ].join('\n')) + const result = classifyEntries([ + entry('@acme/private-utils', '1.2.3', 'sha512-aaa', + 'https://nexus.local/repository/npm-private/@acme/private-utils/-/private-utils-1.2.3.tgz'), + ], config, {}) + expect(result.embed.map(e => e.name)).toEqual(['@acme/private-utils']) + }) + + it('keeps a scope-mapped entry in the embed tier when its mapping references an unset variable', () => { + // An @scope:registry mapping that fails to expand is never the public + // registry, so the scope tier's no-lookup guarantee must hold — + // 'undecided' could transmit the private name under the opt-in. + const config = parseNpmrc([ + 'registry=https://registry.npmjs.org/', + '@broken:registry=${RED862_UNSET}', + ].join('\n')) + const result = classifyEntries([ + entry('@broken/pkg', '1.0.0', 'sha512-aaa'), + entry('fine-pkg', '1.0.0', 'sha512-bbb'), + ], config, {}) + expect(result.embed.map(e => e.name)).toEqual(['@broken/pkg']) + expect(result.public.map(e => e.name)).toEqual(['fine-pkg']) + }) + + it('classifies an unscoped entry as undecided when the default registry mapping references an unset variable', () => { + const config = parseNpmrc('registry=${RED862_UNSET}') + const result = classifyEntries([entry('some-pkg', '1.0.0', 'sha512-aaa')], config, {}) + expect(result.undecided.map(e => e.name)).toEqual(['some-pkg']) + }) + + it('never lets registry configuration vouch for a non-public recorded source', () => { + // The artifact demonstrably came from a non-public host; a later + // .npmrc pointing at the public registry proves nothing about it. + const config = parseNpmrc('registry=https://registry.npmjs.org/') + const result = classifyEntries([ + entry('bar', '2.0.0', 'sha512-bbb', 'https://nexus.local/repository/npm/bar/-/bar-2.0.0.tgz'), + ], config, {}) + expect(result.undecided.map(e => e.name)).toEqual(['bar']) + }) +}) + +describe('NexusRegistryApi', () => { + describe('forRegistry()', () => { + it('derives the REST base from a Nexus content URL', () => { + expect(NexusRegistryApi.forRegistry('https://nexus.local/repository/npm-group/', new Map(), {})) + .toBeDefined() + }) + + it('returns undefined for URLs without the Nexus repository layout', () => { + expect(NexusRegistryApi.forRegistry('https://registry.example.com/npm/', new Map(), {})) + .toBeUndefined() + }) + }) + + describe('hosted-inventory interrogation', () => { + // Composes the same three steps production performs (materializer's + // per-instance memoization is why no composite method exists on the + // class itself). + const listHosted = async (api: NexusRegistryApi): Promise> => { + const repositories = await api.listRepositories() + api.assertSourceRepoVisible(repositories) + return await api.hostedInventory(repositories) + } + + let server: http.Server + let serverUrl: string + let requests: Array<{ url: string, authorization?: string }> + let mode: 'ok' | 'forbidden' | 'garbage' | 'filtered' + + beforeEach(async () => { + requests = [] + mode = 'ok' + server = http.createServer((req, res) => { + requests.push({ url: req.url!, authorization: req.headers.authorization }) + if (mode === 'forbidden') { + res.statusCode = 403 + return res.end('forbidden') + } + if (mode === 'garbage') { + res.setHeader('content-type', 'text/html') + return res.end('captive portal') + } + const respond = (body: unknown) => { + res.setHeader('content-type', 'application/json') + res.end(JSON.stringify(body)) + } + if (req.url === '/service/rest/v1/repositories') { + if (mode === 'filtered') { + // A permission-filtered listing that omits the group the + // project installs from. + return respond([{ name: 'maven-releases', format: 'maven2', type: 'hosted' }]) + } + return respond([ + { name: 'npm-private', format: 'npm', type: 'hosted' }, + { name: 'npm-extra', format: 'npm', type: 'hosted' }, + { name: 'npm-proxy', format: 'npm', type: 'proxy' }, + { name: 'npm-group', format: 'npm', type: 'group' }, + { name: 'maven-releases', format: 'maven2', type: 'hosted' }, + ]) + } + if (req.url === '/service/rest/v1/components?repository=npm-private') { + // First page with a continuation token, mirroring the real API. + return respond({ + items: [{ + repository: 'npm-private', + format: 'npm', + group: 'acme', + name: 'private-utils', + version: '1.2.3', + assets: [{ + checksum: { sha1: 'aa'.repeat(20), sha512: 'bb'.repeat(64) }, + npm: { name: '@acme/private-utils', version: '1.2.3' }, + }], + }], + continuationToken: 'page-2', + }) + } + if (req.url === '/service/rest/v1/components?repository=npm-private&continuationToken=page-2') { + return respond({ + items: [{ + repository: 'npm-private', + format: 'npm', + group: null, + name: 'legacy-private-pkg', + version: '2.1.0', + // No npm metadata on the asset: the group/name fallback is + // exercised. + assets: [{ checksum: { sha1: 'cc'.repeat(20) } }], + }], + continuationToken: null, + }) + } + if (req.url === '/service/rest/v1/components?repository=npm-extra') { + return respond({ items: [], continuationToken: null }) + } + res.statusCode = 404 + res.end('not found') + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const { address, port } = server.address() as AddressInfo + serverUrl = `http://${address}:${port}/repository/npm-group/` + }) + + afterEach(async () => { + await new Promise((resolve, reject) => server.close(err => err ? reject(err) : resolve())) + }) + + it('enumerates all hosted npm repositories with pagination', async () => { + const api = NexusRegistryApi.forRegistry(serverUrl, new Map(), {})! + const inventory = await listHosted(api) + expect([...inventory.keys()].sort()).toEqual([ + '@acme/private-utils@1.2.3', + 'legacy-private-pkg@2.1.0', + ]) + expect(requests.map(r => r.url)).toEqual([ + '/service/rest/v1/repositories', + '/service/rest/v1/components?repository=npm-private', + '/service/rest/v1/components?repository=npm-private&continuationToken=page-2', + '/service/rest/v1/components?repository=npm-extra', + ]) + }) + + it('sends the npm credentials configured for the registry', async () => { + const config = parseNpmrc(`//127.0.0.1:${(server.address() as AddressInfo).port}/:_authToken=secret`) + const api = NexusRegistryApi.forRegistry(serverUrl, config, {})! + await listHosted(api) + expect(requests[0].authorization).toBe('Bearer secret') + }) + + it('treats a listing that omits the source repository as permission-filtered', async () => { + mode = 'filtered' + const api = NexusRegistryApi.forRegistry(serverUrl, new Map(), {})! + await expect(listHosted(api)).rejects.toThrow(/filtered by permissions/) + }) + + it('fails rather than truncating when pagination exceeds the page guard', async () => { + const workingListener = server.listeners('request')[0] as http.RequestListener + server.removeAllListeners('request') + let pages = 0 + server.on('request', (req, res) => { + if (req.url!.startsWith('/service/rest/v1/components?repository=npm-private')) { + pages++ + res.setHeader('content-type', 'application/json') + return res.end(JSON.stringify({ items: [], continuationToken: `page-${pages}` })) + } + workingListener(req as never, res as never) + }) + const api = NexusRegistryApi.forRegistry(serverUrl, new Map(), {})! + await expect(listHosted(api)).rejects.toThrow(/more hosted components than detection is prepared/) + // The fail-fast property: bounded pages, not an unbounded walk. + expect(pages).toBeLessThanOrEqual(51) + }) + + it('degrades when no hosted npm repositories are visible', async () => { + const workingListener2 = server.listeners('request')[0] as http.RequestListener + server.removeAllListeners('request') + server.on('request', (req, res) => { + if (req.url === '/service/rest/v1/repositories') { + res.setHeader('content-type', 'application/json') + // The source group is visible, but no hosted repos are. + return res.end(JSON.stringify([ + { name: 'npm-group', format: 'npm', type: 'group' }, + { name: 'npm-proxy', format: 'npm', type: 'proxy' }, + ])) + } + workingListener2(req as never, res as never) + }) + const api = NexusRegistryApi.forRegistry(serverUrl, new Map(), {})! + await expect(listHosted(api)).rejects.toThrow(/No npm hosted repositories are visible/) + }) + + it('reports an inaccessible API as DetectionUnavailableError', async () => { + mode = 'forbidden' + const api = NexusRegistryApi.forRegistry(serverUrl, new Map(), {})! + await expect(listHosted(api)).rejects.toThrow(DetectionUnavailableError) + }) + + it('reports an unexpected response shape as DetectionUnavailableError', async () => { + mode = 'garbage' + const api = NexusRegistryApi.forRegistry(serverUrl, new Map(), {})! + await expect(listHosted(api)).rejects.toThrow(DetectionUnavailableError) + }) + }) +}) + +describe('decideWithHostedInventory()', () => { + it('embeds hosted entries and marks the rest public', () => { + const hosted = entry('@acme/private-utils', '1.2.3', 'sha512-aaa') + const proxied = entry('is-odd', '3.0.1', 'sha512-bbb') + const verdicts = decideWithHostedInventory( + [hosted, proxied], + new Set(['@acme/private-utils@1.2.3']), + ) + expect(verdicts.get(hosted)).toBe('embed') + expect(verdicts.get(proxied)).toBe('public') + }) +}) + +describe('diffAgainstPublicRegistry()', () => { + let server: http.Server + let serverUrl: string + let requests: string[] + + const publicContent = 'public tarball bytes' + const publicIntegrity = sha512Of(publicContent) + const publicShasum = createHash('sha1').update(publicContent).digest('hex') + + beforeEach(async () => { + requests = [] + server = http.createServer((req, res) => { + requests.push(req.url!) + const respond = (body: unknown) => { + res.setHeader('content-type', 'application/json') + res.end(JSON.stringify(body)) + } + switch (req.url) { + case '/public-pkg': + return respond({ versions: { '1.0.0': { dist: { integrity: publicIntegrity } } } }) + case '/shasum-only-pkg': + return respond({ versions: { '1.0.0': { dist: { shasum: publicShasum } } } }) + case '/shadowed-pkg': + return respond({ versions: { '1.0.0': { dist: { integrity: sha512Of('a different artifact') } } } }) + case '/version-gap-pkg': + return respond({ versions: { '9.9.9': { dist: { integrity: publicIntegrity } } } }) + case '/garbage-pkg': + return respond({ hello: 'captive portal' }) + case '/malformed-dist-pkg': + return respond({ versions: { '1.0.0': { dist: { shasum: 123, integrity: 42 } } } }) + default: + res.statusCode = 404 + res.end('not found') + } + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const { address, port } = server.address() as AddressInfo + serverUrl = `http://${address}:${port}/` + }) + + afterEach(async () => { + await new Promise((resolve, reject) => server.close(err => err ? reject(err) : resolve())) + }) + + const diff = (entries: LockfileRegistryPackage[]) => + diffAgainstPublicRegistry(entries, { publicRegistryUrl: serverUrl }) + + it('marks an integrity match as public', async () => { + const e = entry('public-pkg', '1.0.0', publicIntegrity) + await expect(diff([e])).resolves.toEqual(new Map([[e, 'public']])) + }) + + it('matches legacy shasum-only public metadata', async () => { + const e = entry('shasum-only-pkg', '1.0.0', `sha1-${createHash('sha1').update(publicContent).digest('base64')}`) + await expect(diff([e])).resolves.toEqual(new Map([[e, 'public']])) + }) + + it('embeds on malformed dist field types instead of rejecting', async () => { + const e = entry('malformed-dist-pkg', '1.0.0', publicIntegrity) + await expect(diff([e])).resolves.toEqual(new Map([[e, 'embed']])) + }) + + it('embeds on integrity mismatch (shadowed name)', async () => { + const e = entry('shadowed-pkg', '1.0.0', publicIntegrity) + await expect(diff([e])).resolves.toEqual(new Map([[e, 'embed']])) + }) + + it('embeds when the version is absent publicly', async () => { + const e = entry('version-gap-pkg', '1.0.0', publicIntegrity) + await expect(diff([e])).resolves.toEqual(new Map([[e, 'embed']])) + }) + + it('embeds when the name does not exist publicly (404)', async () => { + const e = entry('no-such-pkg', '1.0.0', publicIntegrity) + await expect(diff([e])).resolves.toEqual(new Map([[e, 'embed']])) + }) + + it('embeds sha512 entries when public metadata only has an incomparable hash', async () => { + const e = entry('shasum-only-pkg', '1.0.0', publicIntegrity) + await expect(diff([e])).resolves.toEqual(new Map([[e, 'embed']])) + }) + + it('fetches one packument per unique name', async () => { + await diff([ + entry('public-pkg', '1.0.0', publicIntegrity), + entry('public-pkg', '2.0.0', publicIntegrity), + entry('no-such-pkg', '1.0.0', publicIntegrity), + ]) + expect(requests.sort()).toEqual(['/no-such-pkg', '/public-pkg']) + }) + + it('encodes scoped names', async () => { + await diff([entry('@acme/foo', '1.0.0', publicIntegrity)]) + expect(requests).toEqual(['/@acme%2Ffoo']) + }) + + it('rejects a 200 that is not a packument instead of guessing', async () => { + await expect(diff([entry('garbage-pkg', '1.0.0', publicIntegrity)])) + .rejects.toThrow(DetectionUnavailableError) + }) + + it('reports an unreachable registry as DetectionUnavailableError', async () => { + await expect(diffAgainstPublicRegistry( + [entry('foo', '1.0.0', publicIntegrity)], + { publicRegistryUrl: 'http://127.0.0.1:1/' }, + )).rejects.toThrow(DetectionUnavailableError) + }) +}) diff --git a/packages/cli/src/services/embedded-packages/__tests__/integrity.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/integrity.spec.ts index 22272b120..6f26a02a5 100644 --- a/packages/cli/src/services/embedded-packages/__tests__/integrity.spec.ts +++ b/packages/cli/src/services/embedded-packages/__tests__/integrity.spec.ts @@ -2,7 +2,14 @@ import { createHash } from 'node:crypto' import { describe, it, expect } from 'vitest' -import { integrityHashToHex, parseIntegrity, strongestIntegrityHash, verifyIntegrity } from '../integrity.js' +import { + integrityHashToHex, + integrityIntersects, + parseIntegrity, + shasumToIntegrity, + strongestIntegrityHash, + verifyIntegrity, +} from '../integrity.js' const content = Buffer.from('fake tarball content') const sha512 = `sha512-${createHash('sha512').update(content).digest('base64')}` @@ -63,3 +70,25 @@ describe('integrityHashToHex()', () => { expect(integrityHashToHex(hash)).toBe(createHash('sha512').update(content).digest('hex')) }) }) + +describe('shasumToIntegrity()', () => { + it('converts a hex sha1 shasum to its SRI form', () => { + expect(shasumToIntegrity(createHash('sha1').update(content).digest('hex'))).toBe(sha1) + }) +}) + +describe('integrityIntersects()', () => { + it('matches when a common algorithm agrees', () => { + expect(integrityIntersects(sha512, `${sha1} ${sha512}`)).toBe(true) + expect(integrityIntersects(sha1, `${sha1} ${sha512}`)).toBe(true) + }) + + it('rejects a disagreement on a common algorithm', () => { + const other = `sha512-${createHash('sha512').update('other').digest('base64')}` + expect(integrityIntersects(sha512, other)).toBe(false) + }) + + it('is false when no algorithm is shared (incomparable)', () => { + expect(integrityIntersects(sha512, sha1)).toBe(false) + }) +}) diff --git a/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts index c4b5bd174..90e86dc57 100644 --- a/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts +++ b/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts @@ -571,4 +571,823 @@ packages: expect(requests).toHaveLength(1) }) }) + + describe('materialize() with detection', () => { + const pubIntegrity = `sha512-${createHash('sha512').update('public artifact bytes').digest('base64')}` + + // The server plays three roles: the project's Nexus-shaped registry + // (content under /repository/, REST API under /service/rest/v1) and, + // for fallback tests, a fake public registry under /public/. + let restMode: 'ok' | 'forbidden' + let publicBarMode: 'ok' | 'error' + let registryUrl: string + + const usePublicAwareServer = () => { + server.removeAllListeners('request') + server.on('request', (req, res) => { + requests.push({ url: req.url!, authorization: req.headers.authorization }) + const respond = (body: unknown) => { + res.setHeader('content-type', 'application/json') + res.end(JSON.stringify(body)) + } + if (req.url!.startsWith('/service/rest/v1/')) { + if (restMode === 'forbidden') { + res.statusCode = 403 + return res.end('forbidden') + } + if (req.url === '/service/rest/v1/repositories') { + return respond([ + { name: 'npm-private', format: 'npm', type: 'hosted' }, + { name: 'npm-proxy', format: 'npm', type: 'proxy' }, + { name: 'npm-group', format: 'npm', type: 'group' }, + ]) + } + if (req.url!.startsWith('/service/rest/v1/components?repository=npm-private')) { + return respond({ + items: [ + { + repository: 'npm-private', + format: 'npm', + group: null, + name: 'bar', + version: '2.0.0', + assets: [{ checksum: {}, npm: { name: 'bar', version: '2.0.0' } }], + }, + { + repository: 'npm-private', + format: 'npm', + group: null, + name: 'bar', + version: '3.0.0', + assets: [{ checksum: {}, npm: { name: 'bar', version: '3.0.0' } }], + }, + { + repository: 'npm-private', + format: 'npm', + group: null, + name: 'odd-pkg', + version: '1.0.0', + assets: [{ checksum: {}, npm: { name: 'odd-pkg', version: '1.0.0' } }], + }, + ], + continuationToken: null, + }) + } + res.statusCode = 404 + return res.end('not found') + } + if (publicBarMode === 'error' && req.url === '/public/bar') { + res.statusCode = 500 + return res.end('boom') + } + if (req.url === '/public/pub-pkg') { + return respond({ versions: { '1.2.3': { dist: { integrity: pubIntegrity } } } }) + } + if (req.url!.startsWith('/public/')) { + res.statusCode = 404 + return res.end('not found') + } + if (req.url!.endsWith('.tgz')) { + return res.end(barTarball) + } + res.statusCode = 404 + res.end('not found') + }) + } + + const detectLockfile = () => ` +lockfileVersion: '9.0' +packages: + bar@2.0.0: + resolution: {integrity: ${barIntegrity}} + pub-pkg@1.2.3: + resolution: {integrity: ${pubIntegrity}} +` + + const makeDetecting = (specs: string[] = [], overrides: Record = {}) => + makeMaterializer(specs, { + detect: true, + publicRegistryUrl: `${serverUrl}public/`, + ...overrides, + }) + + // A function rather than a constant because registryUrl is assigned in + // beforeEach. + const barLockEntry = () => ({ + version: '2.0.0', + resolved: `${registryUrl}bar/-/bar-2.0.0.tgz`, + integrity: barIntegrity, + }) + + const writeNpmLockfile = async ( + packages: Record, + ) => { + const npmLockfilePath = path.join(workspaceRoot, 'package-lock.json') + await fs.writeFile(npmLockfilePath, JSON.stringify({ + lockfileVersion: 3, + packages: Object.fromEntries( + Object.entries(packages).map(([name, entry]) => [`node_modules/${name}`, entry]), + ), + })) + return npmLockfilePath + } + + const captureStderr = async (fn: () => Promise): Promise => { + const written: string[] = [] + const original = process.stderr.write.bind(process.stderr) + process.stderr.write = ((chunk: string) => { + written.push(String(chunk)) + return true + }) as never + try { + await fn() + } finally { + process.stderr.write = original + } + return written + } + + beforeEach(async () => { + restMode = 'ok' + publicBarMode = 'ok' + usePublicAwareServer() + registryUrl = `${serverUrl}repository/npm-group/` + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), `registry=${registryUrl}\n`) + await fs.writeFile(lockfilePath, detectLockfile()) + }) + + it('embeds only privately hosted packages, asking only the private registry', async () => { + const tarballs = await makeDetecting().materialize() + expect(tarballs.map(t => `${t.name}@${t.version}`)).toEqual(['bar@2.0.0']) + expect(tarballs[0].detected).toBe(true) + expect(requests.map(r => r.url).sort()).toEqual([ + '/repository/npm-group/bar/-/bar-2.0.0.tgz', + '/service/rest/v1/components?repository=npm-private', + '/service/rest/v1/repositories', + ]) + // The load-bearing privacy property: nothing was sent to the public + // registry. + expect(requests.some(r => r.url.startsWith('/public/'))).toBe(false) + }) + + it('reuses the summary cache on an unchanged lockfile', async () => { + await makeDetecting().materialize() + requests = [] + const tarballs = await makeDetecting().materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + expect(requests).toHaveLength(0) + }) + + it('re-interrogates the registry API on lockfile changes without re-downloading', async () => { + await makeDetecting().materialize() + requests = [] + await fs.writeFile(lockfilePath, `${detectLockfile()} baz@3.0.0: + resolution: {integrity: ${barIntegrity}} +`) + const tarballs = await makeDetecting().materialize() + expect(tarballs.map(t => t.name).sort()).toEqual(['bar']) + // The registry API is asked again (inventory verdicts depend on + // registry topology and are deliberately not cached per entry), but + // the already-cached tarball is not re-downloaded. + expect(requests.map(r => r.url).every(url => url.startsWith('/service/rest/v1/'))).toBe(true) + }) + + it('caches fallback verdicts per entry so only new entries are diffed', async () => { + restMode = 'forbidden' + await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + requests = [] + await fs.writeFile(lockfilePath, `${detectLockfile()} baz@3.0.0: + resolution: {integrity: ${barIntegrity}} +`) + await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/'))).toEqual(['/public/baz']) + }) + + it('lets an explicit entry take over its name, but warns about pin-blocked private versions', async () => { + // Both bar versions are in the lockfile and privately hosted. The + // explicit pin owns the name, so detection must not add bar@3.0.0 — + // but it warns, because detection proved private a version the + // bundle will not carry. + await fs.writeFile(lockfilePath, `${detectLockfile()} bar@3.0.0: + resolution: {integrity: ${barIntegrity}} +`) + let tarballs: Awaited> = [] + const written = await captureStderr(async () => { + tarballs = await makeDetecting(['bar@2.0.0']).materialize() + }) + expect(tarballs.map(t => `${t.name}@${t.version}`)).toEqual(['bar@2.0.0']) + expect(tarballs[0].detected).toBeUndefined() + const warning = written.find(line => line.includes('bar@3.0.0')) + expect(warning).toBeDefined() + expect(warning).toContain('pins their names to other versions') + expect(tarballs[0].detected).toBeUndefined() + }) + + it('embeds scope-mapped packages without any registry API traffic', async () => { + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), [ + 'registry=https://registry.npmjs.org/', + `@acme:registry=${registryUrl}`, + ].join('\n')) + await fs.writeFile(lockfilePath, ` +lockfileVersion: '9.0' +packages: + '@acme/foo@1.2.3': + resolution: {integrity: ${fooIntegrity}} + pub-pkg@1.2.3: + resolution: {integrity: ${pubIntegrity}} +`) + server.removeAllListeners('request') + server.on('request', (req, res) => { + requests.push({ url: req.url!, authorization: req.headers.authorization }) + res.end(fooTarball) + }) + + const tarballs = await makeDetecting().materialize() + expect(tarballs.map(t => t.name)).toEqual(['@acme/foo']) + expect(requests.map(r => r.url)).toEqual(['/repository/npm-group/@acme/foo/-/foo-1.2.3.tgz']) + }) + + it('skips undecided packages with a warning when the registry API is unavailable', async () => { + restMode = 'forbidden' + const tarballs = await makeDetecting().materialize() + expect(tarballs).toEqual([]) + // No fallback to the public registry without the explicit opt-in. + expect(requests.some(r => r.url.startsWith('/public/'))).toBe(false) + }) + + it('keeps scope-mapped embeds when the undecided tier degrades', async () => { + restMode = 'forbidden' + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), [ + `registry=${registryUrl}`, + `@acme:registry=${serverUrl}repository/npm-scope/`, + ].join('\n')) + await fs.writeFile(lockfilePath, `${detectLockfile()} '@acme/foo@1.2.3': + resolution: {integrity: ${fooIntegrity}} +`) + const workingServer = server.listeners('request')[0] as http.RequestListener + server.removeAllListeners('request') + server.on('request', (req, res) => { + if (req.url!.includes('foo')) { + requests.push({ url: req.url!, authorization: req.headers.authorization }) + return res.end(fooTarball) + } + workingServer(req as never, res as never) + }) + + const tarballs = await makeDetecting().materialize() + // The undecided entries (bar, pub-pkg) are skipped with a warning, + // but the scope-mapped package detection already proved private with + // zero network is still embedded. + expect(tarballs.map(t => t.name)).toEqual(['@acme/foo']) + }) + + it('degrades with a warning naming the unset variable a registry mapping references', async () => { + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), 'registry=${RED862_UNSET_REGISTRY}\n') + let tarballs: Awaited> = [] + const written = await captureStderr(async () => { + tarballs = await makeDetecting().materialize() + }) + expect(tarballs).toEqual([]) + const warning = written.find(line => line.includes('could not determine')) + expect(warning).toBeDefined() + expect(warning).toContain('RED862_UNSET_REGISTRY') + }) + + it('skips detection with a warning for a registry URL without the Nexus layout', async () => { + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), `registry=${serverUrl}\n`) + const tarballs = await makeDetecting().materialize() + expect(tarballs).toEqual([]) + expect(requests.some(r => r.url.startsWith('/public/'))).toBe(false) + }) + + it('does not cache degraded runs', async () => { + restMode = 'forbidden' + await makeDetecting().materialize() + restMode = 'ok' + requests = [] + const tarballs = await makeDetecting().materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + }) + + it('uses the public registry diff when the fallback is opted into', async () => { + restMode = 'forbidden' + const tarballs = await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + // bar is missing from /public/ (404 => embed), pub-pkg matches. + expect(tarballs.map(t => t.name)).toEqual(['bar']) + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/')).sort()) + .toEqual(['/public/bar', '/public/pub-pkg']) + }) + + it('skips an auto-detected tarball that fails to download instead of failing the run', async () => { + const workingServer = server.listeners('request')[0] as http.RequestListener + server.removeAllListeners('request') + server.on('request', (req, res) => { + if (req.url!.endsWith('.tgz')) { + requests.push({ url: req.url!, authorization: req.headers.authorization }) + res.statusCode = 404 + return res.end('gone') + } + workingServer(req as never, res as never) + }) + + const tarballs = await makeDetecting().materialize() + expect(tarballs).toEqual([]) + }) + + it('still fails hard when an explicit tarball cannot be downloaded', async () => { + server.removeAllListeners('request') + server.on('request', (req, res) => { + requests.push({ url: req.url!, authorization: req.headers.authorization }) + res.statusCode = 404 + res.end('gone') + }) + + await expect(makeMaterializer(['bar@2.0.0']).materialize()) + .rejects.toThrow(/Failed to download embedded package 'bar@2\.0\.0'/) + }) + + it('ignores cache entries that the lockfile does not vouch for', async () => { + // Prime the summary cache, then tamper with it: inject a key for a + // package that is not in the lockfile at all. + await makeDetecting().materialize() + const summaryDir = path.join( + cacheDir, 'embedded-packages', 'detection', + ) + const [summaryFile] = (await fs.readdir(summaryDir)).filter(name => name.startsWith('summary-')) + const summaryPath = path.join(summaryDir, summaryFile) + const summary = JSON.parse(await fs.readFile(summaryPath, 'utf8')) + summary.embedKeys.push('evil-package@6.6.6::sha512-evil') + await fs.writeFile(summaryPath, JSON.stringify(summary)) + + requests = [] + const tarballs = await makeDetecting().materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + }) + + it('skips detection for unsupported lockfiles instead of failing', async () => { + const yarnLockfilePath = path.join(workspaceRoot, 'yarn.lock') + await fs.writeFile(yarnLockfilePath, '') + const tarballs = await makeDetecting([], { lockfilePath: yarnLockfilePath }).materialize() + expect(tarballs).toEqual([]) + }) + + it('captures the degraded-run warning with its count and remediation options', async () => { + restMode = 'forbidden' + const written = await captureStderr(async () => { + await makeDetecting().materialize() + }) + const warning = written.find(line => line.includes('could not determine')) + expect(warning).toBeDefined() + expect(warning).toContain('2 package(s)') + expect(warning).toContain('checks.embeddedPackages') + expect(warning).toContain('--no-detect-embedded-packages') + }) + + it('detects across npm package-lock.json lockfiles', async () => { + const npmLockfilePath = await writeNpmLockfile({ + 'bar': barLockEntry(), + 'pub-pkg': { + version: '1.2.3', + resolved: 'https://registry.npmjs.org/pub-pkg/-/pub-pkg-1.2.3.tgz', + integrity: pubIntegrity, + }, + }) + const tarballs = await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + // bar's recorded source is the private registry and it is hosted + // there; pub-pkg's public resolved URL proves it public with zero + // lookups. + expect(tarballs.map(t => `${t.name}@${t.version}`)).toEqual(['bar@2.0.0']) + }) + + it('isolates registry groups: one broken registry does not stop another from deciding', async () => { + // bar's recorded source is the working Nexus-shaped registry; + // odd-pkg's recorded source is a Nexus-shaped URL on an unreachable + // instance, forming a second group that degrades on its own. + const npmLockfilePath = await writeNpmLockfile({ + 'bar': barLockEntry(), + 'odd-pkg': { + version: '1.0.0', + resolved: 'http://127.0.0.1:1/repository/npm-x/odd-pkg/-/odd-pkg-1.0.0.tgz', + integrity: pubIntegrity, + }, + }) + const tarballs = await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + }) + + it('embeds a same-origin odd-shaped source when the instance hosts it, and caches', async () => { + // odd-pkg's recorded source is not Nexus-shaped but shares the + // configured registry's origin: the conservative fallback may prove + // it private (hosted => embed). bar and odd-pkg are both hosted, so + // nothing is skipped and the summary caches. + const npmLockfilePath = await writeNpmLockfile({ + 'bar': barLockEntry(), + 'odd-pkg': { + version: '1.0.0', + resolved: `${serverUrl}npm/odd-pkg/-/odd-pkg-1.0.0.tgz`, + integrity: barIntegrity, + }, + }) + const tarballs1 = await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + expect(tarballs1.map(t => t.name).sort()).toEqual(['bar', 'odd-pkg']) + // Both groups (bar authoritative, odd-pkg conservative) target one + // instance with identical credentials, so the hosted inventory is + // fetched once. + expect(requests.map(r => r.url).filter(url => url.startsWith('/service/rest/v1/components'))) + .toHaveLength(1) + requests = [] + // Cached summary => zero requests on the second run proves the first + // run was not degraded. + const tarballs2 = await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + expect(tarballs2.map(t => t.name).sort()).toEqual(['bar', 'odd-pkg']) + expect(requests).toHaveLength(0) + }) + + it('never mints a public verdict from the conservative same-origin fallback', async () => { + // not-hosted-pkg shares the configured registry's origin but is not + // in its hosted inventory: its availability is unknown, so it is + // skipped with a warning (degraded, not cached) instead of being + // silently declared public. + const npmLockfilePath = await writeNpmLockfile({ + 'not-hosted-pkg': { + version: '1.0.0', + resolved: `${serverUrl}npm/not-hosted-pkg/-/not-hosted-pkg-1.0.0.tgz`, + integrity: pubIntegrity, + }, + }) + let tarballs: Awaited> = [] + const written = await captureStderr(async () => { + tarballs = await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + }) + expect(tarballs).toEqual([]) + // The headline privacy property of the no-opt-in branch: the + // undecided name is never sent to the public registry. + expect(requests.some(r => r.url.startsWith('/public/'))).toBe(false) + // The REST API answered fine, so the remedy list must not send the + // user chasing REST permissions — but still offer what helps. + const warning = written.find(line => line.includes('could not determine')) + expect(warning).toBeDefined() + expect(warning).not.toContain('REST API') + expect(warning).toContain('checks.embeddedPackages') + expect(warning).toContain('detectEmbeddedPackagesFallback') + requests = [] + // Degraded => not cached => the next run interrogates again. + await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + expect(requests.length).toBeGreaterThan(0) + }) + + it('memoizes instance data across groups while running the visibility guard per group', async () => { + // Two groups on the same instance: bar from npm-group (visible), + // hidden-pkg from npm-hidden (not in the repository listing). The + // second group degrades on its own visibility guard while the first + // decides — and the repository listing is fetched only once. + const npmLockfilePath = await writeNpmLockfile({ + 'bar': barLockEntry(), + 'hidden-pkg': { + version: '1.0.0', + resolved: `${serverUrl}repository/npm-hidden/hidden-pkg/-/hidden-pkg-1.0.0.tgz`, + integrity: pubIntegrity, + }, + }) + const tarballs = await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + expect(requests.map(r => r.url).filter(url => url === '/service/rest/v1/repositories')) + .toHaveLength(1) + }) + + it('does not share instance data between groups with different credentials', async () => { + // Two Nexus-shaped repos on one instance, each with its own token. + // The repository listing is permission-filtered per token, so the + // memoized listing and inventory must not bleed between the groups: + // sharing token A's listing with the npm-b group would hide npm-b's + // repository and silently drop pkg-b. + const port = (server.address() as AddressInfo).port + server.removeAllListeners('request') + server.on('request', (req, res) => { + requests.push({ url: req.url!, authorization: req.headers.authorization }) + const repo = req.headers.authorization === 'Bearer token-a' + ? 'npm-a' + : req.headers.authorization === 'Bearer token-b' ? 'npm-b' : undefined + const respond = (body: unknown) => { + res.setHeader('content-type', 'application/json') + res.end(JSON.stringify(body)) + } + if (req.url!.endsWith('.tgz')) { + return res.end(barTarball) + } + if (repo === undefined) { + res.statusCode = 403 + return res.end('forbidden') + } + if (req.url === '/service/rest/v1/repositories') { + return respond([{ name: repo, format: 'npm', type: 'hosted' }]) + } + if (req.url!.startsWith(`/service/rest/v1/components?repository=${repo}`)) { + const name = repo === 'npm-a' ? 'pkg-a' : 'pkg-b' + return respond({ + items: [{ + repository: repo, + format: 'npm', + group: null, + name, + version: '1.0.0', + assets: [{ checksum: {}, npm: { name, version: '1.0.0' } }], + }], + continuationToken: null, + }) + } + res.statusCode = 404 + res.end('not found') + }) + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), [ + `registry=${serverUrl}repository/npm-a/`, + `//127.0.0.1:${port}/repository/npm-a/:_authToken=token-a`, + `//127.0.0.1:${port}/repository/npm-b/:_authToken=token-b`, + ].join('\n')) + const npmLockfilePath = await writeNpmLockfile({ + 'pkg-a': { + version: '1.0.0', + resolved: `${serverUrl}repository/npm-a/pkg-a/-/pkg-a-1.0.0.tgz`, + integrity: barIntegrity, + }, + 'pkg-b': { + version: '1.0.0', + resolved: `${serverUrl}repository/npm-b/pkg-b/-/pkg-b-1.0.0.tgz`, + integrity: barIntegrity, + }, + }) + const tarballs = await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + expect(tarballs.map(t => t.name).sort()).toEqual(['pkg-a', 'pkg-b']) + // Each group interrogates with its own credentials. + expect(requests.map(r => r.url).filter(url => url === '/service/rest/v1/repositories')) + .toHaveLength(2) + expect(requests.map(r => r.url).filter(url => url.startsWith('/service/rest/v1/components'))) + .toHaveLength(2) + }) + + it('does not let a version-pinned spec silence degradation for other versions of the name', async () => { + // Both odd-pkg versions are undecidable (the REST API answers 403). + // The pinned spec covers only 1.0.0; 2.0.0 is neither materialized + // nor decided, so the run must stay degraded (uncached) and warn. + restMode = 'forbidden' + const npmLockfilePath = await writeNpmLockfile({ + 'odd-pkg': { + version: '1.0.0', + resolved: `${registryUrl}odd-pkg/-/odd-pkg-1.0.0.tgz`, + integrity: barIntegrity, + }, + 'x/node_modules/odd-pkg': { + version: '2.0.0', + resolved: `${registryUrl}odd-pkg/-/odd-pkg-2.0.0.tgz`, + integrity: barIntegrity, + }, + }) + await makeDetecting(['odd-pkg@1.0.0'], { lockfilePath: npmLockfilePath }).materialize() + requests = [] + // Degraded => not cached => the next run interrogates again. + await makeDetecting(['odd-pkg@1.0.0'], { lockfilePath: npmLockfilePath }).materialize() + expect(requests.some(r => r.url.startsWith('/service/rest/v1/'))).toBe(true) + }) + + it('trusts a public-registry proof for conservative same-origin entries when opted in', async () => { + // pub-pkg's recorded source shares the configured registry's origin + // but is not hosted on it. The hosted inventory's silence leaves it + // undecided, but the opted-in public-registry diff settles it with + // an integrity proof: no degradation, and the summary caches. + const npmLockfilePath = await writeNpmLockfile({ + 'bar': barLockEntry(), + 'pub-pkg': { + version: '1.2.3', + resolved: `${serverUrl}npm/pub-pkg/-/pub-pkg-1.2.3.tgz`, + integrity: pubIntegrity, + }, + }) + const detectOpts = { lockfilePath: npmLockfilePath, detectionFallback: 'public-registry' } + const tarballs1 = await makeDetecting([], detectOpts).materialize() + expect(tarballs1.map(t => t.name)).toEqual(['bar']) + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/'))).toEqual(['/public/pub-pkg']) + requests = [] + // Zero requests on the second run proves the first run was not + // degraded and its summary was cached. + const tarballs2 = await makeDetecting([], detectOpts).materialize() + expect(tarballs2.map(t => t.name)).toEqual(['bar']) + expect(requests).toHaveLength(0) + }) + + it('re-detects when the explicit list changes (explicit specs are part of the summary key)', async () => { + // Prime the cache with no explicit specs. + await makeDetecting().materialize() + requests = [] + await makeDetecting().materialize() + expect(requests).toHaveLength(0) + // A changed explicit list must not reuse the summary. + await makeDetecting(['bar@2.0.0']).materialize() + expect(requests.length).toBeGreaterThan(0) + }) + + it('degrades for foreign-origin undecidable sources unless they are listed explicitly', async () => { + // A second server on its own origin plays the foreign registry the + // artifact was recorded from (non-Nexus-shaped URL layout). + const foreignServer = http.createServer((req, res) => res.end(barTarball)) + await new Promise(resolve => foreignServer.listen(0, '127.0.0.1', resolve)) + const foreignPort = (foreignServer.address() as AddressInfo).port + try { + const npmLockfilePath = await writeNpmLockfile({ + 'bar': barLockEntry(), + 'odd-pkg': { + version: '1.0.0', + resolved: `http://127.0.0.1:${foreignPort}/npm/odd-pkg/-/odd-pkg-1.0.0.tgz`, + integrity: barIntegrity, + }, + }) + + // Undecidable foreign origin: the run degrades, so nothing is + // cached and the second run interrogates again. + await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + requests = [] + await makeDetecting([], { lockfilePath: npmLockfilePath }).materialize() + expect(requests.length).toBeGreaterThan(0) + + // Listing the undecidable package explicitly covers it: the run no + // longer counts as degraded and the summary caches. + await makeDetecting(['odd-pkg@1.0.0'], { lockfilePath: npmLockfilePath }).materialize() + requests = [] + const tarballs = await makeDetecting(['odd-pkg@1.0.0'], { lockfilePath: npmLockfilePath }) + .materialize() + expect(tarballs.map(t => t.name).sort()).toEqual(['bar', 'odd-pkg']) + expect(requests).toHaveLength(0) + } finally { + await new Promise((resolve, reject) => foreignServer.close(err => err ? reject(err) : resolve())) + } + }) + + it('reaches the opted-in fallback when credential expansion fails', async () => { + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), [ + `registry=${registryUrl}`, + `//127.0.0.1:${(server.address() as AddressInfo).port}/:_authToken=\${RED862_UNSET_TOKEN}`, + ].join('\n')) + let tarballs: Awaited> = [] + const written = await captureStderr(async () => { + tarballs = await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + }) + // The registry API tier cannot even resolve credentials, but the + // opt-in public diff is still reached and decides (bar 404s publicly + // => embed). The download of bar then fails soft on the same broken + // credential, so nothing materializes — the assertion is about the + // fallback being reachable. + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/')).sort()) + .toEqual(['/public/bar', '/public/pub-pkg']) + expect(tarballs).toEqual([]) + // A broken credential mapping is a configuration problem too, and + // must be reported as one, not silently papered over. + expect(written.find(line => line.includes('configuration problem'))).toContain('RED862_UNSET_TOKEN') + }) + + it('does not send explicitly listed names to the public registry fallback', async () => { + // bar is explicitly listed, so its verdict would be discarded at + // rehydration anyway — its name must not reach the public registry + // even with the fallback opted in. Only pub-pkg is diffed. + restMode = 'forbidden' + const tarballs = await makeDetecting(['bar'], { detectionFallback: 'public-registry' }).materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/'))).toEqual(['/public/pub-pkg']) + }) + + it('applies cached per-entry proofs even when the public fallback is not opted in', async () => { + // Run 1 (opted in) proves bar private and caches the verdicts. Run 2 + // has the fallback off and a changed lockfile (summary miss): the + // cached proofs are a pure disk read, so bar is still embedded and + // pub-pkg stays excluded while only the new unknown entry degrades — + // and nothing is sent to /public/. This is the documented contract: + // verdicts continue to apply after opting back out. + restMode = 'forbidden' + await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + requests = [] + await fs.writeFile(lockfilePath, `${detectLockfile()} baz@3.0.0: + resolution: {integrity: ${barIntegrity}} +`) + const tarballs = await makeDetecting().materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + expect(requests.some(r => r.url.startsWith('/public/'))).toBe(false) + }) + + it('embeds an explicitly listed scope-mapped package exactly once, without warnings', async () => { + // The explicit spec covers @acme/foo's only lockfile version, so + // detection is never even consulted about it — it materializes once + // via the explicit path, with no pin-blocked warning and no + // registry API traffic. + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), [ + `registry=${registryUrl}`, + `@acme:registry=${registryUrl}`, + ].join('\n')) + await fs.writeFile(lockfilePath, ` +lockfileVersion: '9.0' +packages: + '@acme/foo@1.2.3': + resolution: {integrity: ${barIntegrity}} +`) + let tarballs: Awaited> = [] + const written = await captureStderr(async () => { + tarballs = await makeDetecting(['@acme/foo']).materialize() + }) + expect(tarballs.map(t => `${t.name}@${t.version}`)).toEqual(['@acme/foo@1.2.3']) + expect(tarballs[0].detected).toBeUndefined() + expect(written.filter(line => line.startsWith('Warning:'))).toEqual([]) + expect(requests.some(r => r.url.startsWith('/service/'))).toBe(false) + }) + + it('persists partial public-diff verdicts when a lookup fails, and resumes where it left off', async () => { + restMode = 'forbidden' + publicBarMode = 'error' + // Run 1: pub-pkg's packument succeeds (an integrity proof) while + // bar's lookup fails; the partial proof must be persisted. + await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/')).sort()) + .toEqual(['/public/bar', '/public/pub-pkg']) + requests = [] + // Run 2: only bar — the still-unknown name — is transmitted again. + await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/'))).toEqual(['/public/bar']) + }) + + it('does not hang when a lookup fails with more names than the detection concurrency', async () => { + // Regression guard: aborting the diff by clearing the task queue + // would leave the cleared tasks' promises unsettled and hang the + // run forever once the number of unique names exceeds the queue + // concurrency (10). + restMode = 'forbidden' + publicBarMode = 'error' + const many = Object.fromEntries(Array.from({ length: 15 }, (_, i) => [`pkg-${i}`, { + version: '1.0.0', + resolved: `${registryUrl}pkg-${i}/-/pkg-${i}-1.0.0.tgz`, + integrity: barIntegrity, + }])) + const npmLockfilePath = await writeNpmLockfile({ ...many, bar: barLockEntry() }) + const tarballs = await makeDetecting([], { lockfilePath: npmLockfilePath, detectionFallback: 'public-registry' }) + .materialize() + // bar's lookup fails (degrading the run); every pkg-N task starts + // before bar's (bar is last in the lockfile), so all 15 get their + // 404 => embed verdict and materialize. + expect(tarballs.map(t => t.name).sort()).toEqual( + Array.from({ length: 15 }, (_, i) => `pkg-${i}`).sort()) + }) + + it('reaches the opted-in fallback when the registry mapping references an unset variable', async () => { + // A configuration error must not bypass the opted-in diff — it can + // decide the packages without the configured registry. (Downloads of + // the proven-private entries then fail soft on the same broken + // configuration, so nothing materializes.) + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), 'registry=${RED862_UNSET_REGISTRY}\n') + let tarballs: Awaited> = [] + const written = await captureStderr(async () => { + tarballs = await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + }) + expect(requests.map(r => r.url).filter(url => url.startsWith('/public/')).sort()) + .toEqual(['/public/bar', '/public/pub-pkg']) + expect(tarballs).toEqual([]) + // The fallback deciding the packages must not hide the underlying + // configuration problem. + const configWarning = written.find(line => line.includes('configuration problem')) + expect(configWarning).toBeDefined() + expect(configWarning).toContain('RED862_UNSET_REGISTRY') + // The problem persists, so the warning must recur on the next run — + // served entirely from the verdict cache, with zero network. + requests = [] + const written2 = await captureStderr(async () => { + await makeDetecting([], { detectionFallback: 'public-registry' }).materialize() + }) + expect(written2.find(line => line.includes('configuration problem'))).toContain('RED862_UNSET_REGISTRY') + expect(requests).toHaveLength(0) + }) + + it('returns nothing, silently, for an options shape without workspace root and lockfile', async () => { + const written = await captureStderr(async () => { + const materializer = makeMaterializer([], { + detect: true, + workspaceRoot: undefined, + lockfilePath: undefined, + }) + await expect(materializer.materialize()).resolves.toEqual([]) + }) + expect(written).toEqual([]) + }) + + it('announces auto-embedded packages on an informational line, not a warning', async () => { + const written = await captureStderr(async () => { + await makeDetecting().materialize() + }) + const announcement = written.find(line => line.includes('auto-detected private package')) + expect(announcement).toBeDefined() + expect(announcement).toContain('bar@2.0.0') + expect(announcement).toContain('--no-detect-embedded-packages') + expect(announcement!.startsWith('Warning:')).toBe(false) + }) + + it('does not run detection when disabled', async () => { + const tarballs = await makeMaterializer(['bar@2.0.0'], { publicRegistryUrl: `${serverUrl}public/` }) + .materialize() + expect(tarballs.map(t => t.name)).toEqual(['bar']) + expect(requests.map(r => r.url).some(url => url.startsWith('/public/') || url.startsWith('/service/'))).toBe(false) + }) + }) }) diff --git a/packages/cli/src/services/embedded-packages/__tests__/npmrc.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/npmrc.spec.ts index 8370a4ef1..b4ab058d6 100644 --- a/packages/cli/src/services/embedded-packages/__tests__/npmrc.spec.ts +++ b/packages/cli/src/services/embedded-packages/__tests__/npmrc.spec.ts @@ -100,7 +100,7 @@ describe('npmrcConfigFromEnv()', () => { describe('defaultNpmrcPaths()', () => { it('orders context dir before workspace root before home', () => { - expect(defaultNpmrcPaths('/ws', '/home/user', '/ws/packages/a')).toEqual([ + expect(defaultNpmrcPaths('/ws', '/home/user', '/ws/packages/a', {})).toEqual([ path.join('/ws/packages/a', '.npmrc'), path.join('/ws', '.npmrc'), path.join('/home/user', '.npmrc'), @@ -108,11 +108,43 @@ describe('defaultNpmrcPaths()', () => { }) it('deduplicates when the context dir is the workspace root', () => { - expect(defaultNpmrcPaths('/ws', '/home/user', '/ws')).toEqual([ + expect(defaultNpmrcPaths('/ws', '/home/user', '/ws', {})).toEqual([ path.join('/ws', '.npmrc'), path.join('/home/user', '.npmrc'), ]) }) + + it('lets npm_config_userconfig replace the user-level path, like npm', () => { + expect(defaultNpmrcPaths('/ws', '/home/user', undefined, { npm_config_userconfig: '/etc/ci-npmrc' })).toEqual([ + path.join('/ws', '.npmrc'), + '/etc/ci-npmrc', + ]) + expect(defaultNpmrcPaths('/ws', '/home/user', undefined, { NPM_CONFIG_USERCONFIG: '/etc/ci-npmrc' })).toEqual([ + path.join('/ws', '.npmrc'), + '/etc/ci-npmrc', + ]) + // npm ignores empty env config values. + expect(defaultNpmrcPaths('/ws', '/home/user', undefined, { npm_config_userconfig: '' })).toEqual([ + path.join('/ws', '.npmrc'), + path.join('/home/user', '.npmrc'), + ]) + }) + + it('expands a leading ~ in npm_config_userconfig against the home directory', () => { + expect(defaultNpmrcPaths('/ws', '/home/user', undefined, { npm_config_userconfig: '~/.npmrc-work' })).toEqual([ + path.join('/ws', '.npmrc'), + path.join('/home/user', '.npmrc-work'), + ]) + expect(defaultNpmrcPaths('/ws', '/home/user', undefined, { npm_config_userconfig: '~' })).toEqual([ + path.join('/ws', '.npmrc'), + '/home/user', + ]) + // Only a leading tilde segment is home-relative. + expect(defaultNpmrcPaths('/ws', '/home/user', undefined, { npm_config_userconfig: '/etc/~npmrc' })).toEqual([ + path.join('/ws', '.npmrc'), + '/etc/~npmrc', + ]) + }) }) describe('resolveRegistryUrl()', () => { diff --git a/packages/cli/src/services/embedded-packages/detection-cache.ts b/packages/cli/src/services/embedded-packages/detection-cache.ts new file mode 100644 index 000000000..18e789d2c --- /dev/null +++ b/packages/cli/src/services/embedded-packages/detection-cache.ts @@ -0,0 +1,267 @@ +import { createHash, randomUUID } from 'node:crypto' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import process from 'node:process' + +import Debug from 'debug' + +import { resolveCacheDirs } from './cache.js' +import { DetectionVerdict } from './detection.js' +import { LockfileRegistryPackage } from './lockfile-packages.js' +import { NpmrcConfig, expandedCredentialEntries, expandedRegistryEntries } from './npmrc.js' + +const debug = Debug('checkly:cli:services:embedded-packages') + +/** + * Bump to invalidate cached detection summaries when anything that could + * alter the embed set changes — the detection algorithm itself, but also + * lockfile enumeration and registry-configuration handling. + */ +export const DETECTOR_VERSION = 3 + +/** + * Versions the per-entry verdict file separately from the summaries: + * verdicts are immutable integrity proofs (see {@link verdictKey}), so a + * summary-semantics bump must not discard them — for opted-in users a + * discarded verdict means re-transmitting a package name to the public + * registry, not just a latency cost. Bump only when verdict semantics or + * the key format change. + */ +const VERDICTS_VERSION = 2 + +export interface DetectionSummary { + /** + * {@link verdictKey} values of the packages to embed. Deliberately keys + * only: the caller rehydrates full entries (including the tarball URL + * and integrity used for downloads) from the current lockfile, so a + * tampered or stale cache can never introduce an artifact the lockfile + * does not vouch for. + */ + embedKeys: string[] +} + +/** + * The key for a detection verdict. Verdicts are immutable under this key: + * a published artifact can never change, so an integrity match against the + * public registry can never un-match, and a stale `embed` verdict is + * harmless because over-embedding is allowed by the bundle contract. + */ +export function verdictKey (entry: LockfileRegistryPackage): string { + return `${entry.name}@${entry.version}::${entry.integrity}` +} + +/** + * The digest identifying a whole detection run: the lockfile bytes, the + * registry-affecting npm configuration (`registry` and `@scope:registry` + * entries, with `${VAR}` references expanded), the (expanded) credential + * entries, and the explicitly configured package names — any of these + * changing must invalidate the summary even when the lockfile is + * unchanged. + */ +export function detectionInputDigest ( + lockfileContent: string, + npmrcConfig: NpmrcConfig, + env: NodeJS.ProcessEnv = process.env, + explicitSpecs: string[] = [], +): string { + // Expanded values: a registry remap expressed through an environment + // variable reference must invalidate the summary too. + const registryEntries = expandedRegistryEntries(npmrcConfig, env) + // Credentials influence detection results: the registry API filters its + // repository listing by permission, so a verdict produced under one set + // of credentials must not outlive a credentials change — including a + // token rotated behind a `${NPM_TOKEN}` reference, hence the expansion. + // The values only feed the hash; they are never stored. + const credentialEntries = expandedCredentialEntries(npmrcConfig, env) + return createHash('sha256') + .update(lockfileContent) + .update('\0') + .update(JSON.stringify(registryEntries)) + .update('\0') + .update(JSON.stringify(credentialEntries)) + // Explicitly configured entries participate: entries detection could + // not decide may still count as covered (non-degraded, hence + // cacheable) when the user listed them, so changing the explicit list + // must trigger re-detection. + .update('\0') + .update(JSON.stringify([...explicitSpecs].sort())) + .digest('hex') +} + +/** + * Persistent detection state in the CLI cache (same multi-root layout as + * the tarball cache: project-local `node_modules/.cache/checkly` first, + * per-user directory as read tier and write fallback). Two levels: + * + * - a summary (the full embed set) keyed by {@link detectionInputDigest}, + * making repeat runs with an unchanged lockfile free, and + * - per-entry verdicts keyed by {@link verdictKey}, so a lockfile change + * only pays for entries not seen before. + * + * All operations are best-effort: a cache problem degrades to re-detection, + * never to a user-facing error. + */ +export class DetectionCache { + #rootDirs: string[] + + constructor (rootDirs: string | string[]) { + this.#rootDirs = Array.isArray(rootDirs) ? rootDirs : [rootDirs] + } + + static default ( + env: NodeJS.ProcessEnv = process.env, + projectRoot?: string, + platform: NodeJS.Platform = process.platform, + homedir = os.homedir(), + ): DetectionCache { + return new DetectionCache(resolveCacheDirs(env, projectRoot, platform, homedir) + .map(dir => path.join(dir, 'embedded-packages', 'detection'))) + } + + #summaryFilename (inputDigest: string): string { + return `summary-v${DETECTOR_VERSION}-${inputDigest}.json` + } + + #verdictsFilename (): string { + return `verdicts-v${VERDICTS_VERSION}.json` + } + + async #readJsonFrom (rootDir: string, filename: string): Promise { + try { + return JSON.parse(await fs.readFile(path.join(rootDir, filename), 'utf8')) as T + } catch { + return undefined + } + } + + async #readJson (filename: string): Promise { + for (const rootDir of this.#rootDirs) { + const value = await this.#readJsonFrom(rootDir, filename) + if (value !== undefined) { + return value + } + } + return undefined + } + + async #writeJson (filename: string, value: unknown): Promise { + for (const rootDir of this.#rootDirs) { + const filePath = path.join(rootDir, filename) + const tempPath = `${filePath}.${process.pid}.${randomUUID()}.tmp` + try { + await fs.mkdir(rootDir, { recursive: true }) + await fs.writeFile(tempPath, JSON.stringify(value)) + await fs.rename(tempPath, filePath) + return rootDir + } catch (err) { + debug('detection cache write to %s failed: %s', rootDir, (err as Error).message) + } finally { + await fs.rm(tempPath, { force: true }).catch(() => {}) + } + } + return undefined + } + + async getSummary (inputDigest: string): Promise { + const summary = await this.#readJson(this.#summaryFilename(inputDigest)) + // Validate the shape: a structurally wrong cache file must degrade to + // a miss, not throw downstream. + if (!Array.isArray(summary?.embedKeys) || summary.embedKeys.some(key => typeof key !== 'string')) { + return undefined + } + return summary + } + + async putSummary (inputDigest: string, summary: DetectionSummary): Promise { + const rootDir = await this.#writeJson(this.#summaryFilename(inputDigest), summary) + if (rootDir !== undefined) { + await this.#pruneSummaries(rootDir) + } + } + + /** + * Keeps only the most recent summary files: they are keyed by lockfile + * revision and would otherwise accumulate forever. + */ + async #pruneSummaries (rootDir: string, keep = 10): Promise { + try { + const names = (await fs.readdir(rootDir)).filter(name => /^summary-v\d+-[0-9a-f]+\.json$/.test(name)) + if (names.length <= keep) { + return + } + const stats = await Promise.all(names.map(async name => ({ + name, + mtimeMs: (await fs.stat(path.join(rootDir, name))).mtimeMs, + }))) + stats.sort((a, b) => b.mtimeMs - a.mtimeMs) + for (const { name } of stats.slice(keep)) { + await fs.rm(path.join(rootDir, name), { force: true }) + } + } catch (err) { + debug('detection cache prune failed: %s', (err as Error).message) + } + } + + async getVerdicts (): Promise> { + // Merge across every root: with a readable-but-unwritable primary + // root, writes land in the fallback root, and a first-hit read would + // permanently ignore them. + let merged: Record = {} + for (const rootDir of [...this.#rootDirs].reverse()) { + const verdicts = await this.#readJsonFrom(rootDir, this.#verdictsFilename()) + if (typeof verdicts !== 'object' || verdicts === null || Array.isArray(verdicts)) { + continue + } + merged = { + ...merged, + ...Object.fromEntries(Object.entries(verdicts) + .filter(([, value]) => value === 'public' || value === 'embed')), + } + } + return merged + } + + /** + * Merges the given verdicts into the stored map. Concurrent writers can + * race (last write wins); acceptable for an immutable-verdict cache + * whose entries are only ever re-derivable. + */ + async putVerdicts (verdicts: Record): Promise { + let merged = { ...await this.getVerdicts(), ...verdicts } + // Bound the map: entries are immutable and re-derivable, so when years + // of dependency churn blow past the cap it is cheaper to start over + // (keeping the fresh verdicts) than to rewrite an ever-growing file on + // every run. + const MAX_VERDICTS = 10_000 + if (Object.keys(merged).length > MAX_VERDICTS) { + merged = { ...verdicts } + } + const rootDir = await this.#writeJson(this.#verdictsFilename(), merged) + if (rootDir !== undefined) { + await this.#pruneStaleVerdicts(rootDir) + } + } + + /** + * Verdict files from superseded versions are never read or written again + * and would otherwise sit in the cache (often persisted by CI) forever. + * Only strictly OLDER versions are removed — a newer CLI's file must + * survive an older CLI running against the same (e.g. per-user) cache + * root — and only in the root this instance just wrote to, so CLIs of + * different versions sharing the other roots are left alone. + */ + async #pruneStaleVerdicts (rootDir: string): Promise { + try { + const names = (await fs.readdir(rootDir)).filter(name => { + const match = /^verdicts-v(\d+)\.json$/.exec(name) + return match !== null && Number(match[1]) < VERDICTS_VERSION + }) + for (const name of names) { + await fs.rm(path.join(rootDir, name), { force: true }) + } + } catch (err) { + debug('detection cache prune failed: %s', (err as Error).message) + } + } +} diff --git a/packages/cli/src/services/embedded-packages/detection.ts b/packages/cli/src/services/embedded-packages/detection.ts new file mode 100644 index 000000000..b532b124f --- /dev/null +++ b/packages/cli/src/services/embedded-packages/detection.ts @@ -0,0 +1,511 @@ +import axios from 'axios' +import Debug from 'debug' +import PQueue from 'p-queue' + +import { assignProxy } from '../proxy.js' +import { integrityIntersects, shasumToIntegrity } from './integrity.js' +import { LockfileRegistryPackage } from './lockfile-packages.js' +import { DEFAULT_REGISTRY_URL, NpmrcConfig, resolveAuthHeader, resolveRegistryUrl } from './npmrc.js' + +const debug = Debug('checkly:cli:services:embedded-packages') + +export const PUBLIC_REGISTRY_URL = DEFAULT_REGISTRY_URL + +// registry.yarnpkg.com is a long-standing alias serving the same artifacts. +const PUBLIC_REGISTRY_HOSTS = new Set(['registry.npmjs.org', 'registry.yarnpkg.com']) + +const API_TIMEOUT_MS = 30_000 +const MAX_RESPONSE_BYTES = 50 * 1024 * 1024 +const DETECTION_CONCURRENCY = 10 + +function isPublicRegistryUrl (url: string): boolean { + try { + return PUBLIC_REGISTRY_HOSTS.has(new URL(url).host) + } catch { + return false + } +} + +export interface ClassifiedEntries { + /** Provably resolves from the public registry — never embed. */ + public: LockfileRegistryPackage[] + /** + * Resolves from a non-public registry through an explicit scope mapping — + * embed without any lookup. Over-embedding is allowed by the bundle + * contract, and scoped registries overwhelmingly host private packages. + */ + embed: LockfileRegistryPackage[] + /** + * Cannot be decided from configuration alone (a non-public *default* + * registry may proxy public packages verbatim) — needs the private + * registry's API, or the opt-in public-registry fallback, to decide. + */ + undecided: LockfileRegistryPackage[] +} + +/** + * Classifies lockfile registry entries by what npm configuration alone can + * prove, without any network traffic. + */ +export function classifyEntries ( + entries: LockfileRegistryPackage[], + npmrcConfig: NpmrcConfig, + env: NodeJS.ProcessEnv = process.env, +): ClassifiedEntries { + const result: ClassifiedEntries = { public: [], embed: [], undecided: [] } + + for (const entry of entries) { + // A lockfile-recorded public tarball URL (package-lock.json `resolved`) + // names the artifact's actual source and proves publicness. + if (entry.tarballUrl !== undefined && isPublicRegistryUrl(entry.tarballUrl)) { + result.public.push(entry) + continue + } + + // A scope explicitly mapped to a non-public registry marks the package + // private regardless of any recorded non-public source — npm lockfiles + // record `resolved` for every entry, and this must not defeat the + // zero-network scope tier. + const scope = entry.name.startsWith('@') ? entry.name.slice(0, entry.name.indexOf('/')) : undefined + const scopeMapped = scope !== undefined + && (npmrcConfig.has(`${scope}:registry`) || npmrcConfig.has(`${scope.toLowerCase()}:registry`)) + let registryUrl: string + try { + registryUrl = resolveRegistryUrl(npmrcConfig, entry.name, env) + } catch (err) { + // E.g. an unset ${VAR} in this entry's registry mapping. A + // scope-mapped entry stays in the no-lookup embed tier — an + // explicit @scope:registry mapping that fails to expand is never + // the public registry (the default needs no variable), and + // 'undecided' could transmit the private name under the opt-in. + // Others become undecided instead of aborting classification. + debug('classify %s: cannot resolve registry: %s', entry.name, (err as Error).message) + if (scopeMapped) { + result.embed.push(entry) + } else { + result.undecided.push(entry) + } + continue + } + if (scopeMapped && !isPublicRegistryUrl(registryUrl)) { + result.embed.push(entry) + continue + } + + // A non-public recorded source cannot be vouched for by registry + // configuration: the artifact may be a proxied public package or a + // private one. + if (entry.tarballUrl !== undefined) { + result.undecided.push(entry) + continue + } + + if (isPublicRegistryUrl(registryUrl)) { + result.public.push(entry) + continue + } + + result.undecided.push(entry) + } + + return result +} + +export type DetectionVerdict = 'public' | 'embed' + +/** + * Thrown when a detection tier cannot produce verdicts. Always handled by + * the caller as "detection degraded" (a warning, never a failed run). + */ +export class DetectionUnavailableError extends Error { + /** + * Verdicts the public-registry diff had already collected when the + * failure occurred. Callers persist and apply these: every transmitted + * package name should yield a durable verdict, so a retry never has to + * send the same name again. + */ + partialVerdicts?: Map + + /** + * True when granting the configured npm credentials access to the + * registry's REST API could plausibly fix the failure. Drives whether + * the degraded-run warning suggests that remedy — advice that would + * only mislead for failures unrelated to REST permissions. + */ + restAccessRemediable?: boolean + + constructor (message: string, options?: ErrorOptions & { restAccessRemediable?: boolean }) { + super(message, options) + this.name = 'DetectionUnavailableError' + this.restAccessRemediable = options?.restAccessRemediable + } +} + +async function apiGet (url: string, headers: Record): Promise { + const response = await axios.get(url, assignProxy(url, { + headers, + timeout: API_TIMEOUT_MS, + maxContentLength: MAX_RESPONSE_BYTES, + })) + return response.data +} + +/** + * Parses a Sonatype Nexus content URL (`/repository//...`) + * into its instance base and repository name — the single home of the + * Nexus URL-shape assumption. Undefined for other layouts. + */ +export function parseNexusContentUrl (url: string): { instanceBase: string, repoName: string } | undefined { + const marker = '/repository/' + const index = url.indexOf(marker) + if (index === -1) { + return undefined + } + const repoName = url.slice(index + marker.length).split('/')[0] + if (repoName === '') { + return undefined + } + return { instanceBase: url.slice(0, index), repoName } +} + +/** + * The repository content base of a Nexus-shaped URL + * (`/repository//`), or undefined for other layouts. + */ +export function nexusContentBase (url: string): string | undefined { + const parsed = parseNexusContentUrl(url) + if (parsed === undefined) { + return undefined + } + return `${parsed.instanceBase}/repository/${parsed.repoName}/` +} + +/** + * Interrogates the private registry (Sonatype Nexus Repository) about + * which packages it hosts, using only endpoints of the registry the + * project already talks to — private package names are never sent + * anywhere else. Credentials are the ones `.npmrc` holds for the + * registry's content endpoints; instances commonly accept them for the + * REST API too, and any refusal degrades to the configured fallback. + */ +export class NexusRegistryApi { + #restBase: string + #sourceRepoName: string + #authHeader?: string + + constructor (restBase: string, sourceRepoName: string, authHeader?: string) { + this.#restBase = restBase + this.#sourceRepoName = sourceRepoName + this.#authHeader = authHeader + } + + /** + * Derives the instance's REST base from an npm registry URL: Nexus + * content URLs have the shape `/repository//`, so + * everything before `/repository/` is the instance base (which may + * include a context path). Returns undefined for URLs without that + * shape (not Nexus, or an unsupported layout). + */ + static forRegistry ( + registryUrl: string, + npmrcConfig: NpmrcConfig, + env: NodeJS.ProcessEnv, + ): NexusRegistryApi | undefined { + const parsed = parseNexusContentUrl(registryUrl) + if (parsed === undefined) { + return undefined + } + const restBase = `${parsed.instanceBase}/service/rest/v1` + return new NexusRegistryApi(restBase, parsed.repoName, resolveAuthHeader(npmrcConfig, registryUrl, env)) + } + + /** + * Key for per-run memoization of listings/inventories: the listing is + * permission-filtered, so results are only shareable between groups + * using the same instance AND the same credentials. + */ + get cacheKey (): string { + return `${this.#restBase}\0${this.#authHeader ?? ''}` + } + + async #get (path: string): Promise { + const headers: Record = { accept: 'application/json' } + if (this.#authHeader !== undefined) { + headers.authorization = this.#authHeader + } + return await apiGet(`${this.#restBase}${path}`, headers) + } + + /** + * The instance's (permission-filtered) repository listing. Split from + * the inventory so that callers sharing one instance across groups can + * memoize the expensive parts per instance while still running + * {@link assertSourceRepoVisible} for each group's own source repo. + */ + async listRepositories (): Promise { + let repositories: unknown + try { + repositories = await this.#get('/repositories') + } catch (err) { + throw new DetectionUnavailableError( + `The registry's REST API is not accessible with the configured npm credentials`, + { cause: err, restAccessRemediable: true }, + ) + } + + if (!Array.isArray(repositories)) { + throw new DetectionUnavailableError(`The registry's repository listing has an unexpected shape`) + } + + return repositories + } + + /** + * The listing is permission-filtered per repository. If it does not + * even include the repository this group installs from, we are clearly + * not seeing everything, and an absent hosted repo cannot be taken as + * proof that nothing is privately hosted. + */ + assertSourceRepoVisible (repositories: unknown[]): void { + if (!repositories.some((repo: any) => repo?.name === this.#sourceRepoName)) { + throw new DetectionUnavailableError( + `The registry's repository listing does not include '${this.#sourceRepoName}',` + + ` so it appears to be filtered by permissions`, + { restAccessRemediable: true }, + ) + } + } + + async hostedInventory (repositories: unknown[]): Promise> { + const hostedNpmRepos = repositories + .filter((repo: any): repo is { name: string } => + typeof repo?.name === 'string' && repo?.format === 'npm' && repo?.type === 'hosted') + .map(repo => repo.name) + + // Zero visible hosted npm repositories is indistinguishable from a + // permission-filtered listing, and treating it as "nothing is + // privately hosted" would silently under-embed — the one harmful + // direction. Degrade instead; a genuinely hosted-free instance's users + // see the warning once and pick a remedy. + if (hostedNpmRepos.length === 0) { + throw new DetectionUnavailableError( + `No npm hosted repositories are visible to the configured credentials —` + + ` either none exist or the repository listing is permission-filtered`, + { restAccessRemediable: true }, + ) + } + + debug('nexus: npm hosted repositories: %j', hostedNpmRepos) + + const inventory = new Set() + // Page guard per registry instance: hosted npm repos hold curated + // private packages, not mirrors of the world. An instance bigger than + // this fails fast (~50 requests) rather than being walked on every + // run, and a truncated inventory is never passed off as authoritative. + const maxPages = 50 + let pagesUsed = 0 + for (const repoName of hostedNpmRepos) { + let continuationToken: string | undefined + while (true) { + if (pagesUsed >= maxPages) { + throw new DetectionUnavailableError( + `The registry has more hosted components than detection is prepared to enumerate`, + ) + } + pagesUsed++ + const query = continuationToken !== undefined + ? `&continuationToken=${encodeURIComponent(continuationToken)}` + : '' + let response: any + try { + response = await this.#get(`/components?repository=${encodeURIComponent(repoName)}${query}`) + } catch (err) { + throw new DetectionUnavailableError( + `Listing components of repository '${repoName}' failed`, + { cause: err, restAccessRemediable: true }, + ) + } + const items = response?.items + if (!Array.isArray(items)) { + throw new DetectionUnavailableError( + `The component listing of repository '${repoName}' has an unexpected shape`, + ) + } + + for (const item of items) { + if (item?.format !== 'npm' || typeof item?.version !== 'string') { + continue + } + for (const asset of Array.isArray(item.assets) ? item.assets : []) { + // The npm metadata on the asset carries the full (scoped) + // package name; fall back to reassembling it from the + // component's group/name split. + const name: unknown = asset?.npm?.name + ?? (typeof item.group === 'string' && item.group !== '' + ? `@${item.group}/${item.name}` + : item.name) + if (typeof name !== 'string') { + continue + } + inventory.add(`${name}@${item.version}`) + } + } + + continuationToken = typeof response?.continuationToken === 'string' + ? response.continuationToken + : undefined + if (continuationToken === undefined) { + break + } + } + } + + debug('nexus: %d hosted npm package versions', inventory.size) + + return inventory + } +} + +/** + * Decides undecided entries against the private registry's hosted + * inventory: a `name@version` present in a hosted repository is private — + * embed it; one absent from every hosted repository necessarily arrived + * through a proxy of the public registry — public. + */ +export function decideWithHostedInventory ( + entries: LockfileRegistryPackage[], + inventory: Set, +): Map { + const verdicts = new Map() + for (const entry of entries) { + const hosted = inventory.has(`${entry.name}@${entry.version}`) + verdicts.set(entry, hosted ? 'embed' : 'public') + debug('detect %s@%s: %s (registry api)', entry.name, entry.version, verdicts.get(entry)) + } + return verdicts +} + +interface PackumentVersionDist { + integrity?: string + shasum?: string +} + +export interface DiffOptions { + /** Public registry base URL; tests point this at a local server. */ + publicRegistryUrl?: string +} + +/** + * The opt-in fallback: decides undecided entries by comparing their + * lockfile integrity against the public registry's metadata, one + * abbreviated packument per unique name. A package is public only when the + * exact version exists publicly with a provably identical artifact; + * anything else — the name or version missing, or the integrity + * incomparable or different (a shadowed name or private fork) — means + * embed. + * + * This necessarily transmits the queried package names — including + * private ones — to the public registry, which is why it never runs + * unless `checks.detectEmbeddedPackagesFallback` is set to + * `'public-registry'`. + */ +export async function diffAgainstPublicRegistry ( + entries: LockfileRegistryPackage[], + options: DiffOptions = {}, +): Promise> { + const registryUrl = options.publicRegistryUrl ?? PUBLIC_REGISTRY_URL + + const byName = new Map() + for (const entry of entries) { + const group = byName.get(entry.name) ?? [] + group.push(entry) + byName.set(entry.name, group) + } + + const verdicts = new Map() + const queue = new PQueue({ concurrency: DETECTION_CONCURRENCY }) + + let failure: unknown + await queue.addAll([...byName.entries()].map(([name, group]) => async () => { + // Once one lookup fails the fallback is abandoned: tasks that have not + // fetched yet return without sending their package name. (Clearing the + // queue instead would leave the cleared tasks' promises unsettled and + // hang addAll forever.) The verdicts collected so far still travel + // with the failure — discarding them would force the next run to + // re-transmit the same names for nothing. + if (failure !== undefined) { + return + } + try { + const versions = await fetchPackumentVersions(registryUrl, name) + for (const entry of group) { + const dist = versions?.[entry.version] + // Field types are unvalidated registry data; a malformed value must + // become a recorded failure, never an unhandled throw. + const publicIntegrity = [ + typeof dist?.integrity === 'string' ? dist.integrity : undefined, + typeof dist?.shasum === 'string' ? shasumToIntegrity(dist.shasum) : undefined, + ] + .filter((value): value is string => value !== undefined) + .join(' ') + const isPublic = publicIntegrity !== '' && integrityIntersects(entry.integrity, publicIntegrity) + verdicts.set(entry, isPublic ? 'public' : 'embed') + debug('detect %s@%s: %s (public registry diff)', entry.name, entry.version, verdicts.get(entry)) + } + } catch (err) { + failure ??= err + } + })) + + if (failure !== undefined) { + const err = failure instanceof DetectionUnavailableError + ? failure + : new DetectionUnavailableError( + `The public registry diff failed unexpectedly`, { cause: failure }) + err.partialVerdicts = verdicts + throw err + } + + return verdicts +} + +async function fetchPackumentVersions ( + registryUrl: string, + name: string, +): Promise | undefined> { + const url = `${registryUrl}${name.replace('/', '%2F')}` + let data: any + try { + const response = await axios.get(url, assignProxy(url, { + headers: { + // The abbreviated "install" packument: much smaller, still carries + // per-version dist integrity. + accept: 'application/vnd.npm.install-v1+json', + }, + timeout: API_TIMEOUT_MS, + maxContentLength: MAX_RESPONSE_BYTES, + validateStatus: status => status === 200 || status === 404, + })) + if (response.status === 404) { + return undefined + } + data = response.data + } catch (err) { + throw new DetectionUnavailableError( + `The public npm registry is not reachable for the detection fallback`, + { cause: err }, + ) + } + + // A 200 that is not a packument (an interfering proxy, a captive portal) + // must not silently count as "nothing exists publicly": that would mark + // every package as private and poison the verdict cache. + if (typeof data?.versions !== 'object' || data.versions === null) { + throw new DetectionUnavailableError( + `The public registry returned an unexpected response for a package metadata request`, + ) + } + + const versions: Record = data.versions + return Object.fromEntries(Object.entries(versions).map(([version, meta]) => [version, meta?.dist])) +} diff --git a/packages/cli/src/services/embedded-packages/integrity.ts b/packages/cli/src/services/embedded-packages/integrity.ts index cc13d371a..a03997a9b 100644 --- a/packages/cli/src/services/embedded-packages/integrity.ts +++ b/packages/cli/src/services/embedded-packages/integrity.ts @@ -71,3 +71,23 @@ export function verifyIntegrity (content: Buffer, integrity: string): boolean { export function integrityHashToHex (hash: IntegrityHash): string { return Buffer.from(hash.digestBase64, 'base64').toString('hex') } + +/** + * The SRI form of a legacy hex sha1 shasum (registry packuments expose old + * artifacts with `dist.shasum` only, no `dist.integrity`). + */ +export function shasumToIntegrity (shasumHex: string): string { + return `sha1-${Buffer.from(shasumHex, 'hex').toString('base64')}` +} + +/** + * Whether two SRI strings agree on at least one common algorithm: same + * algorithm and same digest for it. Returns false when they share no + * supported algorithm — the caller must treat that as "incomparable", not + * as a match. + */ +export function integrityIntersects (a: string, b: string): boolean { + const hashesB = parseIntegrity(b) + return parseIntegrity(a).some(hashA => + hashesB.some(hashB => hashB.algorithm === hashA.algorithm && hashB.digestBase64 === hashA.digestBase64)) +} diff --git a/packages/cli/src/services/embedded-packages/lockfile-packages.ts b/packages/cli/src/services/embedded-packages/lockfile-packages.ts index 6208076dd..2a0eac6de 100644 --- a/packages/cli/src/services/embedded-packages/lockfile-packages.ts +++ b/packages/cli/src/services/embedded-packages/lockfile-packages.ts @@ -57,9 +57,9 @@ export class UnsupportedLockfileError extends Error { * registry packages and excluded (git/file/link/integrity-less) entries. * Supports `pnpm-lock.yaml` (v6/v9) and `package-lock.json` (v2/v3). */ -export async function loadLockfilePackages (lockfilePath: string): Promise { +export async function loadLockfilePackages (lockfilePath: string, content?: string): Promise { const basename = path.basename(lockfilePath) - const content = await fs.readFile(lockfilePath, 'utf8') + content ??= await fs.readFile(lockfilePath, 'utf8') switch (basename) { case 'pnpm-lock.yaml': diff --git a/packages/cli/src/services/embedded-packages/materializer.ts b/packages/cli/src/services/embedded-packages/materializer.ts index 930954e81..ccc6dad73 100644 --- a/packages/cli/src/services/embedded-packages/materializer.ts +++ b/packages/cli/src/services/embedded-packages/materializer.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs/promises' import os from 'node:os' import path from 'node:path' import process from 'node:process' @@ -8,6 +9,16 @@ import PQueue from 'p-queue' import { assignProxy } from '../proxy.js' import { TarballCache, lookupNpmCacache } from './cache.js' +import { DetectionCache, detectionInputDigest, verdictKey } from './detection-cache.js' +import { + DetectionUnavailableError, + DetectionVerdict, + NexusRegistryApi, + classifyEntries, + decideWithHostedInventory, + diffAgainstPublicRegistry, + nexusContentBase, +} from './detection.js' import { verifyIntegrity } from './integrity.js' import { LockfileRegistryPackage, @@ -39,6 +50,12 @@ export interface EmbeddedPackagesIssue { export interface PlannedTarball extends LockfileRegistryPackage { /** Archive filename, e.g. `@acme+foo@1.2.3.tgz` (scope slash → `+`). */ archiveFilename: string + /** + * Present when auto-detection selected this tarball. Detected tarballs + * fail soft: a download problem skips the tarball with a warning instead + * of aborting the run, unlike explicitly configured ones. + */ + detected?: true } export interface EmbeddedPackagesPlan { @@ -75,6 +92,19 @@ export class EmbeddedPackageError extends Error { export interface EmbeddedPackagesMaterializerOptions { /** Raw `checks.embeddedPackages` entries. */ specs: string[] + /** + * Whether to auto-detect packages to embed from the lockfile in addition + * to the explicit specs. Detected entries never override an explicitly + * configured name (per-name precedence). + */ + detect?: boolean + /** + * What to do when detection cannot decide packages without querying the + * public npm registry (which would transmit private package names). + * `'skip'` (default) leaves them un-embedded with a warning; + * `'public-registry'` opts into the integrity diff against public npm. + */ + detectionFallback?: 'skip' | 'public-registry' /** Absolute path of the workspace root lockfile, when one exists. */ lockfilePath?: string /** Workspace root directory, used to locate the root `.npmrc`. */ @@ -86,12 +116,19 @@ export interface EmbeddedPackagesMaterializerOptions { contextDir?: string env?: NodeJS.ProcessEnv homedir?: string + /** Public registry base URL for detection; tests point this at a local server. */ + publicRegistryUrl?: string } const DOWNLOAD_CONCURRENCY = 5 const DOWNLOAD_TIMEOUT_MS = 120_000 const MAX_TARBALL_BYTES = 1024 * 1024 * 1024 +// The skip reason for conservative same-origin entries the instance does +// not host. +const CONSERVATIVE_UNDETERMINED_REASON = `the packages' recorded source shares the configured registry's host but` + + ` is not hosted on it, so their availability cannot be determined` + /** * Removes userinfo credentials from a URL so it can be safely included in * error messages and logs (a registry URL may embed a token). @@ -109,6 +146,29 @@ function redactUrl (url: string): string { } } +/** Per-run shared registry API state, keyed by REST base. */ +interface InstanceState { + repositories: Map> + inventories: Map>> +} + +function getOrCreate (map: Map, key: string, create: () => V): V { + let value = map.get(key) + if (value === undefined) { + value = create() + map.set(key, value) + } + return value +} + +function sameOrigin (a: string, b: string): boolean { + try { + return new URL(a).origin === new URL(b).origin + } catch { + return false + } +} + /** * Resolves the configured `checks.embeddedPackages` specs against the * workspace lockfile (plan) and sources the selected tarballs into the CLI @@ -122,17 +182,20 @@ function redactUrl (url: string): string { export class EmbeddedPackagesMaterializer { #options: EmbeddedPackagesMaterializerOptions #cache: TarballCache + #detectionCache: DetectionCache #env: NodeJS.ProcessEnv #homedir: string #plan?: Promise #materialized?: Promise + #lockfile?: Promise<{ content: string, packages: Awaited> }> constructor (options: EmbeddedPackagesMaterializerOptions) { this.#options = options this.#env = options.env ?? process.env this.#homedir = options.homedir ?? os.homedir() this.#cache = TarballCache.default(this.#env, this.#projectRoot, process.platform, this.#homedir) + this.#detectionCache = DetectionCache.default(this.#env, this.#projectRoot, process.platform, this.#homedir) } get #projectRoot (): string | undefined { @@ -145,8 +208,12 @@ export class EmbeddedPackagesMaterializer { return this.#plan } - #info (message: string): void { - process.stderr.write(`${message}\n`) + #loadLockfile (lockfilePath: string) { + this.#lockfile ??= (async () => { + const content = await fs.readFile(lockfilePath, 'utf8') + return { content, packages: await loadLockfilePackages(lockfilePath, content) } + })() + return this.#lockfile } materialize (): Promise { @@ -155,6 +222,14 @@ export class EmbeddedPackagesMaterializer { } async #createPlan (): Promise { + // Without explicit specs there is nothing to validate: auto-detection + // (when enabled) runs at materialize time and cannot produce spec + // issues, and a project without a lockfile must not fail validation + // just because detection is on by default. + if (this.#options.specs.length === 0) { + return { tarballs: [], issues: [], warnings: [], wildcardMatches: [] } + } + const issues: EmbeddedPackagesIssue[] = [] const warnings: string[] = [] const wildcardMatches: Array<{ spec: string, packages: string[] }> = [] @@ -180,7 +255,7 @@ export class EmbeddedPackagesMaterializer { let packages try { - packages = await loadLockfilePackages(lockfilePath) + packages = (await this.#loadLockfile(lockfilePath)).packages } catch (err) { // Any failure to read or parse the lockfile (missing file, merge // conflict markers, unknown format) becomes a diagnostic naming the @@ -304,7 +379,7 @@ export class EmbeddedPackagesMaterializer { } async #materializeAll (): Promise { - const { tarballs, issues, wildcardMatches } = await this.plan() + const { tarballs: explicitTarballs, issues, wildcardMatches } = await this.plan() // Commands validate before bundling and exit on fatal diagnostics, so // this is a defensive backstop for direct/programmatic use. @@ -324,21 +399,65 @@ export class EmbeddedPackagesMaterializer { ) } - if (tarballs.length === 0) { + const detect = this.#options.detect === true && this.#projectRoot !== undefined + if (explicitTarballs.length === 0 && !detect) { + return [] + } + + // npm configuration problems (e.g. an unreadable .npmrc) must stay + // fatal when the user explicitly configured packages — downloads need + // the registry and credentials — but must not break projects that only + // have default-on detection. + let npmrcConfig: NpmrcConfig + try { + npmrcConfig = await loadNpmrcConfig(defaultNpmrcPaths( + // The project root is always derivable here: explicit tarballs + // imply a lockfile (missing one is a plan issue) and detection is + // gated on it above. + this.#projectRoot!, + this.#homedir, + this.#options.contextDir, + this.#env, + ), this.#env) + } catch (err) { + if (explicitTarballs.length > 0) { + throw err + } + this.#warn(`Embedded package detection skipped: ${(err as Error).message}`) return [] } - // Safe to assert: a missing lockfile is a plan issue, and issues abort - // above. - const npmrcConfig = await loadNpmrcConfig(defaultNpmrcPaths( - this.#projectRoot!, - this.#homedir, - this.#options.contextDir, - ), this.#env) + const tarballs = [...explicitTarballs] + if (detect) { + try { + tarballs.push(...await this.#detectTarballs(npmrcConfig, explicitTarballs)) + } catch (err) { + this.#warn(`Embedded package detection failed and was skipped: ${(err as Error).message}`) + } + } + + if (tarballs.length === 0) { + return [] + } const queue = new PQueue({ concurrency: DOWNLOAD_CONCURRENCY }) - const results = await queue.addAll(tarballs.map(tarball => async (): Promise => { - const filePath = await this.#obtainTarball(tarball, npmrcConfig) + const results = await queue.addAll(tarballs.map(tarball => async (): Promise => { + let filePath: string + try { + filePath = await this.#obtainTarball(tarball, npmrcConfig) + } catch (err) { + // Auto-detected tarballs fail soft: the run proceeds without them, + // exactly as it would have before detection existed. Explicitly + // configured tarballs keep their guarantee. + if (tarball.detected === true) { + this.#warn( + `Could not embed auto-detected package ${tarball.name}@${tarball.version}:` + + ` ${(err as Error).message}`, + ) + return undefined + } + throw err + } return { ...tarball, filePath, @@ -346,7 +465,486 @@ export class EmbeddedPackagesMaterializer { } })) - return results + return results.filter((result): result is MaterializedTarball => result !== undefined) + } + + #warn (message: string): void { + process.stderr.write(`Warning: ${message}\n`) + } + + #info (message: string): void { + process.stderr.write(`${message}\n`) + } + + /** + * Auto-detects lockfile packages the runner cannot fetch from the public + * registry and returns them as planned tarballs, excluding names the + * user configured explicitly (an explicit entry takes over its name). + * + * Everything here fails soft: detection is on by default, so an + * unsupported lockfile, an unavailable registry API, or any unexpected + * error must degrade to "detect nothing extra" with at most a warning, + * unlike explicit specs which error. The caller catches whatever this + * method throws and downgrades it to a warning. + */ + async #detectTarballs (npmrcConfig: NpmrcConfig, explicitTarballs: PlannedTarball[]): Promise { + const { lockfilePath } = this.#options + if (lockfilePath === undefined) { + return [] + } + + let lockfileContent: string + let registry: LockfileRegistryPackage[] + try { + const { content, packages } = await this.#loadLockfile(lockfilePath) + lockfileContent = content + registry = packages.registry + } catch (err) { + debug('detection skipped, cannot enumerate lockfile %s: %s', lockfilePath, (err as Error).message) + return [] + } + + // The plan already resolved every explicit spec against the lockfile + // (unresolvable specs threw before detection could run), so the + // planned tarballs are the authoritative record of explicit coverage: + // an unpinned spec plans every lockfile version of its name, a pinned + // spec only its own. An explicit spec takes over its package NAME for + // embedding, but only the exact versions it materializes count as + // covered for warning/degradation purposes. + const explicitNames = new Set(explicitTarballs.map(tarball => tarball.name)) + const explicitKeys = new Set(explicitTarballs.map(tarball => `${tarball.name}@${tarball.version}`)) + + // Entries whose exact name@version the explicit list already + // materializes are withheld from detection: their verdicts would be + // discarded at rehydration anyway, and this keeps their identities out + // of even the opted-in public registry diff. Key-level on purpose — + // OTHER lockfile versions of a listed name still flow through + // detection (and, when opted in, the public diff, transmitting the + // name) so the pin-blocked warning below can name them. + const detectableRegistry = registry + .filter(entry => !explicitKeys.has(`${entry.name}@${entry.version}`)) + + const inputDigest = detectionInputDigest(lockfileContent, npmrcConfig, this.#env, this.#options.specs) + + let embedKeys: Set + const summary = await this.#detectionCache.getSummary(inputDigest) + if (summary !== undefined) { + debug('detection summary cache hit (%d packages to embed)', summary.embedKeys.length) + embedKeys = new Set(summary.embedKeys) + } else { + const detected = await this.#runDetection(detectableRegistry, npmrcConfig) + embedKeys = detected.embedKeys + if (!detected.degraded) { + await this.#detectionCache.putSummary(inputDigest, { embedKeys: [...embedKeys] }) + } + // Degraded runs still embed what the sound tiers proved (e.g. + // scope-mapped packages), but are deliberately not cached so the + // next run retries the undecided remainder. + } + + // Rehydrate full entries from the lockfile: the cache contributes only + // identities, never artifact locations or hashes. + const detected: PlannedTarball[] = [] + const pinBlocked: string[] = [] + for (const entry of registry) { + if (!embedKeys.has(verdictKey(entry))) { + continue + } + if (explicitNames.has(entry.name)) { + // An explicit entry takes over its package name: detection never + // adds other versions of a listed name. When a pinned spec does + // not materialize this exact version, though, detection has proved + // private a version the bundle will not carry — that must be said + // out loud, not dropped silently. (The exact-covered guard also + // shields against a tampered summary smuggling covered keys in.) + if (!explicitKeys.has(`${entry.name}@${entry.version}`)) { + pinBlocked.push(`${entry.name}@${entry.version}`) + } + continue + } + detected.push({ + ...entry, + archiveFilename: `${entry.name.replace(/\//g, '+')}@${entry.version}.tgz`, + detected: true, + }) + } + if (pinBlocked.length > 0) { + this.#warn( + `Embedded package detection determined the following are private, but they are not embedded` + + ` because 'checks.embeddedPackages' pins their names to other versions: ${pinBlocked.join(', ')}.` + + ` Add them to 'checks.embeddedPackages' to embed them.`, + ) + } + + if (detected.length > 0) { + const names = detected.map(tarball => `${tarball.name}@${tarball.version}`) + const shown = names.slice(0, 8).join(', ') + const more = names.length > 8 ? ` and ${names.length - 8} more` : '' + // Informational, not a warning: this is the feature working as + // designed. + this.#info( + `Embedding ${names.length} auto-detected private package(s): ${shown}${more}.` + + ` Disable with --no-detect-embedded-packages or 'checks.detectEmbeddedPackages: false'.`, + ) + } + + return detected + } + + /** + * Runs detection tiers over the lockfile's registry entries. Returns the + * {@link verdictKey}s to embed plus whether the run degraded (some + * undecided entries could not be classified — the sound tiers' + * results are still returned, but must not be cached as a summary). + * + * Private package names never leave the machine by default: undecided + * entries are resolved by asking the project's own registry which + * packages it hosts. Only the explicit `'public-registry'` fallback ever + * queries public npm with package names, and only its verdicts are + * cached per entry — they compare immutable artifacts, whereas + * registry-inventory verdicts depend on the registry's topology and are + * covered by the summary cache (whose key includes the registry + * configuration) instead. + */ + async #runDetection ( + registry: LockfileRegistryPackage[], + npmrcConfig: NpmrcConfig, + ): Promise<{ embedKeys: Set, degraded: boolean }> { + const classified = classifyEntries(registry, npmrcConfig, this.#env) + debug( + 'detection: %d public by configuration, %d embed by scope mapping, %d undecided', + classified.public.length, classified.embed.length, classified.undecided.length, + ) + + const embedKeys = new Set(classified.embed.map(verdictKey)) + let undecided = classified.undecided + + // Configuration problems are diagnosed from configuration alone, up + // front: a later tier may still decide the entries (or the verdict + // cache may absorb them entirely), but a broken .npmrc must keep + // warning — and keep the run uncached — until it is fixed. Covers both + // registry mappings and credentials, for every entry detection will + // act on (embed-tier entries get downloaded; undecided ones decided). + const configErrors = new Set() + for (const entry of [...classified.embed, ...undecided]) { + const recorded = entry.tarballUrl !== undefined ? nexusContentBase(entry.tarballUrl) : undefined + try { + const entryRegistryUrl = recorded ?? resolveRegistryUrl(npmrcConfig, entry.name, this.#env) + resolveAuthHeader(npmrcConfig, entryRegistryUrl, this.#env) + } catch (err) { + configErrors.add(`the npm configuration could not be resolved (${(err as Error).message})`) + } + } + for (const reason of configErrors) { + this.#warn( + `Embedded package detection hit a configuration problem: ${reason}.` + + ` Detection continues with what it can prove, but the result is not cached and may differ` + + ` from what a correct configuration would produce.`, + ) + } + + if (undecided.length > 0) { + // Per-entry verdicts cached from a previous run are immutable + // integrity proofs (see #diffAndCacheVerdicts). Applying them is a + // pure disk read — no network traffic and no privacy cost — so they + // are deliberately not gated on the public-registry opt-in that + // originally produced them; the cache directory carries the same + // local trust the summary cache already gets. + const known = await this.#detectionCache.getVerdicts() + undecided = undecided.filter(entry => { + const verdict = known[verdictKey(entry)] + if (verdict === 'embed') { + embedKeys.add(verdictKey(entry)) + } + return verdict === undefined + }) + } + + if (undecided.length === 0) { + return { embedKeys, degraded: configErrors.size > 0 } + } + + // Group undecided entries by the registry instance to interrogate: the + // recorded source URL when the lockfile has a usable (Nexus-shaped) + // one — it names the instance the artifact really came from, which + // current configuration may no longer point at. A non-Nexus-shaped + // recorded source falls back to the configured registry only when it + // shares that registry's origin, and only CONSERVATIVELY: the fallback + // instance may prove such an entry private (hosted => embed; safe by + // the over-embed rule) but its silence proves nothing — a same-origin + // host can path-route several registry products — so "not hosted" + // leaves the entry undecided instead of minting a public verdict. + // Entries from unrelated hosts degrade outright ('' group). + interface DetectionGroup { + registryUrl: string + entries: LockfileRegistryPackage[] + conservative: boolean + /** + * Why the group's registry cannot be interrogated, when known at + * grouping time. The group still runs through the tiers so the + * opted-in fallback can decide it; without the opt-in this becomes + * the skip reason. + */ + unavailableReason?: string + } + const groups = new Map() + for (const entry of undecided) { + let registryUrl = entry.tarballUrl !== undefined + ? nexusContentBase(entry.tarballUrl) + : undefined + let conservative = false + let unavailableReason: string | undefined + if (registryUrl === undefined) { + let configured: string | undefined + try { + configured = resolveRegistryUrl(npmrcConfig, entry.name, this.#env) + } catch (err) { + unavailableReason = `the configured registry could not be resolved (${(err as Error).message})` + } + if (entry.tarballUrl === undefined) { + registryUrl = configured ?? '' + } else if (configured !== undefined && sameOrigin(entry.tarballUrl, configured)) { + registryUrl = configured + conservative = true + } else { + registryUrl = '' + } + } + if (registryUrl === '' && unavailableReason === undefined) { + unavailableReason = `The packages' recorded source cannot be interrogated and does not match` + + ` the configured registry` + } + const key = `${conservative ? 'conservative' : 'authoritative'}\0${registryUrl}\0${unavailableReason ?? ''}` + const group = groups.get(key) ?? { registryUrl, entries: [], conservative, unavailableReason } + group.entries.push(entry) + groups.set(key, group) + } + + // Interrogating the same instance twice (two groups sharing one REST + // base) would double the request budget for nothing; share the + // repository listing and inventory per instance. The per-group + // source-repo visibility guard still runs for every group. + const instanceState: InstanceState = { + repositories: new Map(), + inventories: new Map(), + } + const skipped: Array<{ entry: LockfileRegistryPackage, reason: string }> = [] + let restRemediable = false + for (const group of groups.values()) { + try { + const { verdicts, tier } = await this.#decideUndecided( + group.registryUrl, group.entries, npmrcConfig, instanceState, group.unavailableReason) + // The conservative rule only distrusts the hosted inventory's + // silence — a 'public' verdict from the public-registry integrity + // diff is a proof and holds for conservative groups too. + const unresolved: LockfileRegistryPackage[] = [] + for (const [entry, verdict] of verdicts) { + if (verdict === 'embed') { + embedKeys.add(verdictKey(entry)) + } else if (group.conservative && tier === 'registry-inventory') { + unresolved.push(entry) + } + } + if (unresolved.length > 0) { + skipped.push(...await this.#settleConservativeLeftovers(unresolved, embedKeys)) + } + } catch (err) { + const partial = this.#applyPartialVerdicts(err, embedKeys) + const reason = err instanceof DetectionUnavailableError + ? err.message + : `Unexpected error: ${(err as Error).message}` + restRemediable ||= err instanceof DetectionUnavailableError && err.restAccessRemediable === true + skipped.push(...group.entries + .filter(entry => partial?.has(entry) !== true) + .map(entry => ({ entry, reason }))) + } + } + + // Every skipped entry warns and keeps the run uncached: entries the + // explicit list covers were filtered out before the tiers ran. + const degraded = skipped.length > 0 || configErrors.size > 0 + if (skipped.length > 0) { + const reasons = [...new Set(skipped.map(({ reason }) => reason))] + const remedies = [ + ...(configErrors.size > 0 + ? [`fix the configuration problem(s) named in the preceding warning`] + : []), + // Only offered when some failure was actually about REST access — + // for e.g. conservative same-origin skips the REST API answered + // fine, and permission advice would just mislead. + ...(restRemediable + ? [`grant the configured npm credentials access to the registry's REST API` + + ` (detection needs to browse every npm hosted repository on the instance)`] + : []), + `list the packages in 'checks.embeddedPackages'`, + ...(this.#options.detectionFallback !== 'public-registry' + ? [`set 'checks.detectEmbeddedPackagesFallback: "public-registry"' to allow public npm` + + ` registry lookups`] + : []), + `disable detection with --no-detect-embedded-packages or 'checks.detectEmbeddedPackages: false'`, + ] + this.#warn( + `Embedded package detection could not determine whether ${skipped.length} package(s)` + + ` from your registry are private, and skipped embedding them.` + + ` Reason(s): ${reasons.join('; ')}.` + + ` To fix this, ${remedies.slice(0, -1).join(', ')}, or ${remedies[remedies.length - 1]}.`, + ) + } + + return { embedKeys, degraded } + } + + /** + * Conservative-group entries the hosted inventory stayed silent about + * are still undecided. The opted-in public-registry diff can settle them + * (its verdicts are integrity proofs); without the opt-in — or when the + * diff itself fails — they are skipped. + */ + async #settleConservativeLeftovers ( + unresolved: LockfileRegistryPackage[], + embedKeys: Set, + ): Promise> { + if (this.#options.detectionFallback !== 'public-registry') { + return unresolved.map(entry => ({ entry, reason: CONSERVATIVE_UNDETERMINED_REASON })) + } + try { + const diffed = await this.#diffAndCacheVerdicts(unresolved) + for (const [entry, verdict] of diffed) { + if (verdict === 'embed') { + embedKeys.add(verdictKey(entry)) + } + } + return [] + } catch (err) { + const partial = this.#applyPartialVerdicts(err, embedKeys) + // Both branches keep the same-origin context so the warning states + // which tier stayed silent and which one then failed. + const reason = err instanceof DetectionUnavailableError + ? `${CONSERVATIVE_UNDETERMINED_REASON}, and the public registry fallback failed (${err.message})` + : `${CONSERVATIVE_UNDETERMINED_REASON}, and the public registry fallback failed unexpectedly` + + ` (${(err as Error).message})` + return unresolved + .filter(entry => partial?.has(entry) !== true) + .map(entry => ({ entry, reason })) + } + } + + /** + * A failed public diff still carries the verdicts it collected before + * failing. Applies the 'embed' ones and returns the partial map so the + * caller can skip only what genuinely stayed undecided. + */ + #applyPartialVerdicts ( + err: unknown, + embedKeys: Set, + ): Map | undefined { + const partial = err instanceof DetectionUnavailableError ? err.partialVerdicts : undefined + for (const [entry, verdict] of partial ?? []) { + if (verdict === 'embed') { + embedKeys.add(verdictKey(entry)) + } + } + return partial + } + + /** + * Decides one registry's worth of undecided entries: primarily by + * interrogating that registry's REST API (no package names leave the + * machine), with the public-registry integrity diff as an explicit + * opt-in fallback. The returned tier states which of the two produced + * the verdicts — a hosted inventory's 'public' means only "not hosted + * here", whereas the diff's 'public' is an integrity proof. + */ + async #decideUndecided ( + registryUrl: string, + entries: LockfileRegistryPackage[], + npmrcConfig: NpmrcConfig, + instanceState: InstanceState, + unavailableReason?: string, + ): Promise<{ + verdicts: Map + tier: 'registry-inventory' | 'public-diff' + }> { + try { + if (unavailableReason !== undefined) { + throw new DetectionUnavailableError(unavailableReason) + } + const nexus = NexusRegistryApi.forRegistry(registryUrl, npmrcConfig, this.#env) + if (nexus === undefined) { + throw new DetectionUnavailableError( + `The registry URL does not look like a Sonatype Nexus Repository instance,` + + ` which is the only registry API supported for private package detection`, + ) + } + debug('detection: consulting the registry API for %d entries', entries.length) + // Memoized per instance AND credentials (the listing is permission + // filtered); the visibility guard still runs per group, since two + // groups on one instance may install from different repositories. + const repositories = getOrCreate(instanceState.repositories, nexus.cacheKey, + () => nexus.listRepositories()) + nexus.assertSourceRepoVisible(await repositories) + const inventory = getOrCreate(instanceState.inventories, nexus.cacheKey, + () => repositories.then(list => nexus.hostedInventory(list))) + return { verdicts: decideWithHostedInventory(entries, await inventory), tier: 'registry-inventory' } + } catch (err) { + if (this.#options.detectionFallback !== 'public-registry') { + throw err + } + debug('detection: registry API unavailable (%s), using the public registry fallback', (err as Error).message) + try { + return { verdicts: await this.#diffAndCacheVerdicts(entries), tier: 'public-diff' } + } catch (fallbackErr) { + // The fallback failing must not erase the registry tier's failure: + // the warning needs both causes, and the REST remedy stays + // applicable when the registry tier was permission-refused. + const combined = new DetectionUnavailableError( + `${(err as Error).message}; the public registry fallback then also failed` + + ` (${(fallbackErr as Error).message})`, + { + cause: fallbackErr, + restAccessRemediable: + (err instanceof DetectionUnavailableError && err.restAccessRemediable === true) + || (fallbackErr instanceof DetectionUnavailableError && fallbackErr.restAccessRemediable === true), + }, + ) + if (fallbackErr instanceof DetectionUnavailableError) { + combined.partialVerdicts = fallbackErr.partialVerdicts + } + throw combined + } + } + } + + /** + * The opt-in public-registry diff. Every verdict it obtains is persisted + * — including the partial results of a failed run — because these + * verdicts compare immutable artifacts and are cacheable forever, and a + * name transmitted once should never need transmitting again. Callers + * pass cache misses only: #runDetection applies the persistent verdict + * cache before any tier runs. + */ + async #diffAndCacheVerdicts ( + entries: LockfileRegistryPackage[], + ): Promise> { + const persist = async (diffed: Map): Promise => { + if (diffed.size === 0) { + return + } + await this.#detectionCache.putVerdicts( + Object.fromEntries([...diffed].map(([entry, verdict]) => [verdictKey(entry), verdict]))) + } + try { + const diffed = await diffAgainstPublicRegistry(entries, { + publicRegistryUrl: this.#options.publicRegistryUrl, + }) + await persist(diffed) + return diffed + } catch (err) { + if (err instanceof DetectionUnavailableError && err.partialVerdicts !== undefined) { + await persist(err.partialVerdicts) + } + throw err + } } async #obtainTarball (tarball: PlannedTarball, npmrcConfig: NpmrcConfig): Promise { diff --git a/packages/cli/src/services/embedded-packages/npmrc.ts b/packages/cli/src/services/embedded-packages/npmrc.ts index b74574d02..235271285 100644 --- a/packages/cli/src/services/embedded-packages/npmrc.ts +++ b/packages/cli/src/services/embedded-packages/npmrc.ts @@ -1,6 +1,7 @@ import fs from 'node:fs/promises' import os from 'node:os' import path from 'node:path' +import process from 'node:process' export const DEFAULT_REGISTRY_URL = 'https://registry.npmjs.org/' @@ -122,21 +123,43 @@ export async function loadNpmrcConfig ( * The `.npmrc` locations relevant to a project, in npm's precedence order: * the directory the Checkly project lives in (the nearest project config, * which may be a workspace member), the workspace root, then the - * user-level file. (npm's global and builtin configs are not consulted.) + * user-level file — `~/.npmrc`, or the file `npm_config_userconfig` names, + * matching npm's own userconfig override. (npm's global and builtin + * configs are not consulted.) */ export function defaultNpmrcPaths ( workspaceRoot: string, homedir = os.homedir(), contextDir?: string, + env: NodeJS.ProcessEnv = process.env, ): string[] { + const userconfig = env.npm_config_userconfig ?? env.NPM_CONFIG_USERCONFIG const paths = [ ...(contextDir !== undefined ? [path.join(contextDir, '.npmrc')] : []), path.join(workspaceRoot, '.npmrc'), - path.join(homedir, '.npmrc'), + userconfig !== undefined && userconfig !== '' + ? expandTilde(userconfig, homedir) + : path.join(homedir, '.npmrc'), ] return [...new Set(paths)] } +/** + * npm treats path-type config values starting with `~` as home-relative + * (a quoted `NPM_CONFIG_USERCONFIG="~/.npmrc-work"` reaches us with the + * tilde literal). Left unexpanded, the path would silently ENOENT and drop + * the user-level config entirely. + */ +function expandTilde (value: string, homedir: string): string { + if (value === '~') { + return homedir + } + if (value.startsWith('~/') || value.startsWith('~\\')) { + return path.join(homedir, value.slice(2)) + } + return value +} + function expandValue (key: string, value: string, env: NodeJS.ProcessEnv): string { return value.replace(/\$\{([^}]+)\}/g, (_, varName: string) => { const envValue = env[varName] @@ -155,6 +178,57 @@ function getExpanded (config: NpmrcConfig, key: string, env: NodeJS.ProcessEnv): return expandValue(key, value, env) } +/** + * The registry-affecting configuration entries (`registry` and + * `@scope:registry`), with `${VAR}` references expanded against the given + * environment (kept verbatim when the variable is unset, so the result is + * deterministic). Sorted by key. Used to key detection caches: the + * *effective* registry mapping must invalidate them, including when only a + * referenced environment variable changes. + */ +export function expandedRegistryEntries ( + config: NpmrcConfig, + env: NodeJS.ProcessEnv = process.env, +): Array<[string, string]> { + return expandedEntries(config, env, key => key === 'registry' || key.endsWith(':registry')) +} + +function expandedEntries ( + config: NpmrcConfig, + env: NodeJS.ProcessEnv, + keep: (key: string) => boolean, +): Array<[string, string]> { + const entries: Array<[string, string]> = [] + for (const [key, value] of config) { + if (!keep(key)) { + continue + } + let expanded: string + try { + expanded = expandValue(key, value, env) + } catch { + expanded = value + } + entries.push([key, expanded]) + } + return entries.sort(([a], [b]) => a.localeCompare(b)) +} + +/** + * The credential configuration entries (nerf-darted `//host/...:key` + * lines), with `${VAR}` references expanded against the given environment + * (kept verbatim when the variable is unset). Sorted by key. Used to key + * detection caches: rotating a token — including through the standard + * `${NPM_TOKEN}` indirection — must invalidate them, since the registry + * API filters results by permission. Values only ever feed a hash. + */ +export function expandedCredentialEntries ( + config: NpmrcConfig, + env: NodeJS.ProcessEnv = process.env, +): Array<[string, string]> { + return expandedEntries(config, env, key => key.startsWith('//')) +} + /** * Resolves the registry URL for a package name: the `@scope:registry` entry * if the package is scoped and one exists, the `registry` entry otherwise, diff --git a/packages/cli/src/services/project-parser.ts b/packages/cli/src/services/project-parser.ts index 19620948e..e169379bd 100644 --- a/packages/cli/src/services/project-parser.ts +++ b/packages/cli/src/services/project-parser.ts @@ -46,6 +46,8 @@ type ProjectParseOpts = { playwrightConfigPath?: string include?: string | string[] embeddedPackages?: string[] + detectEmbeddedPackages?: boolean + detectEmbeddedPackagesFallback?: 'skip' | 'public-registry' playwrightChecks?: PlaywrightSlimmedProp[] loadPlaywrightChecksOnly?: boolean warnOnWebServerConfig?: boolean @@ -146,6 +148,8 @@ export async function parseProject (opts: ProjectParseOpts): Promise { playwrightConfigPath, include, embeddedPackages, + detectEmbeddedPackages, + detectEmbeddedPackagesFallback, playwrightChecks, loadPlaywrightChecksOnly, warnOnWebServerConfig, @@ -186,6 +190,8 @@ export async function parseProject (opts: ProjectParseOpts): Promise { Session.verifyRuntimeDependencies = verifyRuntimeDependencies ?? true Session.ignoreDirectoriesMatch = ignoreDirectoriesMatch Session.embeddedPackages = embeddedPackages + Session.detectEmbeddedPackages = detectEmbeddedPackages + Session.detectEmbeddedPackagesFallback = detectEmbeddedPackagesFallback // The materializer snapshots specs and workspace paths at first use, so a // repeated in-process parse with different options must not reuse it. Session.embeddedPackagesMaterializer = undefined