diff --git a/node/README.md b/node/README.md index 4287e630..cf904b46 100644 --- a/node/README.md +++ b/node/README.md @@ -96,6 +96,7 @@ options: Specify NW.js version (will use latest otherwise) --nwjs-node-headers Download the NW.js node headers --nwjs-ffmpeg Download prebuilt ffmpeg for current NW.js version + --pnpm-store-version Specify the store version for pnpm v9 lockfile. Default is v10. --no-xdg-layout Don't use the XDG layout for caches --node-sdk-extension NODE_SDK_EXTENSION Flatpak node SDK extension (e.g. org.freedesktop.Sdk.Extension.node24//25.08) diff --git a/node/flatpak_node_generator/main.py b/node/flatpak_node_generator/main.py index 07ee6e0a..c9d09626 100644 --- a/node/flatpak_node_generator/main.py +++ b/node/flatpak_node_generator/main.py @@ -14,6 +14,7 @@ from .providers import ProviderFactory from .providers.npm import NpmLockfileProvider, NpmModuleProvider, NpmProviderFactory from .providers.pnpm import ( + STORE_VERSION_ARGUMENT_DEFAULT, PnpmLockfileProvider, PnpmProviderFactory, ) @@ -113,6 +114,10 @@ async def _async_main() -> None: help='Use the ChromeDriver version associated with the given ' 'Electron version for node-chromedriver', ) + parser.add_argument( + '--pnpm-store-version', + help=f'Specify the store version for pnpm v9 lockfile. Default is {STORE_VERSION_ARGUMENT_DEFAULT}', + ) # Deprecated alternative to --node-chromedriver-from-electron parser.add_argument('--electron-chromedriver', help=argparse.SUPPRESS) parser.add_argument( @@ -209,6 +214,7 @@ async def _async_main() -> None: PnpmLockfileProvider.Options( no_devel=args.no_devel, registry=args.registry, + store_version=args.pnpm_store_version, ), ) provider_factory = PnpmProviderFactory(lockfile_root, pnpm_options) diff --git a/node/flatpak_node_generator/package.py b/node/flatpak_node_generator/package.py index 9e81f7b9..4fd8048b 100644 --- a/node/flatpak_node_generator/package.py +++ b/node/flatpak_node_generator/package.py @@ -143,6 +143,7 @@ class Lockfile: path: Path version: int cache_key: str | None = None + store_version: str | None = None class Package(NamedTuple): diff --git a/node/flatpak_node_generator/populate_pnpm_store.py b/node/flatpak_node_generator/populate_pnpm_store.py index 46233e96..1603be38 100644 --- a/node/flatpak_node_generator/populate_pnpm_store.py +++ b/node/flatpak_node_generator/populate_pnpm_store.py @@ -6,14 +6,151 @@ import json import os import re +import sqlite3 +import struct import sys import tarfile import time +from collections.abc import Mapping _SANITIZE_RE = re.compile(r'[\\/:*?"<>|]') _MAX_LENGTH_WITHOUT_HASH = 120 +def _msgpack_pack(obj: object) -> bytes: + """Minimal msgpack packer matching msgpackr with useRecords: true. + + msgpackr reserves the byte range 0x40-0x7F for record IDs when useRecords + is enabled (pnpm's config). Standard msgpack uses 0x00-0x7F for positive + fixints, but this packer caps them at 0x3F and emits a uint 8 prefix (0xcc) + for 64-255 to avoid aliasing with record ID bytes. + """ + + def _pack_int(val: int) -> bytes: + # Positive fixint capped at 0x3F — 0x40-0x7F are msgpackr record IDs + if 0 <= val <= 0x3F: + return val.to_bytes(1, 'big') + if -32 <= val < 0: + return val.to_bytes(1, 'big', signed=True) + if val >= 0: + if val <= 0xFF: + return b'\xcc' + val.to_bytes(1, 'big') + if val <= 0xFFFF: + return b'\xcd' + val.to_bytes(2, 'big') + if val <= 0xFFFFFFFF: + return b'\xce' + val.to_bytes(4, 'big') + return b'\xcf' + val.to_bytes(8, 'big') + if val >= -0x80: + return b'\xd0' + val.to_bytes(1, 'big', signed=True) + if val >= -0x8000: + return b'\xd1' + val.to_bytes(2, 'big', signed=True) + if val >= -0x80000000: + return b'\xd2' + val.to_bytes(4, 'big', signed=True) + return b'\xd3' + val.to_bytes(8, 'big', signed=True) + + def _pack_str(val: str) -> bytes: + data = val.encode('utf-8') + length = len(data) + if length <= 0x1F: + return (0xA0 | length).to_bytes(1, 'big') + data + if length <= 0xFF: + return b'\xd9' + length.to_bytes(1, 'big') + data + if length <= 0xFFFF: + return b'\xda' + length.to_bytes(2, 'big') + data + return b'\xdb' + length.to_bytes(4, 'big') + data + + if obj is None: + return b'\xc0' + if isinstance(obj, bool): + return b'\xc3' if obj else b'\xc2' + if isinstance(obj, int): + return _pack_int(obj) + if isinstance(obj, float): + return b'\xcb' + struct.pack('>d', obj) + if isinstance(obj, str): + return _pack_str(obj) + if isinstance(obj, dict): + length = len(obj) + if length <= 0x0F: + result = (0x80 | length).to_bytes(1, 'big') + elif length <= 0xFFFF: + result = b'\xde' + length.to_bytes(2, 'big') + else: + result = b'\xdf' + length.to_bytes(4, 'big') + for key, val in obj.items(): + result += _msgpack_pack(key) + _msgpack_pack(val) + return result + if isinstance(obj, (list, tuple)): + length = len(obj) + if length <= 0x0F: + result = (0x90 | length).to_bytes(1, 'big') + elif length <= 0xFFFF: + result = b'\xdc' + length.to_bytes(2, 'big') + else: + result = b'\xdd' + length.to_bytes(4, 'big') + for item in obj: + result += _msgpack_pack(item) + return result + raise TypeError(f'Unsupported type for msgpack: {type(obj)}') + + +RECORD_HEADER = b'\xd4\x72' + + +def _pack_v11_store_entry( + algo: str, + files: dict[str, dict[str, object]], + manifest: dict[str, str] | None = None, +) -> bytes: + """Encode a store v11 entry using msgpackr-compatible record extensions. + + msgpackr with ``useRecords: true, moreTypes: true`` decodes: + + - standard msgpack maps (0x80/0xde/0xdf) as JavaScript ``Map`` objects + (iterable, supports ``for..of``) + - record extensions (0x72) as plain objects (dot-notation access) + + The ``files`` field must be a Map so pnpm can iterate it with ``for..of``. + Everything else must use record extensions so dot-notation works + (e.g. ``pkgIndex.algo``, ``info.digest``, ``manifest.name``). + """ + + def _record(obj: Mapping[str, object], struct_id: int) -> bytes: + keys = list(obj.keys()) + result = RECORD_HEADER + struct_id.to_bytes(1, 'big') + result += _msgpack_pack(keys) + for k in keys: + result += _msgpack_pack(obj[k]) + return result + + def _fixmap(length: int) -> bytes: + if length <= 0x0F: + return (0x80 | length).to_bytes(1, 'big') + if length <= 0xFFFF: + return b'\xde' + length.to_bytes(2, 'big') + return b'\xdf' + length.to_bytes(4, 'big') + + # Pack file entries as records, build files map as standard msgpack map + files_map_bytes = _fixmap(len(files)) + for fname, finfo in files.items(): + files_map_bytes += _msgpack_pack(fname) + _record(finfo, 0x41) + + # Outer store_entry as record (struct_id 0x40) + store_entry_keys = ['algo', 'requiresBuild', 'files'] + if manifest is not None: + store_entry_keys.append('manifest') + + # Record with fixed entries + result = RECORD_HEADER + b'\x40' + _msgpack_pack(store_entry_keys) + result += _msgpack_pack(algo) # algo + result += _msgpack_pack(False) # requiresBuild + result += files_map_bytes # files (standard map → iterable Map in JS) + + if manifest is not None: + result += _record(manifest, 0x42) + return result + + def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None: with open(manifest_path, encoding='utf-8') as f: manifest = json.load(f) @@ -21,9 +158,24 @@ def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None store_version = manifest['store_version'] packages = manifest['packages'] + index_db: sqlite3.Connection | None = None + store = os.path.join(store_dir, store_version) os.makedirs(os.path.join(store, 'files'), exist_ok=True) - os.makedirs(os.path.join(store, 'index'), exist_ok=True) + if store_version == 'v11': + index_db = sqlite3.connect(os.path.join(store, 'index.db')) + index_db.execute('PRAGMA busy_timeout=5000') + index_db.execute('PRAGMA journal_mode=WAL') + index_db.execute('PRAGMA synchronous=NORMAL') + index_db.execute('PRAGMA temp_store=MEMORY') + index_db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + else: + os.makedirs(os.path.join(store, 'index'), exist_ok=True) now = int(time.time() * 1000) @@ -36,26 +188,37 @@ def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None tarball_path=tarball_path, pkg_name=info['name'], pkg_version=info['version'], - integrity_hex=info['integrity_hex'], + integrity=info['integrity'], + integrity_digest=info['integrity_digest'], + integrity_algo=info['integrity_algo'], store=store, now=now, tarball_url=info.get('tarball_url'), store_version=store_version, + index_db=index_db, ) + if index_db is not None: + index_db.commit() + index_db.close() + def _process_tarball( *, tarball_path: str, pkg_name: str, pkg_version: str, - integrity_hex: str, + integrity: str, + integrity_digest: str, + integrity_algo: str, store: str, now: int, tarball_url: str | None = None, store_version: str = 'v3', + index_db: sqlite3.Connection | None = None, ) -> None: index_files: dict[str, dict[str, object]] = {} + file_digests: dict[str, str] = {} real_pkg_name = pkg_name real_pkg_version = pkg_version @@ -104,47 +267,107 @@ def _process_tarball( 'mode': member.mode, 'size': len(data), } + file_digests[rel_name] = file_hex - index_data = { - 'name': real_pkg_name, - 'version': real_pkg_version, - 'requiresBuild': False, - 'files': index_files, - } - - idx_prefix = integrity_hex[:2] - idx_rest = integrity_hex[2:64] - pkg_id = _SANITIZE_RE.sub('+', f'{pkg_name}@{pkg_version}') - idx_dir = os.path.join(store, 'index', idx_prefix) - os.makedirs(idx_dir, exist_ok=True) - idx_path = os.path.join(idx_dir, f'{idx_rest}-{pkg_id}.json') - with open(idx_path, 'w', encoding='utf-8') as out: - json.dump(index_data, out) - - # For tarball-URL packages, also create an index entry keyed by the URL hash - # this is how pnpm looks up tarball deps without integrity - if tarball_url: - if store_version == 'v3': - url_hash = hashlib.sha256(tarball_url.encode()).hexdigest() - url_idx_prefix = url_hash[:2] - url_idx_rest = url_hash[2:64] - url_idx_dir = os.path.join(store, 'index', url_idx_prefix) - os.makedirs(url_idx_dir, exist_ok=True) - url_idx_path = os.path.join(url_idx_dir, f'{url_idx_rest}-{pkg_id}.json') - with open(url_idx_path, 'w', encoding='utf-8') as out: - json.dump(index_data, out) - else: - url_dir_name = re.sub(r'[:/]', '+', tarball_url) - if ( - len(url_dir_name) > _MAX_LENGTH_WITHOUT_HASH - or url_dir_name != url_dir_name.lower() - ): - url_dir_name = f'{url_dir_name[: _MAX_LENGTH_WITHOUT_HASH - 33]}_{hashlib.sha256(url_dir_name.encode()).hexdigest()[:32]}' - url_idx_dir = os.path.join(store, url_dir_name) - os.makedirs(url_idx_dir, exist_ok=True) - url_idx_path = os.path.join(url_idx_dir, 'integrity.json') - with open(url_idx_path, 'w', encoding='utf-8') as out: - json.dump(index_data, out) + if store_version == 'v11': + assert index_db is not None + # pnpm v11 store index keys use the raw package name (e.g. @scope/pkg), + # not the filesystem-sanitized form (@scope+pkg), since keys are stored + # in SQLite, not as filenames. + raw_pkg_id = f'{pkg_name}@{pkg_version}' + key = f'{integrity_algo}-{integrity}\t{raw_pkg_id}' + + v11_files: dict[str, dict[str, object]] = {} + for rel_name, finfo in index_files.items(): + checked_at = finfo['checkedAt'] + assert isinstance(checked_at, int) + v11_files[rel_name] = { + 'checkedAt': float(checked_at), # float64 avoids msgpackr BigInt + 'digest': file_digests[rel_name], + 'mode': finfo['mode'], + 'size': finfo['size'], + } + + manifest = None + if real_pkg_name or real_pkg_version: + manifest = { + 'name': real_pkg_name, + 'version': real_pkg_version, + } + + entry_bytes = _pack_v11_store_entry(integrity_algo, v11_files, manifest) + + # It's currently not possible to fully determine which store key pnpm will use, + # so we insert multiple keys to ensure pnpm can find the entry it wants. + + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (key, entry_bytes), + ) + + if tarball_url: + url_key = f'{tarball_url}\t{raw_pkg_id}' + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (url_key, entry_bytes), + ) + + # pnpm looks up git-hosted tarballs (codeload.github.com, + # bitbucket.org, gitlab.com) and tarballs without integrity by + # {tarball_url}\tbuilt(not-built) — see pickStoreIndexKey in @pnpm/store.index. + pkgid_key = f'{tarball_url}\t' + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (pkgid_key + 'built', entry_bytes), + ) + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (pkgid_key + 'not-built', entry_bytes), + ) + else: + pkg_id = _SANITIZE_RE.sub('+', f'{pkg_name}@{pkg_version}') + + index_data = { + 'name': real_pkg_name, + 'version': real_pkg_version, + 'requiresBuild': False, + 'files': index_files, + } + + idx_prefix = integrity_digest[:2] + idx_rest = integrity_digest[2:64] + idx_dir = os.path.join(store, 'index', idx_prefix) + os.makedirs(idx_dir, exist_ok=True) + idx_path = os.path.join(idx_dir, f'{idx_rest}-{pkg_id}.json') + with open(idx_path, 'w', encoding='utf-8') as out: + json.dump(index_data, out) + + # For tarball-URL packages, also create an index entry keyed by the URL hash + # this is how pnpm looks up tarball deps without integrity + if tarball_url: + if store_version == 'v3': + url_hash = hashlib.sha256(tarball_url.encode()).hexdigest() + url_idx_prefix = url_hash[:2] + url_idx_rest = url_hash[2:64] + url_idx_dir = os.path.join(store, 'index', url_idx_prefix) + os.makedirs(url_idx_dir, exist_ok=True) + url_idx_path = os.path.join( + url_idx_dir, f'{url_idx_rest}-{pkg_id}.json' + ) + with open(url_idx_path, 'w', encoding='utf-8') as out: + json.dump(index_data, out) + else: + url_dir_name = re.sub(r'[:/]', '+', tarball_url) + if ( + len(url_dir_name) > _MAX_LENGTH_WITHOUT_HASH + or url_dir_name != url_dir_name.lower() + ): + url_dir_name = f'{url_dir_name[: _MAX_LENGTH_WITHOUT_HASH - 33]}_{hashlib.sha256(url_dir_name.encode()).hexdigest()[:32]}' + url_idx_dir = os.path.join(store, url_dir_name) + os.makedirs(url_idx_dir, exist_ok=True) + url_idx_path = os.path.join(url_idx_dir, 'integrity.json') + with open(url_idx_path, 'w', encoding='utf-8') as out: + json.dump(index_data, out) if __name__ == '__main__': diff --git a/node/flatpak_node_generator/providers/pnpm.py b/node/flatpak_node_generator/providers/pnpm.py index fa3c11f5..9e6cd8e3 100644 --- a/node/flatpak_node_generator/providers/pnpm.py +++ b/node/flatpak_node_generator/providers/pnpm.py @@ -28,14 +28,16 @@ _V6_FORMAT_VERSIONS = {6, 7} _SUPPORTED_VERSIONS = {6, 7, 9} -_STORE_VERSION_BY_LOCKFILE: dict[int, str] = { - 6: 'v3', - 7: 'v3', - 9: 'v10', +_STORE_VERSION_BY_LOCKFILE: dict[int, list[str]] = { + 6: ['v3'], + 7: ['v3'], + 9: ['v10', 'v11'], } _POPULATE_STORE_SCRIPT = Path(__file__).parents[1] / 'populate_pnpm_store.py' +STORE_VERSION_ARGUMENT_DEFAULT = _STORE_VERSION_BY_LOCKFILE[9][0] + class PnpmLockfileProvider(LockfileProvider): """Parses pnpm-lock.yaml (v6/v7 and v9) into Package objects.""" @@ -46,10 +48,12 @@ class PnpmLockfileProvider(LockfileProvider): class Options(NamedTuple): no_devel: bool registry: str + store_version: str | None = None def __init__(self, options: 'PnpmLockfileProvider.Options') -> None: self.no_devel = options.no_devel self.registry = options.registry.rstrip('/') + self.store_version = options.store_version def _get_tarball_url( self, @@ -93,6 +97,17 @@ def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: f'Supported versions: {supported}.' ) + supported_store_versions = _STORE_VERSION_BY_LOCKFILE[major] + store_version = self.store_version + if store_version is None: + store_version = supported_store_versions[0] + elif store_version not in supported_store_versions: + supported = ', '.join(str(v) for v in sorted(supported_store_versions)) + raise ValueError( + f"{lockfile_path}: lockfileVersion {raw_version} doesn't support store version {self.store_version}. " + f'Supported versions: {supported}.' + ) + if self.no_devel and major not in _V6_FORMAT_VERSIONS: print( 'WARNING: --no-devel is not yet supported for pnpm lockfile v9; ' @@ -100,7 +115,7 @@ def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: file=sys.stderr, ) - lockfile = Lockfile(lockfile_path, major) + lockfile = Lockfile(lockfile_path, major, store_version=store_version) packages_dict: dict[str, Any] = data.get('packages', {}) if not packages_dict: @@ -181,11 +196,21 @@ def __exit__( self._finalize() async def generate_package(self, package: Package) -> None: - if self._store_version is None: - self._store_version = _STORE_VERSION_BY_LOCKFILE[package.lockfile.version] - source = package.source + sv = package.lockfile.store_version + if sv is None: + raise TypeError( + f'{package.name}@{package.version}: lockfile provides no store version' + ) + if self._store_version is None: + self._store_version = sv + elif self._store_version != sv: + raise ValueError( + f'{package.name}@{package.version}: store version mismatch: ' + f'expected {self._store_version!r}, got {sv!r}' + ) + if isinstance(source, ResolvedSource): assert source.resolved is not None @@ -239,7 +264,9 @@ def _add_store_population_script(self) -> None: entry: dict[str, str] = { 'name': info.name, 'version': info.version, - 'integrity_hex': info.integrity.digest, + 'integrity': info.integrity.to_base64(), + 'integrity_digest': info.integrity.digest, + 'integrity_algo': info.integrity.algorithm, } if info.version.startswith(('http://', 'https://')): entry['tarball_url'] = info.version @@ -263,7 +290,12 @@ def _add_store_population_script(self) -> None: ) def _add_pnpm_config(self) -> None: - self.gen.add_command(f'echo "store-dir=$PWD/{self.store_dir}" >> .npmrc') + if self._store_version == 'v11': + self.gen.add_command( + f'echo "storeDir=$PWD/{self.store_dir}" >> pnpm-workspace.yaml' + ) + else: + self.gen.add_command(f'echo "store-dir=$PWD/{self.store_dir}" >> .npmrc') class PnpmProviderFactory(ProviderFactory): diff --git a/node/poetry.lock b/node/poetry.lock index d3de5bfe..2e3ecab9 100644 --- a/node/poetry.lock +++ b/node/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" diff --git a/node/tests/test_pnpm.py b/node/tests/test_pnpm.py index b902850d..42d5c513 100644 --- a/node/tests/test_pnpm.py +++ b/node/tests/test_pnpm.py @@ -93,10 +93,11 @@ def test_lockfile_v9(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9, store_version='v10') lockfile.path.write_text(TEST_LOCKFILE_V9) packages = list(provider.process_lockfile(lockfile.path)) @@ -146,10 +147,11 @@ def test_lockfile_v6(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 6) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 6, store_version='v3') lockfile.path.write_text(TEST_LOCKFILE_V6) packages = list(provider.process_lockfile(lockfile.path)) @@ -199,6 +201,7 @@ def test_lockfile_v6_no_devel(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=True, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -218,6 +221,7 @@ def test_lockfile_v5_rejected(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -237,6 +241,7 @@ def test_lockfile_unsupported_version_rejected(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -258,6 +263,7 @@ def test_lockfile_v9_no_devel_warns( PnpmLockfileProvider.Options( no_devel=True, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -306,10 +312,11 @@ def test_lockfile_v9_git_and_local(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9, store_version='v10') lockfile.path.write_text(TEST_LOCKFILE_V9_GIT_AND_LOCAL) packages = list(provider.process_lockfile(lockfile.path)) @@ -358,13 +365,13 @@ def test_pnpm_module_provider_tarball_url(tmp_path: Path) -> None: tarball_name='normal-pkg-1.0.0.tgz', name='normal-pkg', version='1.0.0', - integrity=Integrity('sha512', 'abc123def456'), + integrity=Integrity('sha512', 'abc123def456ab'), ), PnpmModuleProvider._TarballInfo( tarball_name='url-pkg-http-123.tgz', name='url-pkg', version='http://example.com/url-pkg.tgz', - integrity=Integrity('sha512', 'fedcba654'), + integrity=Integrity('sha512', 'fedcba65401234'), ), PnpmModuleProvider._TarballInfo( tarball_name='url-pkg-https-123.tgz', @@ -391,6 +398,10 @@ def test_pnpm_module_provider_tarball_url(tmp_path: Path) -> None: for tarball in provider._tarballs: pkg = packages[tarball.tarball_name] assert pkg['version'] == tarball.version + expected_integrity = ( + f'{tarball.integrity.algorithm}-{tarball.integrity.to_base64()}' + ) + assert f'{pkg["integrity_algo"]}-{pkg["integrity"]}' == expected_integrity if tarball.version.startswith(('http://', 'https://')): assert pkg['tarball_url'] == tarball.version else: @@ -418,9 +429,8 @@ async def test_pnpm_module_provider_missing_integrity( ) provider = PnpmModuleProvider(gen, special, tmp_path) - provider._store_version = 'v3' - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9, store_version='v3') test_data = b'dummy tarball content' test_digest = hashlib.sha256(test_data).hexdigest() diff --git a/node/tests/test_populate_pnpm_store.py b/node/tests/test_populate_pnpm_store.py index c9d0e940..f00b1df0 100644 --- a/node/tests/test_populate_pnpm_store.py +++ b/node/tests/test_populate_pnpm_store.py @@ -1,10 +1,191 @@ import hashlib import json import re +import sqlite3 +import struct import tarfile from pathlib import Path -from flatpak_node_generator.populate_pnpm_store import _process_tarball +from flatpak_node_generator.integrity import Integrity +from flatpak_node_generator.populate_pnpm_store import ( + _pack_v11_store_entry, + _process_tarball, + populate_store, +) + + +def _decode_v11(data: bytes) -> dict[str, object]: + """Decode msgpackr record-extension encoding to plain dicts/lists. + + msgpackr with ``useRecords: true, moreTypes: true`` encodes: + + - plain objects as |d4 72 struct_id fixarray-of-keys values...| + - Maps as standard msgpack maps (0x80/0xde/0xdf) + + This decoder returns nested dicts (for both records and maps) so the + test assertions can use dictionary-style access. + """ + + def _read(offset: int) -> tuple[object, int]: + if offset >= len(data): + raise ValueError('unexpected end of data') + byte = data[offset] + offset += 1 + + # positive fixint + if byte <= 0x7F: + return byte, offset + # fixmap + if 0x80 <= byte <= 0x8F: + length = byte & 0x0F + result: dict[str, object] = {} + for _ in range(length): + k, offset = _read_str(offset) + v, offset = _read(offset) + result[k] = v + return result, offset + # fixarray + if 0x90 <= byte <= 0x9F: + length = byte & 0x0F + arr: list[object] = [] + for _ in range(length): + v, offset = _read(offset) + arr.append(v) + return arr, offset + # fixstr + if 0xA0 <= byte <= 0xBF: + length = byte & 0x1F + return data[offset : offset + length].decode('utf-8'), offset + length + # negative fixint + if 0xE0 <= byte <= 0xFF: + return byte - 256, offset + # nil + if byte == 0xC0: + return None, offset + # false / true + if byte == 0xC2: + return False, offset + if byte == 0xC3: + return True, offset + # bin 8/16/32 + if byte == 0xC4: + return data[offset : offset + data[offset]], offset + 1 + data[offset] + if byte == 0xC5: + length = int.from_bytes(data[offset : offset + 2], 'big') + return data[offset + 2 : offset + 2 + length], offset + 2 + length + if byte == 0xC6: + length = int.from_bytes(data[offset : offset + 4], 'big') + return data[offset + 4 : offset + 4 + length], offset + 4 + length + # float 32 + if byte == 0xCA: + return struct.unpack('>f', data[offset : offset + 4])[0], offset + 4 + # float 64 + if byte == 0xCB: + return struct.unpack('>d', data[offset : offset + 8])[0], offset + 8 + # uint 8/16/32/64 + if byte == 0xCC: + return data[offset], offset + 1 + if byte == 0xCD: + return int.from_bytes(data[offset : offset + 2], 'big'), offset + 2 + if byte == 0xCE: + return int.from_bytes(data[offset : offset + 4], 'big'), offset + 4 + if byte == 0xCF: + return int.from_bytes(data[offset : offset + 8], 'big'), offset + 8 + # int 8/16/32/64 + if byte == 0xD0: + return int.from_bytes( + data[offset : offset + 1], 'big', signed=True + ), offset + 1 + if byte == 0xD1: + return int.from_bytes( + data[offset : offset + 2], 'big', signed=True + ), offset + 2 + if byte == 0xD2: + return int.from_bytes( + data[offset : offset + 4], 'big', signed=True + ), offset + 4 + if byte == 0xD3: + return int.from_bytes( + data[offset : offset + 8], 'big', signed=True + ), offset + 8 + # fixext 4 with record extension + if byte == 0xD4 and data[offset] == 0x72: + offset += 1 # skip ext type 0x72 + offset += 1 # skip struct_id byte + keys, offset = _read(offset) + assert isinstance(keys, list), f'expected array of keys, got {type(keys)}' + record: dict[str, object] = {} + for k in keys: + v, offset = _read(offset) + record[str(k)] = v + return record, offset + # str 8/16/32 + if byte == 0xD9: + length = data[offset] + return data[offset + 1 : offset + 1 + length].decode( + 'utf-8' + ), offset + 1 + length + if byte == 0xDA: + length = int.from_bytes(data[offset : offset + 2], 'big') + return data[offset + 2 : offset + 2 + length].decode( + 'utf-8' + ), offset + 2 + length + if byte == 0xDB: + length = int.from_bytes(data[offset : offset + 4], 'big') + return data[offset + 4 : offset + 4 + length].decode( + 'utf-8' + ), offset + 4 + length + # map 16/32 + if byte == 0xDE: + length = int.from_bytes(data[offset : offset + 2], 'big') + offset += 2 + m: dict[str, object] = {} + for _ in range(length): + k, offset = _read_str(offset) + v, offset = _read(offset) + m[k] = v + return m, offset + if byte == 0xDF: + length = int.from_bytes(data[offset : offset + 4], 'big') + offset += 4 + m2: dict[str, object] = {} + for _ in range(length): + k, offset = _read_str(offset) + v, offset = _read(offset) + m2[k] = v + return m2, offset + # array 16/32 + if byte == 0xDC: + length = int.from_bytes(data[offset : offset + 2], 'big') + offset += 2 + arr2: list[object] = [] + for _ in range(length): + v, offset = _read(offset) + arr2.append(v) + return arr2, offset + if byte == 0xDD: + length = int.from_bytes(data[offset : offset + 4], 'big') + offset += 4 + arr3: list[object] = [] + for _ in range(length): + v, offset = _read(offset) + arr3.append(v) + return arr3, offset + + raise ValueError( + f'unsupported msgpack byte: 0x{byte:02X} at offset {offset - 1}' + ) + + def _read_str(offset: int) -> tuple[str, int]: + val, offset = _read(offset) + assert isinstance(val, str), f'expected str, got {type(val)}' + return val, offset + + result, offset = _read(0) + if offset != len(data): + raise ValueError(f'extra bytes at end: {offset} < {len(data)}') + assert isinstance(result, dict) + return result def _create_tarball(path: Path, files: dict[str, str | bytes]) -> None: @@ -33,11 +214,17 @@ def test_process_tarball_normal(tmp_path: Path) -> None: {'package/package.json': pkg_json, 'package/index.js': "console.log('hello');"}, ) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='fallback-pkg', pkg_version='0.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, ) @@ -61,11 +248,17 @@ def test_process_tarball_malformed_package_json(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/package.json': '{ malformed: json '}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='fallback-pkg', pkg_version='0.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, ) @@ -87,11 +280,17 @@ def test_process_tarball_with_tarball_url_v3(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -114,11 +313,17 @@ def test_process_tarball_with_tarball_url_v6(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -138,11 +343,17 @@ def test_process_tarball_with_uppercase_path(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -163,11 +374,17 @@ def test_process_tarball_with_long_path(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -179,3 +396,284 @@ def test_process_tarball_with_long_path(tmp_path: Path) -> None: url_idx_file = store_dir / normalized_tarball_url / 'integrity.json' assert url_idx_file.exists() + + +def test_populate_store_v11(tmp_path: Path) -> None: + manifest_path = tmp_path / 'manifest.json' + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' + pkg_json = json.dumps({'name': 'real-pkg', 'version': '1.2.3'}) + + integrity = Integrity( + 'sha512', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + _create_tarball( + tar_path, + {'package/package.json': pkg_json, 'package/index.js': "console.log('hello');"}, + ) + + manifest_json = json.dumps( + { + 'store_version': 'v11', + 'packages': { + 'pkg.tgz': { + 'name': 'real-pkg', + 'version': '1.2.3', + 'integrity': integrity.to_base64(), + 'integrity_digest': integrity.digest, + 'integrity_algo': integrity.algorithm, + } + }, + } + ) + + with open(manifest_path, 'w', encoding='utf-8') as f: + f.write(manifest_json) + + populate_store(str(manifest_path), str(tmp_path), str(store_dir)) + + db = sqlite3.connect(str(store_dir / 'v11' / 'index.db')) + + pkg_id = 'real-pkg@1.2.3' + expected_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + + row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (expected_key,) + ).fetchone() + assert row is not None, f'No row found for key {expected_key}' + + data = _decode_v11(row[0]) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert data['manifest'] == {'name': 'real-pkg', 'version': '1.2.3'} + assert isinstance(data['files'], dict) + assert 'package.json' in data['files'] + assert 'index.js' in data['files'] + + +def test_process_tarball_v11(tmp_path: Path) -> None: + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' / 'v11' + pkg_json = json.dumps({'name': 'real-pkg', 'version': '1.2.3'}) + + _create_tarball( + tar_path, + {'package/package.json': pkg_json, 'package/index.js': "console.log('hello');"}, + ) + + store_dir.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(str(store_dir / 'index.db')) + db.execute('PRAGMA busy_timeout=5000') + db.execute('PRAGMA journal_mode=WAL') + db.execute('PRAGMA synchronous=NORMAL') + db.execute('PRAGMA temp_store=MEMORY') + db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + + integrity = Integrity( + 'sha512', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + try: + _process_tarball( + tarball_path=str(tar_path), + pkg_name='fallback-pkg', + pkg_version='0.0.0', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, + store=str(store_dir), + now=1234567890, + store_version='v11', + index_db=db, + ) + db.commit() + + pkg_id = 'fallback-pkg@0.0.0' + expected_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + + row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (expected_key,) + ).fetchone() + assert row is not None, f'No row found for key {expected_key}' + + data = _decode_v11(row[0]) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert data['manifest'] == {'name': 'real-pkg', 'version': '1.2.3'} + assert isinstance(data['files'], dict) + assert 'package.json' in data['files'] + assert 'index.js' in data['files'] + finally: + db.close() + + +def test_process_tarball_v11_no_package_json(tmp_path: Path) -> None: + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' / 'v11' + + _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + + store_dir.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(str(store_dir / 'index.db')) + db.execute('PRAGMA busy_timeout=5000') + db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + try: + _process_tarball( + tarball_path=str(tar_path), + pkg_name='no-manifest-pkg', + pkg_version='2.0.0', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, + store=str(store_dir), + now=1234567890, + store_version='v11', + index_db=db, + ) + db.commit() + + pkg_id = 'no-manifest-pkg@2.0.0' + expected_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + + row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (expected_key,) + ).fetchone() + assert row is not None + + data = _decode_v11(row[0]) + # Fallback name/version used since there's no real package.json + assert data['manifest'] == { + 'name': 'no-manifest-pkg', + 'version': '2.0.0', + } + finally: + db.close() + + +def test_process_tarball_v11_with_tarball_url(tmp_path: Path) -> None: + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' / 'v11' + tarball_url = 'https://example.com/pkg.tgz' + + _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + + store_dir.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(str(store_dir / 'index.db')) + db.execute('PRAGMA busy_timeout=5000') + db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + try: + _process_tarball( + tarball_path=str(tar_path), + pkg_name='pkg', + pkg_version='1.0.0', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, + store=str(store_dir), + now=1234567890, + store_version='v11', + index_db=db, + tarball_url=tarball_url, + ) + db.commit() + + # Check main integrity-based entry exists + pkg_id = 'pkg@1.0.0' + main_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + main_row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (main_key,) + ).fetchone() + assert main_row is not None + + # Check tarball_url-based entry exists + url_key = f'{tarball_url}\t{pkg_id}' + url_row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (url_key,) + ).fetchone() + assert url_row is not None + + # Check pkgId-based entry exists ({pkgId}\tbuilt — git-hosted tarball format) + pkgid_key = f'{tarball_url}\tbuilt' + pkgid_row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (pkgid_key,) + ).fetchone() + assert pkgid_row is not None + finally: + db.close() + + +def test_pack_v11_store_entry() -> None: + """Verify _pack_v11_store_entry produces msgpackr-compatible record encoding.""" + files = { + 'index.js': { + 'checkedAt': 1234567890, + 'digest': 'abc123', + 'mode': 420, + 'size': 100, + }, + 'package.json': { + 'checkedAt': 111111, + 'digest': 'def456', + 'mode': 384, + 'size': 200, + }, + } + manifest = {'name': 'test-pkg', 'version': '1.0.0'} + packed = _pack_v11_store_entry('sha512', files, manifest) + + data = _decode_v11(packed) + assert isinstance(data, dict) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert isinstance(data['files'], dict) + assert data['files']['index.js']['digest'] == 'abc123' + assert data['files']['index.js']['mode'] == 420 + assert data['files']['index.js']['size'] == 100 + assert data['files']['index.js']['checkedAt'] == 1234567890 + assert data['files']['package.json']['digest'] == 'def456' + assert data['manifest'] == manifest + + +def test_pack_v11_store_entry_no_manifest() -> None: + """Verify _pack_v11_store_entry works without a manifest.""" + files = { + 'index.js': {'checkedAt': 123, 'digest': 'hex123', 'mode': 420, 'size': 50} + } + packed = _pack_v11_store_entry('sha512', files, None) + + data = _decode_v11(packed) + assert isinstance(data, dict) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert 'manifest' not in data + files_obj = data['files'] + assert isinstance(files_obj, dict) + index_js = files_obj['index.js'] + assert isinstance(index_js, dict) + assert index_js['digest'] == 'hex123'