From 4dcceeb30a9036872bdb07402bd6fcbf427f6ccb Mon Sep 17 00:00:00 2001 From: LuoRain Date: Mon, 11 May 2026 19:19:21 +0800 Subject: [PATCH 1/5] feat/node: pnpm store v11 support --- node/README.md | 1 + node/flatpak_node_generator/main.py | 6 + node/flatpak_node_generator/package.py | 1 + .../populate_pnpm_store.py | 307 ++++++++++-- node/flatpak_node_generator/providers/pnpm.py | 44 +- node/poetry.lock | 76 ++- node/tests/test_pnpm.py | 24 +- node/tests/test_populate_pnpm_store.py | 457 +++++++++++++++++- 8 files changed, 847 insertions(+), 69 deletions(-) diff --git a/node/README.md b/node/README.md index 4287e630..cf904b46 100644 --- a/node/README.md +++ b/node/README.md @@ -96,6 +96,7 @@ options: Specify NW.js version (will use latest otherwise) --nwjs-node-headers Download the NW.js node headers --nwjs-ffmpeg Download prebuilt ffmpeg for current NW.js version + --pnpm-store-version Specify the store version for pnpm v9 lockfile. Default is v10. --no-xdg-layout Don't use the XDG layout for caches --node-sdk-extension NODE_SDK_EXTENSION Flatpak node SDK extension (e.g. org.freedesktop.Sdk.Extension.node24//25.08) diff --git a/node/flatpak_node_generator/main.py b/node/flatpak_node_generator/main.py index 07ee6e0a..c9d09626 100644 --- a/node/flatpak_node_generator/main.py +++ b/node/flatpak_node_generator/main.py @@ -14,6 +14,7 @@ from .providers import ProviderFactory from .providers.npm import NpmLockfileProvider, NpmModuleProvider, NpmProviderFactory from .providers.pnpm import ( + STORE_VERSION_ARGUMENT_DEFAULT, PnpmLockfileProvider, PnpmProviderFactory, ) @@ -113,6 +114,10 @@ async def _async_main() -> None: help='Use the ChromeDriver version associated with the given ' 'Electron version for node-chromedriver', ) + parser.add_argument( + '--pnpm-store-version', + help=f'Specify the store version for pnpm v9 lockfile. Default is {STORE_VERSION_ARGUMENT_DEFAULT}', + ) # Deprecated alternative to --node-chromedriver-from-electron parser.add_argument('--electron-chromedriver', help=argparse.SUPPRESS) parser.add_argument( @@ -209,6 +214,7 @@ async def _async_main() -> None: PnpmLockfileProvider.Options( no_devel=args.no_devel, registry=args.registry, + store_version=args.pnpm_store_version, ), ) provider_factory = PnpmProviderFactory(lockfile_root, pnpm_options) diff --git a/node/flatpak_node_generator/package.py b/node/flatpak_node_generator/package.py index 9e81f7b9..4fd8048b 100644 --- a/node/flatpak_node_generator/package.py +++ b/node/flatpak_node_generator/package.py @@ -143,6 +143,7 @@ class Lockfile: path: Path version: int cache_key: str | None = None + store_version: str | None = None class Package(NamedTuple): diff --git a/node/flatpak_node_generator/populate_pnpm_store.py b/node/flatpak_node_generator/populate_pnpm_store.py index 46233e96..3e7d689b 100644 --- a/node/flatpak_node_generator/populate_pnpm_store.py +++ b/node/flatpak_node_generator/populate_pnpm_store.py @@ -6,14 +6,149 @@ import json import os import re +import sqlite3 +import struct import sys import tarfile import time +from collections.abc import Mapping _SANITIZE_RE = re.compile(r'[\\/:*?"<>|]') _MAX_LENGTH_WITHOUT_HASH = 120 +def _msgpack_pack(obj: object) -> bytes: + """Minimal msgpack packer matching msgpackr with useRecords: true. + + msgpackr reserves the byte range 0x40-0x7F for record IDs when useRecords + is enabled (pnpm's config). Standard msgpack uses 0x00-0x7F for positive + fixints, but this packer caps them at 0x3F and emits a uint 8 prefix (0xcc) + for 64-255 to avoid aliasing with record ID bytes. + """ + + def _pack_int(val: int) -> bytes: + # Positive fixint capped at 0x3F — 0x40-0x7F are msgpackr record IDs + if 0 <= val <= 0x3F: + return val.to_bytes(1, 'big') + if -32 <= val < 0: + return val.to_bytes(1, 'big', signed=True) + if val >= 0: + if val <= 0xFF: + return b'\xcc' + val.to_bytes(1, 'big') + if val <= 0xFFFF: + return b'\xcd' + val.to_bytes(2, 'big') + if val <= 0xFFFFFFFF: + return b'\xce' + val.to_bytes(4, 'big') + return b'\xcf' + val.to_bytes(8, 'big') + if val >= -0x80: + return b'\xd0' + val.to_bytes(1, 'big', signed=True) + if val >= -0x8000: + return b'\xd1' + val.to_bytes(2, 'big', signed=True) + if val >= -0x80000000: + return b'\xd2' + val.to_bytes(4, 'big', signed=True) + return b'\xd3' + val.to_bytes(8, 'big', signed=True) + + def _pack_str(val: str) -> bytes: + data = val.encode('utf-8') + length = len(data) + if length <= 0x1F: + return (0xA0 | length).to_bytes(1, 'big') + data + if length <= 0xFF: + return b'\xd9' + length.to_bytes(1, 'big') + data + if length <= 0xFFFF: + return b'\xda' + length.to_bytes(2, 'big') + data + return b'\xdb' + length.to_bytes(4, 'big') + data + + if obj is None: + return b'\xc0' + if isinstance(obj, bool): + return b'\xc3' if obj else b'\xc2' + if isinstance(obj, int): + return _pack_int(obj) + if isinstance(obj, float): + return b'\xcb' + struct.pack('>d', obj) + if isinstance(obj, str): + return _pack_str(obj) + if isinstance(obj, dict): + length = len(obj) + if length <= 0x0F: + result = (0x80 | length).to_bytes(1, 'big') + elif length <= 0xFFFF: + result = b'\xde' + length.to_bytes(2, 'big') + else: + result = b'\xdf' + length.to_bytes(4, 'big') + for key, val in obj.items(): + result += _msgpack_pack(key) + _msgpack_pack(val) + return result + if isinstance(obj, (list, tuple)): + length = len(obj) + if length <= 0x0F: + result = (0x90 | length).to_bytes(1, 'big') + elif length <= 0xFFFF: + result = b'\xdc' + length.to_bytes(2, 'big') + else: + result = b'\xdd' + length.to_bytes(4, 'big') + for item in obj: + result += _msgpack_pack(item) + return result + raise TypeError(f'Unsupported type for msgpack: {type(obj)}') + + +RECORD_HEADER = b'\xd4\x72' + + +def _pack_v11_store_entry( + files: dict[str, dict[str, object]], + manifest: dict[str, str] | None = None, +) -> bytes: + """Encode a store v11 entry using msgpackr-compatible record extensions. + + msgpackr with ``useRecords: true, moreTypes: true`` decodes: + + - standard msgpack maps (0x80/0xde/0xdf) as JavaScript ``Map`` objects + (iterable, supports ``for..of``) + - record extensions (0x72) as plain objects (dot-notation access) + + The ``files`` field must be a Map so pnpm can iterate it with ``for..of``. + Everything else must use record extensions so dot-notation works + (e.g. ``pkgIndex.algo``, ``info.digest``, ``manifest.name``). + """ + + def _record(obj: Mapping[str, object], struct_id: int) -> bytes: + keys = list(obj.keys()) + result = RECORD_HEADER + struct_id.to_bytes(1, 'big') + result += _msgpack_pack(keys) + for k in keys: + result += _msgpack_pack(obj[k]) + return result + + def _fixmap(length: int) -> bytes: + if length <= 0x0F: + return (0x80 | length).to_bytes(1, 'big') + if length <= 0xFFFF: + return b'\xde' + length.to_bytes(2, 'big') + return b'\xdf' + length.to_bytes(4, 'big') + + # Pack file entries as records, build files map as standard msgpack map + files_map_bytes = _fixmap(len(files)) + for fname, finfo in files.items(): + files_map_bytes += _msgpack_pack(fname) + _record(finfo, 0x41) + + # Outer store_entry as record (struct_id 0x40) + store_entry_keys = ['algo', 'requiresBuild', 'files'] + if manifest is not None: + store_entry_keys.append('manifest') + + result = b'\xd4\x72\x40' + _msgpack_pack(store_entry_keys) + result += _msgpack_pack('sha512') # algo + result += _msgpack_pack(False) # requiresBuild + result += files_map_bytes # files (standard map → iterable Map in JS) + + if manifest is not None: + result += _record(manifest, 0x42) + return result + + def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None: with open(manifest_path, encoding='utf-8') as f: manifest = json.load(f) @@ -21,9 +156,24 @@ def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None store_version = manifest['store_version'] packages = manifest['packages'] + index_db: sqlite3.Connection | None = None + store = os.path.join(store_dir, store_version) os.makedirs(os.path.join(store, 'files'), exist_ok=True) - os.makedirs(os.path.join(store, 'index'), exist_ok=True) + if store_version == 'v11': + index_db = sqlite3.connect(os.path.join(store, 'index.db')) + index_db.execute('PRAGMA busy_timeout=5000') + index_db.execute('PRAGMA journal_mode=WAL') + index_db.execute('PRAGMA synchronous=NORMAL') + index_db.execute('PRAGMA temp_store=MEMORY') + index_db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + else: + os.makedirs(os.path.join(store, 'index'), exist_ok=True) now = int(time.time() * 1000) @@ -36,26 +186,37 @@ def populate_store(manifest_path: str, tarball_dir: str, store_dir: str) -> None tarball_path=tarball_path, pkg_name=info['name'], pkg_version=info['version'], - integrity_hex=info['integrity_hex'], + integrity=info['integrity'], + integrity_digest=info['integrity_digest'], + integrity_algo=info['integrity_algo'], store=store, now=now, tarball_url=info.get('tarball_url'), store_version=store_version, + index_db=index_db, ) + if index_db is not None: + index_db.commit() + index_db.close() + def _process_tarball( *, tarball_path: str, pkg_name: str, pkg_version: str, - integrity_hex: str, + integrity: str, + integrity_digest: str, + integrity_algo: str, store: str, now: int, tarball_url: str | None = None, store_version: str = 'v3', + index_db: sqlite3.Connection | None = None, ) -> None: index_files: dict[str, dict[str, object]] = {} + file_digests: dict[str, str] = {} real_pkg_name = pkg_name real_pkg_version = pkg_version @@ -104,47 +265,107 @@ def _process_tarball( 'mode': member.mode, 'size': len(data), } + file_digests[rel_name] = file_hex - index_data = { - 'name': real_pkg_name, - 'version': real_pkg_version, - 'requiresBuild': False, - 'files': index_files, - } - - idx_prefix = integrity_hex[:2] - idx_rest = integrity_hex[2:64] - pkg_id = _SANITIZE_RE.sub('+', f'{pkg_name}@{pkg_version}') - idx_dir = os.path.join(store, 'index', idx_prefix) - os.makedirs(idx_dir, exist_ok=True) - idx_path = os.path.join(idx_dir, f'{idx_rest}-{pkg_id}.json') - with open(idx_path, 'w', encoding='utf-8') as out: - json.dump(index_data, out) - - # For tarball-URL packages, also create an index entry keyed by the URL hash - # this is how pnpm looks up tarball deps without integrity - if tarball_url: - if store_version == 'v3': - url_hash = hashlib.sha256(tarball_url.encode()).hexdigest() - url_idx_prefix = url_hash[:2] - url_idx_rest = url_hash[2:64] - url_idx_dir = os.path.join(store, 'index', url_idx_prefix) - os.makedirs(url_idx_dir, exist_ok=True) - url_idx_path = os.path.join(url_idx_dir, f'{url_idx_rest}-{pkg_id}.json') - with open(url_idx_path, 'w', encoding='utf-8') as out: - json.dump(index_data, out) - else: - url_dir_name = re.sub(r'[:/]', '+', tarball_url) - if ( - len(url_dir_name) > _MAX_LENGTH_WITHOUT_HASH - or url_dir_name != url_dir_name.lower() - ): - url_dir_name = f'{url_dir_name[: _MAX_LENGTH_WITHOUT_HASH - 33]}_{hashlib.sha256(url_dir_name.encode()).hexdigest()[:32]}' - url_idx_dir = os.path.join(store, url_dir_name) - os.makedirs(url_idx_dir, exist_ok=True) - url_idx_path = os.path.join(url_idx_dir, 'integrity.json') - with open(url_idx_path, 'w', encoding='utf-8') as out: - json.dump(index_data, out) + if store_version == 'v11': + assert index_db is not None + # pnpm v11 store index keys use the raw package name (e.g. @scope/pkg), + # not the filesystem-sanitized form (@scope+pkg), since keys are stored + # in SQLite, not as filenames. + raw_pkg_id = f'{pkg_name}@{pkg_version}' + key = f'{integrity_algo}-{integrity}\t{raw_pkg_id}' + + v11_files: dict[str, dict[str, object]] = {} + for rel_name, finfo in index_files.items(): + checked_at = finfo['checkedAt'] + assert isinstance(checked_at, int) + v11_files[rel_name] = { + 'checkedAt': float(checked_at), # float64 avoids msgpackr BigInt + 'digest': file_digests[rel_name], + 'mode': finfo['mode'], + 'size': finfo['size'], + } + + manifest = None + if real_pkg_name or real_pkg_version: + manifest = { + 'name': real_pkg_name, + 'version': real_pkg_version, + } + + entry_bytes = _pack_v11_store_entry(v11_files, manifest) + + # It's currently not possible to fully determine which store key pnpm will use, + # so we insert multiple keys to ensure pnpm can find the entry it wants. + + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (key, entry_bytes), + ) + + if tarball_url: + url_key = f'{tarball_url}\t{raw_pkg_id}' + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (url_key, entry_bytes), + ) + + # pnpm looks up git-hosted tarballs (codeload.github.com, + # bitbucket.org, gitlab.com) and tarballs without integrity by + # {tarball_url}\tbuilt(not-built) — see pickStoreIndexKey in @pnpm/store.index. + pkgid_key = f'{tarball_url}\t' + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (pkgid_key + 'built', entry_bytes), + ) + index_db.execute( + 'INSERT OR REPLACE INTO package_index (key, data) VALUES (?, ?)', + (pkgid_key + 'not-built', entry_bytes), + ) + else: + pkg_id = _SANITIZE_RE.sub('+', f'{pkg_name}@{pkg_version}') + + index_data = { + 'name': real_pkg_name, + 'version': real_pkg_version, + 'requiresBuild': False, + 'files': index_files, + } + + idx_prefix = integrity_digest[:2] + idx_rest = integrity_digest[2:64] + idx_dir = os.path.join(store, 'index', idx_prefix) + os.makedirs(idx_dir, exist_ok=True) + idx_path = os.path.join(idx_dir, f'{idx_rest}-{pkg_id}.json') + with open(idx_path, 'w', encoding='utf-8') as out: + json.dump(index_data, out) + + # For tarball-URL packages, also create an index entry keyed by the URL hash + # this is how pnpm looks up tarball deps without integrity + if tarball_url: + if store_version == 'v3': + url_hash = hashlib.sha256(tarball_url.encode()).hexdigest() + url_idx_prefix = url_hash[:2] + url_idx_rest = url_hash[2:64] + url_idx_dir = os.path.join(store, 'index', url_idx_prefix) + os.makedirs(url_idx_dir, exist_ok=True) + url_idx_path = os.path.join( + url_idx_dir, f'{url_idx_rest}-{pkg_id}.json' + ) + with open(url_idx_path, 'w', encoding='utf-8') as out: + json.dump(index_data, out) + else: + url_dir_name = re.sub(r'[:/]', '+', tarball_url) + if ( + len(url_dir_name) > _MAX_LENGTH_WITHOUT_HASH + or url_dir_name != url_dir_name.lower() + ): + url_dir_name = f'{url_dir_name[: _MAX_LENGTH_WITHOUT_HASH - 33]}_{hashlib.sha256(url_dir_name.encode()).hexdigest()[:32]}' + url_idx_dir = os.path.join(store, url_dir_name) + os.makedirs(url_idx_dir, exist_ok=True) + url_idx_path = os.path.join(url_idx_dir, 'integrity.json') + with open(url_idx_path, 'w', encoding='utf-8') as out: + json.dump(index_data, out) if __name__ == '__main__': diff --git a/node/flatpak_node_generator/providers/pnpm.py b/node/flatpak_node_generator/providers/pnpm.py index fa3c11f5..f27c5b52 100644 --- a/node/flatpak_node_generator/providers/pnpm.py +++ b/node/flatpak_node_generator/providers/pnpm.py @@ -28,14 +28,16 @@ _V6_FORMAT_VERSIONS = {6, 7} _SUPPORTED_VERSIONS = {6, 7, 9} -_STORE_VERSION_BY_LOCKFILE: dict[int, str] = { - 6: 'v3', - 7: 'v3', - 9: 'v10', +_STORE_VERSION_BY_LOCKFILE: dict[int, list[str]] = { + 6: ['v3'], + 7: ['v3'], + 9: ['v10', 'v11'], } _POPULATE_STORE_SCRIPT = Path(__file__).parents[1] / 'populate_pnpm_store.py' +STORE_VERSION_ARGUMENT_DEFAULT = _STORE_VERSION_BY_LOCKFILE[9][0] + class PnpmLockfileProvider(LockfileProvider): """Parses pnpm-lock.yaml (v6/v7 and v9) into Package objects.""" @@ -46,10 +48,12 @@ class PnpmLockfileProvider(LockfileProvider): class Options(NamedTuple): no_devel: bool registry: str + store_version: str | None = None def __init__(self, options: 'PnpmLockfileProvider.Options') -> None: self.no_devel = options.no_devel self.registry = options.registry.rstrip('/') + self.store_version = options.store_version def _get_tarball_url( self, @@ -93,6 +97,16 @@ def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: f'Supported versions: {supported}.' ) + supported_store_versions = _STORE_VERSION_BY_LOCKFILE[major] + if self.store_version is None: + self.store_version = supported_store_versions[0] + elif self.store_version not in supported_store_versions: + supported = ', '.join(str(v) for v in sorted(supported_store_versions)) + raise ValueError( + f"{lockfile_path}: lockfileVersion {raw_version} doesn't support store version {self.store_version}. " + f'Supported versions: {supported}.' + ) + if self.no_devel and major not in _V6_FORMAT_VERSIONS: print( 'WARNING: --no-devel is not yet supported for pnpm lockfile v9; ' @@ -100,7 +114,7 @@ def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: file=sys.stderr, ) - lockfile = Lockfile(lockfile_path, major) + lockfile = Lockfile(lockfile_path, major, store_version=self.store_version) packages_dict: dict[str, Any] = data.get('packages', {}) if not packages_dict: @@ -181,11 +195,14 @@ def __exit__( self._finalize() async def generate_package(self, package: Package) -> None: - if self._store_version is None: - self._store_version = _STORE_VERSION_BY_LOCKFILE[package.lockfile.version] - source = package.source + if self._store_version is None: + sv = package.lockfile.store_version + if sv is None: + raise TypeError('pnpm expects lockfile provides store version') + self._store_version = sv + if isinstance(source, ResolvedSource): assert source.resolved is not None @@ -239,7 +256,9 @@ def _add_store_population_script(self) -> None: entry: dict[str, str] = { 'name': info.name, 'version': info.version, - 'integrity_hex': info.integrity.digest, + 'integrity': info.integrity.to_base64(), + 'integrity_digest': info.integrity.digest, + 'integrity_algo': info.integrity.algorithm, } if info.version.startswith(('http://', 'https://')): entry['tarball_url'] = info.version @@ -263,7 +282,12 @@ def _add_store_population_script(self) -> None: ) def _add_pnpm_config(self) -> None: - self.gen.add_command(f'echo "store-dir=$PWD/{self.store_dir}" >> .npmrc') + if self._store_version == 'v11': + self.gen.add_command( + f'echo "storeDir=$PWD/{self.store_dir}" >> pnpm-workspace.yaml' + ) + else: + self.gen.add_command(f'echo "store-dir=$PWD/{self.store_dir}" >> .npmrc') class PnpmProviderFactory(ProviderFactory): diff --git a/node/poetry.lock b/node/poetry.lock index d3de5bfe..4fc210f0 100644 --- a/node/poetry.lock +++ b/node/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -623,6 +623,78 @@ files = [ {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, ] +[[package]] +name = "msgpack" +version = "1.1.2" +description = "MessagePack serializer" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2"}, + {file = "msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87"}, + {file = "msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251"}, + {file = "msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a"}, + {file = "msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f"}, + {file = "msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f"}, + {file = "msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9"}, + {file = "msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa"}, + {file = "msgpack-1.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e86a607e558d22985d856948c12a3fa7b42efad264dca8a3ebbcfa2735d786c"}, + {file = "msgpack-1.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:283ae72fc89da59aa004ba147e8fc2f766647b1251500182fac0350d8af299c0"}, + {file = "msgpack-1.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61c8aa3bd513d87c72ed0b37b53dd5c5a0f58f2ff9f26e1555d3bd7948fb7296"}, + {file = "msgpack-1.1.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:454e29e186285d2ebe65be34629fa0e8605202c60fbc7c4c650ccd41870896ef"}, + {file = "msgpack-1.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7bc8813f88417599564fafa59fd6f95be417179f76b40325b500b3c98409757c"}, + {file = "msgpack-1.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bafca952dc13907bdfdedfc6a5f579bf4f292bdd506fadb38389afa3ac5b208e"}, + {file = "msgpack-1.1.2-cp311-cp311-win32.whl", hash = "sha256:602b6740e95ffc55bfb078172d279de3773d7b7db1f703b2f1323566b878b90e"}, + {file = "msgpack-1.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:d198d275222dc54244bf3327eb8cbe00307d220241d9cec4d306d49a44e85f68"}, + {file = "msgpack-1.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:86f8136dfa5c116365a8a651a7d7484b65b13339731dd6faebb9a0242151c406"}, + {file = "msgpack-1.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:70a0dff9d1f8da25179ffcf880e10cf1aad55fdb63cd59c9a49a1b82290062aa"}, + {file = "msgpack-1.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:446abdd8b94b55c800ac34b102dffd2f6aa0ce643c55dfc017ad89347db3dbdb"}, + {file = "msgpack-1.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c63eea553c69ab05b6747901b97d620bb2a690633c77f23feb0c6a947a8a7b8f"}, + {file = "msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:372839311ccf6bdaf39b00b61288e0557916c3729529b301c52c2d88842add42"}, + {file = "msgpack-1.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2929af52106ca73fcb28576218476ffbb531a036c2adbcf54a3664de124303e9"}, + {file = "msgpack-1.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:be52a8fc79e45b0364210eef5234a7cf8d330836d0a64dfbb878efa903d84620"}, + {file = "msgpack-1.1.2-cp312-cp312-win32.whl", hash = "sha256:1fff3d825d7859ac888b0fbda39a42d59193543920eda9d9bea44d958a878029"}, + {file = "msgpack-1.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1de460f0403172cff81169a30b9a92b260cb809c4cb7e2fc79ae8d0510c78b6b"}, + {file = "msgpack-1.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:be5980f3ee0e6bd44f3a9e9dea01054f175b50c3e6cdb692bc9424c0bbb8bf69"}, + {file = "msgpack-1.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4efd7b5979ccb539c221a4c4e16aac1a533efc97f3b759bb5a5ac9f6d10383bf"}, + {file = "msgpack-1.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42eefe2c3e2af97ed470eec850facbe1b5ad1d6eacdbadc42ec98e7dcf68b4b7"}, + {file = "msgpack-1.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fdf7d83102bf09e7ce3357de96c59b627395352a4024f6e2458501f158bf999"}, + {file = "msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fac4be746328f90caa3cd4bc67e6fe36ca2bf61d5c6eb6d895b6527e3f05071e"}, + {file = "msgpack-1.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fffee09044073e69f2bad787071aeec727183e7580443dfeb8556cbf1978d162"}, + {file = "msgpack-1.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5928604de9b032bc17f5099496417f113c45bc6bc21b5c6920caf34b3c428794"}, + {file = "msgpack-1.1.2-cp313-cp313-win32.whl", hash = "sha256:a7787d353595c7c7e145e2331abf8b7ff1e6673a6b974ded96e6d4ec09f00c8c"}, + {file = "msgpack-1.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:a465f0dceb8e13a487e54c07d04ae3ba131c7c5b95e2612596eafde1dccf64a9"}, + {file = "msgpack-1.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:e69b39f8c0aa5ec24b57737ebee40be647035158f14ed4b40e6f150077e21a84"}, + {file = "msgpack-1.1.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e23ce8d5f7aa6ea6d2a2b326b4ba46c985dbb204523759984430db7114f8aa00"}, + {file = "msgpack-1.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c15b7d74c939ebe620dd8e559384be806204d73b4f9356320632d783d1f7939"}, + {file = "msgpack-1.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99e2cb7b9031568a2a5c73aa077180f93dd2e95b4f8d3b8e14a73ae94a9e667e"}, + {file = "msgpack-1.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:180759d89a057eab503cf62eeec0aa61c4ea1200dee709f3a8e9397dbb3b6931"}, + {file = "msgpack-1.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:04fb995247a6e83830b62f0b07bf36540c213f6eac8e851166d8d86d83cbd014"}, + {file = "msgpack-1.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8e22ab046fa7ede9e36eeb4cfad44d46450f37bb05d5ec482b02868f451c95e2"}, + {file = "msgpack-1.1.2-cp314-cp314-win32.whl", hash = "sha256:80a0ff7d4abf5fecb995fcf235d4064b9a9a8a40a3ab80999e6ac1e30b702717"}, + {file = "msgpack-1.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:9ade919fac6a3e7260b7f64cea89df6bec59104987cbea34d34a2fa15d74310b"}, + {file = "msgpack-1.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:59415c6076b1e30e563eb732e23b994a61c159cec44deaf584e5cc1dd662f2af"}, + {file = "msgpack-1.1.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:897c478140877e5307760b0ea66e0932738879e7aa68144d9b78ea4c8302a84a"}, + {file = "msgpack-1.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a668204fa43e6d02f89dbe79a30b0d67238d9ec4c5bd8a940fc3a004a47b721b"}, + {file = "msgpack-1.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5559d03930d3aa0f3aacb4c42c776af1a2ace2611871c84a75afe436695e6245"}, + {file = "msgpack-1.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70c5a7a9fea7f036b716191c29047374c10721c389c21e9ffafad04df8c52c90"}, + {file = "msgpack-1.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f2cb069d8b981abc72b41aea1c580ce92d57c673ec61af4c500153a626cb9e20"}, + {file = "msgpack-1.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d62ce1f483f355f61adb5433ebfd8868c5f078d1a52d042b0a998682b4fa8c27"}, + {file = "msgpack-1.1.2-cp314-cp314t-win32.whl", hash = "sha256:1d1418482b1ee984625d88aa9585db570180c286d942da463533b238b98b812b"}, + {file = "msgpack-1.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:5a46bf7e831d09470ad92dff02b8b1ac92175ca36b087f904a0519857c6be3ff"}, + {file = "msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46"}, + {file = "msgpack-1.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ea5405c46e690122a76531ab97a079e184c0daf491e588592d6a23d3e32af99e"}, + {file = "msgpack-1.1.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9fba231af7a933400238cb357ecccf8ab5d51535ea95d94fc35b7806218ff844"}, + {file = "msgpack-1.1.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a8f6e7d30253714751aa0b0c84ae28948e852ee7fb0524082e6716769124bc23"}, + {file = "msgpack-1.1.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94fd7dc7d8cb0a54432f296f2246bc39474e017204ca6f4ff345941d4ed285a7"}, + {file = "msgpack-1.1.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:350ad5353a467d9e3b126d8d1b90fe05ad081e2e1cef5753f8c345217c37e7b8"}, + {file = "msgpack-1.1.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6bde749afe671dc44893f8d08e83bf475a1a14570d67c4bb5cec5573463c8833"}, + {file = "msgpack-1.1.2-cp39-cp39-win32.whl", hash = "sha256:ad09b984828d6b7bb52d1d1d0c9be68ad781fa004ca39216c8a1e63c0f34ba3c"}, + {file = "msgpack-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:67016ae8c8965124fdede9d3769528ad8284f14d635337ffa6a713a580f6c030"}, + {file = "msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e"}, +] + [[package]] name = "multidict" version = "6.7.1" @@ -1539,4 +1611,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "<4.0,>=3.10" -content-hash = "ad84316ae77727ff6b6501d7beb081c4a6098b1321c193c5b222d7acbea49918" +content-hash = "9a1e4762e038e0e8a65fd277a60a0a60d462019c1ce5233cb24d6159336c6288" diff --git a/node/tests/test_pnpm.py b/node/tests/test_pnpm.py index b902850d..42d5c513 100644 --- a/node/tests/test_pnpm.py +++ b/node/tests/test_pnpm.py @@ -93,10 +93,11 @@ def test_lockfile_v9(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9, store_version='v10') lockfile.path.write_text(TEST_LOCKFILE_V9) packages = list(provider.process_lockfile(lockfile.path)) @@ -146,10 +147,11 @@ def test_lockfile_v6(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 6) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 6, store_version='v3') lockfile.path.write_text(TEST_LOCKFILE_V6) packages = list(provider.process_lockfile(lockfile.path)) @@ -199,6 +201,7 @@ def test_lockfile_v6_no_devel(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=True, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -218,6 +221,7 @@ def test_lockfile_v5_rejected(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -237,6 +241,7 @@ def test_lockfile_unsupported_version_rejected(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -258,6 +263,7 @@ def test_lockfile_v9_no_devel_warns( PnpmLockfileProvider.Options( no_devel=True, registry='https://registry.npmjs.org', + store_version=None, ) ) @@ -306,10 +312,11 @@ def test_lockfile_v9_git_and_local(tmp_path: Path) -> None: PnpmLockfileProvider.Options( no_devel=False, registry='https://registry.npmjs.org', + store_version=None, ) ) - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9, store_version='v10') lockfile.path.write_text(TEST_LOCKFILE_V9_GIT_AND_LOCAL) packages = list(provider.process_lockfile(lockfile.path)) @@ -358,13 +365,13 @@ def test_pnpm_module_provider_tarball_url(tmp_path: Path) -> None: tarball_name='normal-pkg-1.0.0.tgz', name='normal-pkg', version='1.0.0', - integrity=Integrity('sha512', 'abc123def456'), + integrity=Integrity('sha512', 'abc123def456ab'), ), PnpmModuleProvider._TarballInfo( tarball_name='url-pkg-http-123.tgz', name='url-pkg', version='http://example.com/url-pkg.tgz', - integrity=Integrity('sha512', 'fedcba654'), + integrity=Integrity('sha512', 'fedcba65401234'), ), PnpmModuleProvider._TarballInfo( tarball_name='url-pkg-https-123.tgz', @@ -391,6 +398,10 @@ def test_pnpm_module_provider_tarball_url(tmp_path: Path) -> None: for tarball in provider._tarballs: pkg = packages[tarball.tarball_name] assert pkg['version'] == tarball.version + expected_integrity = ( + f'{tarball.integrity.algorithm}-{tarball.integrity.to_base64()}' + ) + assert f'{pkg["integrity_algo"]}-{pkg["integrity"]}' == expected_integrity if tarball.version.startswith(('http://', 'https://')): assert pkg['tarball_url'] == tarball.version else: @@ -418,9 +429,8 @@ async def test_pnpm_module_provider_missing_integrity( ) provider = PnpmModuleProvider(gen, special, tmp_path) - provider._store_version = 'v3' - lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9) + lockfile = Lockfile(tmp_path / 'pnpm-lock.yaml', 9, store_version='v3') test_data = b'dummy tarball content' test_digest = hashlib.sha256(test_data).hexdigest() diff --git a/node/tests/test_populate_pnpm_store.py b/node/tests/test_populate_pnpm_store.py index c9d0e940..caa5589c 100644 --- a/node/tests/test_populate_pnpm_store.py +++ b/node/tests/test_populate_pnpm_store.py @@ -1,10 +1,190 @@ import hashlib import json import re +import sqlite3 +import struct import tarfile from pathlib import Path -from flatpak_node_generator.populate_pnpm_store import _process_tarball +from flatpak_node_generator.integrity import Integrity +from flatpak_node_generator.populate_pnpm_store import ( + _pack_v11_store_entry, + _process_tarball, +) + + +def _decode_v11(data: bytes) -> dict[str, object]: + """Decode msgpackr record-extension encoding to plain dicts/lists. + + msgpackr with ``useRecords: true, moreTypes: true`` encodes: + + - plain objects as |d4 72 struct_id fixarray-of-keys values...| + - Maps as standard msgpack maps (0x80/0xde/0xdf) + + This decoder returns nested dicts (for both records and maps) so the + test assertions can use dictionary-style access. + """ + + def _read(offset: int) -> tuple[object, int]: + if offset >= len(data): + raise ValueError('unexpected end of data') + byte = data[offset] + offset += 1 + + # positive fixint + if byte <= 0x7F: + return byte, offset + # fixmap + if 0x80 <= byte <= 0x8F: + length = byte & 0x0F + result: dict[str, object] = {} + for _ in range(length): + k, offset = _read_str(offset) + v, offset = _read(offset) + result[k] = v + return result, offset + # fixarray + if 0x90 <= byte <= 0x9F: + length = byte & 0x0F + arr: list[object] = [] + for _ in range(length): + v, offset = _read(offset) + arr.append(v) + return arr, offset + # fixstr + if 0xA0 <= byte <= 0xBF: + length = byte & 0x1F + return data[offset : offset + length].decode('utf-8'), offset + length + # negative fixint + if 0xE0 <= byte <= 0xFF: + return byte - 256, offset + # nil + if byte == 0xC0: + return None, offset + # false / true + if byte == 0xC2: + return False, offset + if byte == 0xC3: + return True, offset + # bin 8/16/32 + if byte == 0xC4: + return data[offset : offset + data[offset]], offset + 1 + data[offset] + if byte == 0xC5: + length = int.from_bytes(data[offset : offset + 2], 'big') + return data[offset + 2 : offset + 2 + length], offset + 2 + length + if byte == 0xC6: + length = int.from_bytes(data[offset : offset + 4], 'big') + return data[offset + 4 : offset + 4 + length], offset + 4 + length + # float 32 + if byte == 0xCA: + return struct.unpack('>f', data[offset : offset + 4])[0], offset + 4 + # float 64 + if byte == 0xCB: + return struct.unpack('>d', data[offset : offset + 8])[0], offset + 8 + # uint 8/16/32/64 + if byte == 0xCC: + return data[offset], offset + 1 + if byte == 0xCD: + return int.from_bytes(data[offset : offset + 2], 'big'), offset + 2 + if byte == 0xCE: + return int.from_bytes(data[offset : offset + 4], 'big'), offset + 4 + if byte == 0xCF: + return int.from_bytes(data[offset : offset + 8], 'big'), offset + 8 + # int 8/16/32/64 + if byte == 0xD0: + return int.from_bytes( + data[offset : offset + 1], 'big', signed=True + ), offset + 1 + if byte == 0xD1: + return int.from_bytes( + data[offset : offset + 2], 'big', signed=True + ), offset + 2 + if byte == 0xD2: + return int.from_bytes( + data[offset : offset + 4], 'big', signed=True + ), offset + 4 + if byte == 0xD3: + return int.from_bytes( + data[offset : offset + 8], 'big', signed=True + ), offset + 8 + # fixext 4 with record extension + if byte == 0xD4 and data[offset] == 0x72: + offset += 1 # skip ext type 0x72 + offset += 1 # skip struct_id byte + keys, offset = _read(offset) + assert isinstance(keys, list), f'expected array of keys, got {type(keys)}' + record: dict[str, object] = {} + for k in keys: + v, offset = _read(offset) + record[str(k)] = v + return record, offset + # str 8/16/32 + if byte == 0xD9: + length = data[offset] + return data[offset + 1 : offset + 1 + length].decode( + 'utf-8' + ), offset + 1 + length + if byte == 0xDA: + length = int.from_bytes(data[offset : offset + 2], 'big') + return data[offset + 2 : offset + 2 + length].decode( + 'utf-8' + ), offset + 2 + length + if byte == 0xDB: + length = int.from_bytes(data[offset : offset + 4], 'big') + return data[offset + 4 : offset + 4 + length].decode( + 'utf-8' + ), offset + 4 + length + # map 16/32 + if byte == 0xDE: + length = int.from_bytes(data[offset : offset + 2], 'big') + offset += 2 + m: dict[str, object] = {} + for _ in range(length): + k, offset = _read_str(offset) + v, offset = _read(offset) + m[k] = v + return m, offset + if byte == 0xDF: + length = int.from_bytes(data[offset : offset + 4], 'big') + offset += 4 + m2: dict[str, object] = {} + for _ in range(length): + k, offset = _read_str(offset) + v, offset = _read(offset) + m2[k] = v + return m2, offset + # array 16/32 + if byte == 0xDC: + length = int.from_bytes(data[offset : offset + 2], 'big') + offset += 2 + arr2: list[object] = [] + for _ in range(length): + v, offset = _read(offset) + arr2.append(v) + return arr2, offset + if byte == 0xDD: + length = int.from_bytes(data[offset : offset + 4], 'big') + offset += 4 + arr3: list[object] = [] + for _ in range(length): + v, offset = _read(offset) + arr3.append(v) + return arr3, offset + + raise ValueError( + f'unsupported msgpack byte: 0x{byte:02X} at offset {offset - 1}' + ) + + def _read_str(offset: int) -> tuple[str, int]: + val, offset = _read(offset) + assert isinstance(val, str), f'expected str, got {type(val)}' + return val, offset + + result, offset = _read(0) + if offset != len(data): + raise ValueError(f'extra bytes at end: {offset} < {len(data)}') + assert isinstance(result, dict) + return result def _create_tarball(path: Path, files: dict[str, str | bytes]) -> None: @@ -33,11 +213,17 @@ def test_process_tarball_normal(tmp_path: Path) -> None: {'package/package.json': pkg_json, 'package/index.js': "console.log('hello');"}, ) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='fallback-pkg', pkg_version='0.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, ) @@ -61,11 +247,17 @@ def test_process_tarball_malformed_package_json(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/package.json': '{ malformed: json '}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='fallback-pkg', pkg_version='0.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, ) @@ -87,11 +279,17 @@ def test_process_tarball_with_tarball_url_v3(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -114,11 +312,17 @@ def test_process_tarball_with_tarball_url_v6(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -138,11 +342,17 @@ def test_process_tarball_with_uppercase_path(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -163,11 +373,17 @@ def test_process_tarball_with_long_path(tmp_path: Path) -> None: _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + _process_tarball( tarball_path=str(tar_path), pkg_name='pkg', pkg_version='1.0.0', - integrity_hex='a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, store=str(store_dir), now=1234567890, tarball_url=tarball_url, @@ -179,3 +395,230 @@ def test_process_tarball_with_long_path(tmp_path: Path) -> None: url_idx_file = store_dir / normalized_tarball_url / 'integrity.json' assert url_idx_file.exists() + + +def test_process_tarball_v11(tmp_path: Path) -> None: + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' / 'v11' + pkg_json = json.dumps({'name': 'real-pkg', 'version': '1.2.3'}) + + _create_tarball( + tar_path, + {'package/package.json': pkg_json, 'package/index.js': "console.log('hello');"}, + ) + + store_dir.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(str(store_dir / 'index.db')) + db.execute('PRAGMA busy_timeout=5000') + db.execute('PRAGMA journal_mode=WAL') + db.execute('PRAGMA synchronous=NORMAL') + db.execute('PRAGMA temp_store=MEMORY') + db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + try: + _process_tarball( + tarball_path=str(tar_path), + pkg_name='fallback-pkg', + pkg_version='0.0.0', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, + store=str(store_dir), + now=1234567890, + store_version='v11', + index_db=db, + ) + db.commit() + + pkg_id = 'fallback-pkg@0.0.0' + expected_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + + row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (expected_key,) + ).fetchone() + assert row is not None, f'No row found for key {expected_key}' + + data = _decode_v11(row[0]) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert data['manifest'] == {'name': 'real-pkg', 'version': '1.2.3'} + assert isinstance(data['files'], dict) + assert 'package.json' in data['files'] + assert 'index.js' in data['files'] + finally: + db.close() + + +def test_process_tarball_v11_no_package_json(tmp_path: Path) -> None: + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' / 'v11' + + _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + + store_dir.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(str(store_dir / 'index.db')) + db.execute('PRAGMA busy_timeout=5000') + db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + try: + _process_tarball( + tarball_path=str(tar_path), + pkg_name='no-manifest-pkg', + pkg_version='2.0.0', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, + store=str(store_dir), + now=1234567890, + store_version='v11', + index_db=db, + ) + db.commit() + + pkg_id = 'no-manifest-pkg@2.0.0' + expected_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + + row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (expected_key,) + ).fetchone() + assert row is not None + + data = _decode_v11(row[0]) + # Fallback name/version used since there's no real package.json + assert data['manifest'] == { + 'name': 'no-manifest-pkg', + 'version': '2.0.0', + } + finally: + db.close() + + +def test_process_tarball_v11_with_tarball_url(tmp_path: Path) -> None: + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' / 'v11' + tarball_url = 'https://example.com/pkg.tgz' + + _create_tarball(tar_path, {'package/index.js': "console.log('hello');"}) + + store_dir.mkdir(parents=True, exist_ok=True) + db = sqlite3.connect(str(store_dir / 'index.db')) + db.execute('PRAGMA busy_timeout=5000') + db.execute( + 'CREATE TABLE IF NOT EXISTS package_index (' + ' key TEXT PRIMARY KEY,' + ' data BLOB NOT NULL' + ') WITHOUT ROWID' + ) + + integrity = Integrity( + 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + try: + _process_tarball( + tarball_path=str(tar_path), + pkg_name='pkg', + pkg_version='1.0.0', + integrity=integrity.to_base64(), + integrity_digest=integrity.digest, + integrity_algo=integrity.algorithm, + store=str(store_dir), + now=1234567890, + store_version='v11', + index_db=db, + tarball_url=tarball_url, + ) + db.commit() + + # Check main integrity-based entry exists + pkg_id = 'pkg@1.0.0' + main_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + main_row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (main_key,) + ).fetchone() + assert main_row is not None + + # Check tarball_url-based entry exists + url_key = f'{tarball_url}\t{pkg_id}' + url_row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (url_key,) + ).fetchone() + assert url_row is not None + + # Check pkgId-based entry exists ({pkgId}\tbuilt — git-hosted tarball format) + pkgid_key = f'{tarball_url}\tbuilt' + pkgid_row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (pkgid_key,) + ).fetchone() + assert pkgid_row is not None + finally: + db.close() + + +def test_pack_v11_store_entry() -> None: + """Verify _pack_v11_store_entry produces msgpackr-compatible record encoding.""" + files = { + 'index.js': { + 'checkedAt': 1234567890, + 'digest': 'abc123', + 'mode': 420, + 'size': 100, + }, + 'package.json': { + 'checkedAt': 111111, + 'digest': 'def456', + 'mode': 384, + 'size': 200, + }, + } + manifest = {'name': 'test-pkg', 'version': '1.0.0'} + packed = _pack_v11_store_entry(files, manifest) + + data = _decode_v11(packed) + assert isinstance(data, dict) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert isinstance(data['files'], dict) + assert data['files']['index.js']['digest'] == 'abc123' + assert data['files']['index.js']['mode'] == 420 + assert data['files']['index.js']['size'] == 100 + assert data['files']['index.js']['checkedAt'] == 1234567890 + assert data['files']['package.json']['digest'] == 'def456' + assert data['manifest'] == manifest + + +def test_pack_v11_store_entry_no_manifest() -> None: + """Verify _pack_v11_store_entry works without a manifest.""" + files = { + 'index.js': {'checkedAt': 123, 'digest': 'hex123', 'mode': 420, 'size': 50} + } + packed = _pack_v11_store_entry(files, None) + + data = _decode_v11(packed) + assert isinstance(data, dict) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert 'manifest' not in data + files_obj = data['files'] + assert isinstance(files_obj, dict) + index_js = files_obj['index.js'] + assert isinstance(index_js, dict) + assert index_js['digest'] == 'hex123' From a268a1702574816980acfe4b97429008372ee9a6 Mon Sep 17 00:00:00 2001 From: bbhtt Date: Tue, 19 May 2026 09:40:45 +0530 Subject: [PATCH 2/5] node: Fix store_version mutatin --- node/flatpak_node_generator/providers/pnpm.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/node/flatpak_node_generator/providers/pnpm.py b/node/flatpak_node_generator/providers/pnpm.py index f27c5b52..9e6cd8e3 100644 --- a/node/flatpak_node_generator/providers/pnpm.py +++ b/node/flatpak_node_generator/providers/pnpm.py @@ -98,9 +98,10 @@ def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: ) supported_store_versions = _STORE_VERSION_BY_LOCKFILE[major] - if self.store_version is None: - self.store_version = supported_store_versions[0] - elif self.store_version not in supported_store_versions: + store_version = self.store_version + if store_version is None: + store_version = supported_store_versions[0] + elif store_version not in supported_store_versions: supported = ', '.join(str(v) for v in sorted(supported_store_versions)) raise ValueError( f"{lockfile_path}: lockfileVersion {raw_version} doesn't support store version {self.store_version}. " @@ -114,7 +115,7 @@ def process_lockfile(self, lockfile_path: Path) -> Iterator[Package]: file=sys.stderr, ) - lockfile = Lockfile(lockfile_path, major, store_version=self.store_version) + lockfile = Lockfile(lockfile_path, major, store_version=store_version) packages_dict: dict[str, Any] = data.get('packages', {}) if not packages_dict: @@ -197,11 +198,18 @@ def __exit__( async def generate_package(self, package: Package) -> None: source = package.source + sv = package.lockfile.store_version + if sv is None: + raise TypeError( + f'{package.name}@{package.version}: lockfile provides no store version' + ) if self._store_version is None: - sv = package.lockfile.store_version - if sv is None: - raise TypeError('pnpm expects lockfile provides store version') self._store_version = sv + elif self._store_version != sv: + raise ValueError( + f'{package.name}@{package.version}: store version mismatch: ' + f'expected {self._store_version!r}, got {sv!r}' + ) if isinstance(source, ResolvedSource): assert source.resolved is not None From b68b0fd303bde59bd290ec9e9a756278fdb0e9b9 Mon Sep 17 00:00:00 2001 From: bbhtt Date: Tue, 19 May 2026 09:43:41 +0530 Subject: [PATCH 3/5] Update lockfile --- node/poetry.lock | 76 ++---------------------------------------------- 1 file changed, 2 insertions(+), 74 deletions(-) diff --git a/node/poetry.lock b/node/poetry.lock index 4fc210f0..2e3ecab9 100644 --- a/node/poetry.lock +++ b/node/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -623,78 +623,6 @@ files = [ {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, ] -[[package]] -name = "msgpack" -version = "1.1.2" -description = "MessagePack serializer" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2"}, - {file = "msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87"}, - {file = "msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251"}, - {file = "msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a"}, - {file = "msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f"}, - {file = "msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f"}, - {file = "msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9"}, - {file = "msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa"}, - {file = "msgpack-1.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e86a607e558d22985d856948c12a3fa7b42efad264dca8a3ebbcfa2735d786c"}, - {file = "msgpack-1.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:283ae72fc89da59aa004ba147e8fc2f766647b1251500182fac0350d8af299c0"}, - {file = "msgpack-1.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61c8aa3bd513d87c72ed0b37b53dd5c5a0f58f2ff9f26e1555d3bd7948fb7296"}, - {file = "msgpack-1.1.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:454e29e186285d2ebe65be34629fa0e8605202c60fbc7c4c650ccd41870896ef"}, - {file = "msgpack-1.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7bc8813f88417599564fafa59fd6f95be417179f76b40325b500b3c98409757c"}, - {file = "msgpack-1.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bafca952dc13907bdfdedfc6a5f579bf4f292bdd506fadb38389afa3ac5b208e"}, - {file = "msgpack-1.1.2-cp311-cp311-win32.whl", hash = "sha256:602b6740e95ffc55bfb078172d279de3773d7b7db1f703b2f1323566b878b90e"}, - {file = "msgpack-1.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:d198d275222dc54244bf3327eb8cbe00307d220241d9cec4d306d49a44e85f68"}, - {file = "msgpack-1.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:86f8136dfa5c116365a8a651a7d7484b65b13339731dd6faebb9a0242151c406"}, - {file = "msgpack-1.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:70a0dff9d1f8da25179ffcf880e10cf1aad55fdb63cd59c9a49a1b82290062aa"}, - {file = "msgpack-1.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:446abdd8b94b55c800ac34b102dffd2f6aa0ce643c55dfc017ad89347db3dbdb"}, - {file = "msgpack-1.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c63eea553c69ab05b6747901b97d620bb2a690633c77f23feb0c6a947a8a7b8f"}, - {file = "msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:372839311ccf6bdaf39b00b61288e0557916c3729529b301c52c2d88842add42"}, - {file = "msgpack-1.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2929af52106ca73fcb28576218476ffbb531a036c2adbcf54a3664de124303e9"}, - {file = "msgpack-1.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:be52a8fc79e45b0364210eef5234a7cf8d330836d0a64dfbb878efa903d84620"}, - {file = "msgpack-1.1.2-cp312-cp312-win32.whl", hash = "sha256:1fff3d825d7859ac888b0fbda39a42d59193543920eda9d9bea44d958a878029"}, - {file = "msgpack-1.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1de460f0403172cff81169a30b9a92b260cb809c4cb7e2fc79ae8d0510c78b6b"}, - {file = "msgpack-1.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:be5980f3ee0e6bd44f3a9e9dea01054f175b50c3e6cdb692bc9424c0bbb8bf69"}, - {file = "msgpack-1.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4efd7b5979ccb539c221a4c4e16aac1a533efc97f3b759bb5a5ac9f6d10383bf"}, - {file = "msgpack-1.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42eefe2c3e2af97ed470eec850facbe1b5ad1d6eacdbadc42ec98e7dcf68b4b7"}, - {file = "msgpack-1.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fdf7d83102bf09e7ce3357de96c59b627395352a4024f6e2458501f158bf999"}, - {file = "msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fac4be746328f90caa3cd4bc67e6fe36ca2bf61d5c6eb6d895b6527e3f05071e"}, - {file = "msgpack-1.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fffee09044073e69f2bad787071aeec727183e7580443dfeb8556cbf1978d162"}, - {file = "msgpack-1.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5928604de9b032bc17f5099496417f113c45bc6bc21b5c6920caf34b3c428794"}, - {file = "msgpack-1.1.2-cp313-cp313-win32.whl", hash = "sha256:a7787d353595c7c7e145e2331abf8b7ff1e6673a6b974ded96e6d4ec09f00c8c"}, - {file = "msgpack-1.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:a465f0dceb8e13a487e54c07d04ae3ba131c7c5b95e2612596eafde1dccf64a9"}, - {file = "msgpack-1.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:e69b39f8c0aa5ec24b57737ebee40be647035158f14ed4b40e6f150077e21a84"}, - {file = "msgpack-1.1.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e23ce8d5f7aa6ea6d2a2b326b4ba46c985dbb204523759984430db7114f8aa00"}, - {file = "msgpack-1.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c15b7d74c939ebe620dd8e559384be806204d73b4f9356320632d783d1f7939"}, - {file = "msgpack-1.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99e2cb7b9031568a2a5c73aa077180f93dd2e95b4f8d3b8e14a73ae94a9e667e"}, - {file = "msgpack-1.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:180759d89a057eab503cf62eeec0aa61c4ea1200dee709f3a8e9397dbb3b6931"}, - {file = "msgpack-1.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:04fb995247a6e83830b62f0b07bf36540c213f6eac8e851166d8d86d83cbd014"}, - {file = "msgpack-1.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8e22ab046fa7ede9e36eeb4cfad44d46450f37bb05d5ec482b02868f451c95e2"}, - {file = "msgpack-1.1.2-cp314-cp314-win32.whl", hash = "sha256:80a0ff7d4abf5fecb995fcf235d4064b9a9a8a40a3ab80999e6ac1e30b702717"}, - {file = "msgpack-1.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:9ade919fac6a3e7260b7f64cea89df6bec59104987cbea34d34a2fa15d74310b"}, - {file = "msgpack-1.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:59415c6076b1e30e563eb732e23b994a61c159cec44deaf584e5cc1dd662f2af"}, - {file = "msgpack-1.1.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:897c478140877e5307760b0ea66e0932738879e7aa68144d9b78ea4c8302a84a"}, - {file = "msgpack-1.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a668204fa43e6d02f89dbe79a30b0d67238d9ec4c5bd8a940fc3a004a47b721b"}, - {file = "msgpack-1.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5559d03930d3aa0f3aacb4c42c776af1a2ace2611871c84a75afe436695e6245"}, - {file = "msgpack-1.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70c5a7a9fea7f036b716191c29047374c10721c389c21e9ffafad04df8c52c90"}, - {file = "msgpack-1.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f2cb069d8b981abc72b41aea1c580ce92d57c673ec61af4c500153a626cb9e20"}, - {file = "msgpack-1.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d62ce1f483f355f61adb5433ebfd8868c5f078d1a52d042b0a998682b4fa8c27"}, - {file = "msgpack-1.1.2-cp314-cp314t-win32.whl", hash = "sha256:1d1418482b1ee984625d88aa9585db570180c286d942da463533b238b98b812b"}, - {file = "msgpack-1.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:5a46bf7e831d09470ad92dff02b8b1ac92175ca36b087f904a0519857c6be3ff"}, - {file = "msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46"}, - {file = "msgpack-1.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ea5405c46e690122a76531ab97a079e184c0daf491e588592d6a23d3e32af99e"}, - {file = "msgpack-1.1.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9fba231af7a933400238cb357ecccf8ab5d51535ea95d94fc35b7806218ff844"}, - {file = "msgpack-1.1.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a8f6e7d30253714751aa0b0c84ae28948e852ee7fb0524082e6716769124bc23"}, - {file = "msgpack-1.1.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94fd7dc7d8cb0a54432f296f2246bc39474e017204ca6f4ff345941d4ed285a7"}, - {file = "msgpack-1.1.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:350ad5353a467d9e3b126d8d1b90fe05ad081e2e1cef5753f8c345217c37e7b8"}, - {file = "msgpack-1.1.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6bde749afe671dc44893f8d08e83bf475a1a14570d67c4bb5cec5573463c8833"}, - {file = "msgpack-1.1.2-cp39-cp39-win32.whl", hash = "sha256:ad09b984828d6b7bb52d1d1d0c9be68ad781fa004ca39216c8a1e63c0f34ba3c"}, - {file = "msgpack-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:67016ae8c8965124fdede9d3769528ad8284f14d635337ffa6a713a580f6c030"}, - {file = "msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e"}, -] - [[package]] name = "multidict" version = "6.7.1" @@ -1611,4 +1539,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "<4.0,>=3.10" -content-hash = "9a1e4762e038e0e8a65fd277a60a0a60d462019c1ce5233cb24d6159336c6288" +content-hash = "ad84316ae77727ff6b6501d7beb081c4a6098b1321c193c5b222d7acbea49918" From f8fb302d8e35e030bcfdbe6442061c2a97512019 Mon Sep 17 00:00:00 2001 From: LuoRain Date: Tue, 19 May 2026 12:40:40 +0800 Subject: [PATCH 4/5] node: fix hardcoded algorithm --- node/flatpak_node_generator/populate_pnpm_store.py | 8 +++++--- node/tests/test_populate_pnpm_store.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/node/flatpak_node_generator/populate_pnpm_store.py b/node/flatpak_node_generator/populate_pnpm_store.py index 3e7d689b..1603be38 100644 --- a/node/flatpak_node_generator/populate_pnpm_store.py +++ b/node/flatpak_node_generator/populate_pnpm_store.py @@ -98,6 +98,7 @@ def _pack_str(val: str) -> bytes: def _pack_v11_store_entry( + algo: str, files: dict[str, dict[str, object]], manifest: dict[str, str] | None = None, ) -> bytes: @@ -139,8 +140,9 @@ def _fixmap(length: int) -> bytes: if manifest is not None: store_entry_keys.append('manifest') - result = b'\xd4\x72\x40' + _msgpack_pack(store_entry_keys) - result += _msgpack_pack('sha512') # algo + # Record with fixed entries + result = RECORD_HEADER + b'\x40' + _msgpack_pack(store_entry_keys) + result += _msgpack_pack(algo) # algo result += _msgpack_pack(False) # requiresBuild result += files_map_bytes # files (standard map → iterable Map in JS) @@ -293,7 +295,7 @@ def _process_tarball( 'version': real_pkg_version, } - entry_bytes = _pack_v11_store_entry(v11_files, manifest) + entry_bytes = _pack_v11_store_entry(integrity_algo, v11_files, manifest) # It's currently not possible to fully determine which store key pnpm will use, # so we insert multiple keys to ensure pnpm can find the entry it wants. diff --git a/node/tests/test_populate_pnpm_store.py b/node/tests/test_populate_pnpm_store.py index caa5589c..5bdda92b 100644 --- a/node/tests/test_populate_pnpm_store.py +++ b/node/tests/test_populate_pnpm_store.py @@ -421,7 +421,7 @@ def test_process_tarball_v11(tmp_path: Path) -> None: ) integrity = Integrity( - 'sha256', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + 'sha512', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' ) try: @@ -590,7 +590,7 @@ def test_pack_v11_store_entry() -> None: }, } manifest = {'name': 'test-pkg', 'version': '1.0.0'} - packed = _pack_v11_store_entry(files, manifest) + packed = _pack_v11_store_entry('sha512', files, manifest) data = _decode_v11(packed) assert isinstance(data, dict) @@ -610,7 +610,7 @@ def test_pack_v11_store_entry_no_manifest() -> None: files = { 'index.js': {'checkedAt': 123, 'digest': 'hex123', 'mode': 420, 'size': 50} } - packed = _pack_v11_store_entry(files, None) + packed = _pack_v11_store_entry('sha512', files, None) data = _decode_v11(packed) assert isinstance(data, dict) From 42f11d5d01f16cebd3e66470cce69e018119d40f Mon Sep 17 00:00:00 2001 From: LuoRain Date: Tue, 19 May 2026 13:04:03 +0800 Subject: [PATCH 5/5] node: add populate store integration test for pnpm v11 --- node/tests/test_populate_pnpm_store.py | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/node/tests/test_populate_pnpm_store.py b/node/tests/test_populate_pnpm_store.py index 5bdda92b..f00b1df0 100644 --- a/node/tests/test_populate_pnpm_store.py +++ b/node/tests/test_populate_pnpm_store.py @@ -10,6 +10,7 @@ from flatpak_node_generator.populate_pnpm_store import ( _pack_v11_store_entry, _process_tarball, + populate_store, ) @@ -397,6 +398,60 @@ def test_process_tarball_with_long_path(tmp_path: Path) -> None: assert url_idx_file.exists() +def test_populate_store_v11(tmp_path: Path) -> None: + manifest_path = tmp_path / 'manifest.json' + tar_path = tmp_path / 'pkg.tgz' + store_dir = tmp_path / 'store' + pkg_json = json.dumps({'name': 'real-pkg', 'version': '1.2.3'}) + + integrity = Integrity( + 'sha512', 'a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2' + ) + + _create_tarball( + tar_path, + {'package/package.json': pkg_json, 'package/index.js': "console.log('hello');"}, + ) + + manifest_json = json.dumps( + { + 'store_version': 'v11', + 'packages': { + 'pkg.tgz': { + 'name': 'real-pkg', + 'version': '1.2.3', + 'integrity': integrity.to_base64(), + 'integrity_digest': integrity.digest, + 'integrity_algo': integrity.algorithm, + } + }, + } + ) + + with open(manifest_path, 'w', encoding='utf-8') as f: + f.write(manifest_json) + + populate_store(str(manifest_path), str(tmp_path), str(store_dir)) + + db = sqlite3.connect(str(store_dir / 'v11' / 'index.db')) + + pkg_id = 'real-pkg@1.2.3' + expected_key = f'{integrity.algorithm}-{integrity.to_base64()}\t{pkg_id}' + + row = db.execute( + 'SELECT data FROM package_index WHERE key = ?', (expected_key,) + ).fetchone() + assert row is not None, f'No row found for key {expected_key}' + + data = _decode_v11(row[0]) + assert data['algo'] == 'sha512' + assert data['requiresBuild'] is False + assert data['manifest'] == {'name': 'real-pkg', 'version': '1.2.3'} + assert isinstance(data['files'], dict) + assert 'package.json' in data['files'] + assert 'index.js' in data['files'] + + def test_process_tarball_v11(tmp_path: Path) -> None: tar_path = tmp_path / 'pkg.tgz' store_dir = tmp_path / 'store' / 'v11'