Stream typemap fingerprints and PE output

src/Microsoft.Android.Sdk.TrimmableTypeMap/Generator/BlobBuilderStream.cs

@@ -0,0 +1,147 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection.Metadata;
+
+namespace Microsoft.Android.Sdk.TrimmableTypeMap;
+
+/// <summary>
+/// A read-only, seekable <see cref="Stream"/> over the chunks a <see cref="BlobBuilder"/> already
+/// holds, so a serialised PE image can be hashed and copied to disk without being duplicated into
+/// a second contiguous buffer.
+/// </summary>
+/// <remarks>
+/// Only the chunk arrays are retained — the <see cref="BlobBuilder"/> itself and the metadata
+/// graph that produced it stay collectible — so the live byte count matches what a
+/// <see cref="MemoryStream"/> copy would have held, without the transient second copy.
+/// </remarks>
+sealed class BlobBuilderStream : Stream
+{
+	readonly ArraySegment<byte> [] segments;
+	readonly long [] segmentStarts;
+	readonly long length;
+	long position;
+	int cursor;
+
+	public BlobBuilderStream (BlobBuilder builder)
+	{
+		_ = builder ?? throw new ArgumentNullException (nameof (builder));
+
+		var collected = new List<ArraySegment<byte>> ();
+		foreach (var blob in builder.GetBlobs ()) {
+			var bytes = blob.GetBytes ();
+			if (bytes.Count == 0 || bytes.Array is null) {
+				continue;
+			}
+			collected.Add (bytes);
+		}
+
+		segments = collected.ToArray ();
+		segmentStarts = new long [segments.Length + 1];
+		long total = 0;
+		for (int i = 0; i < segments.Length; i++) {
+			segmentStarts [i] = total;
+			total += segments [i].Count;
+		}
+		segmentStarts [segments.Length] = total;
+		length = total;
+	}
+
+	public override bool CanRead => true;
+
+	public override bool CanSeek => true;
+
+	public override bool CanWrite => false;
+
+	public override long Length => length;
+
+	public override long Position {
+		get => position;
+		set {
+			if (value < 0) {
+				throw new ArgumentOutOfRangeException (nameof (value));
+			}
+			position = value;
+		}
+	}
+
+	public override void Flush ()
+	{
+	}
+
+	public override int Read (byte [] buffer, int offset, int count)
+	{
+		if (buffer is null) {
+			throw new ArgumentNullException (nameof (buffer));
+		}
+		if (offset < 0) {
+			throw new ArgumentOutOfRangeException (nameof (offset));
+		}
+		if (count < 0) {
+			throw new ArgumentOutOfRangeException (nameof (count));
+		}
+		if (buffer.Length - offset < count) {
+			throw new ArgumentException ("The buffer is too small for the requested range.");
+		}
+
+		int copied = 0;
+		while (count > 0 && position < length) {
+			int index = FindSegment (position);
+			var segment = segments [index];
+			int within = (int) (position - segmentStarts [index]);
+			int available = segment.Count - within;
+			int toCopy = Math.Min (available, count);
+			if (segment.Array is null) {
+				break;
+			}
+			Buffer.BlockCopy (segment.Array, segment.Offset + within, buffer, offset, toCopy);
+			position += toCopy;
+			offset += toCopy;
+			count -= toCopy;
+			copied += toCopy;
+		}
+		return copied;
+	}
+
+	public override long Seek (long offset, SeekOrigin origin)
+	{
+		long target = origin switch {
+			SeekOrigin.Begin => offset,
+			SeekOrigin.Current => position + offset,
+			SeekOrigin.End => length + offset,
+			_ => throw new ArgumentOutOfRangeException (nameof (origin)),
+		};
+		if (target < 0) {
+			throw new IOException ("Cannot seek before the beginning of the stream.");
+		}
+		position = target;
+		return position;
+	}
+
+	public override void SetLength (long value) => throw new NotSupportedException ();
+
+	public override void Write (byte [] buffer, int offset, int count) => throw new NotSupportedException ();
+
+	int FindSegment (long offset)
+	{
+		// Reads are overwhelmingly sequential, so try the last used chunk first.
+		if (cursor < segments.Length && offset >= segmentStarts [cursor] && offset < segmentStarts [cursor + 1]) {
+			return cursor;
+		}
+
+		int low = 0;
+		int high = segments.Length - 1;
+		while (low <= high) {
+			int middle = low + ((high - low) / 2);
+			if (offset < segmentStarts [middle]) {
+				high = middle - 1;
+			} else if (offset >= segmentStarts [middle + 1]) {
+				low = middle + 1;
+			} else {
+				cursor = middle;
+				return middle;
+			}
+		}
+		throw new ArgumentOutOfRangeException (nameof (offset));
+	}
+}

src/Microsoft.Android.Sdk.TrimmableTypeMap/Generator/FingerprintWriter.cs

@@ -0,0 +1,183 @@
+using System;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace Microsoft.Android.Sdk.TrimmableTypeMap;
+
+/// <summary>
+/// Streams model fields into one or two SHA-256 hashes without materialising the whole
+/// serialised model in memory.
+/// </summary>
+/// <remarks>
+/// <para>
+/// The byte stream produced for each sink is identical to what
+/// <see cref="System.IO.BinaryWriter"/> would have written into a <see cref="System.IO.MemoryStream"/>:
+/// strings are UTF-8 encoded with a 7-bit encoded byte-length prefix, <see cref="bool"/> is one
+/// byte, and <see cref="int"/> is four bytes little-endian.  Keeping the encoding identical means
+/// the resulting fingerprints — and therefore the deterministic MVIDs derived from them — are
+/// unchanged.
+/// </para>
+/// <para>
+/// Two sinks are supported so the content fingerprint (which seeds the MVID) and the
+/// incremental-build fingerprint can be produced from a single walk over the model. Fields shared
+/// by both fingerprints are UTF-8 encoded once and appended to both sinks; fields belonging to
+/// only one fingerprint are appended to that sink alone.
+/// </para>
+/// </remarks>
+sealed class FingerprintWriter : IDisposable
+{
+	/// <summary>Selects which fingerprint(s) a write applies to.</summary>
+	[Flags]
+	public enum Sink
+	{
+		Content = 1,
+		Incremental = 2,
+		Both = Content | Incremental,
+	}
+
+	// Large enough to absorb the small field writes that dominate the model walk without
+	// paying per-field hash update costs, small enough to stay off the large object heap.
+	const int BufferSize = 8 * 1024;
+
+	readonly IncrementalHash contentHash;
+	readonly IncrementalHash? incrementalHash;
+	readonly byte [] contentBuffer;
+	readonly byte []? incrementalBuffer;
+	byte [] scratch = new byte [512];
+	int contentPosition;
+	int incrementalPosition;
+
+	public FingerprintWriter (bool includeIncremental)
+	{
+		contentHash = IncrementalHash.CreateHash (HashAlgorithmName.SHA256);
+		contentBuffer = new byte [BufferSize];
+		if (includeIncremental) {
+			incrementalHash = IncrementalHash.CreateHash (HashAlgorithmName.SHA256);
+			incrementalBuffer = new byte [BufferSize];
+		}
+	}
+
+	public void WriteString (Sink sink, string value)
+	{
+		int byteCount = Encoding.UTF8.GetByteCount (value);
+		EnsureScratch (byteCount + 5);
+		int offset = Write7BitEncodedInt (scratch, 0, byteCount);
+		Encoding.UTF8.GetBytes (value, 0, value.Length, scratch, offset);
+		Write (sink, scratch, 0, offset + byteCount);
+	}
+
+	public void WriteOptionalString (Sink sink, string? value)
+	{
+		WriteBoolean (sink, value is not null);
+		if (value is not null) {
+			WriteString (sink, value);
+		}
+	}
+
+	public void WriteBoolean (Sink sink, bool value) => WriteByte (sink, value ? (byte) 1 : (byte) 0);
+
+	public void WriteByte (Sink sink, byte value)
+	{
+		scratch [0] = value;
+		Write (sink, scratch, 0, 1);
+	}
+
+	public void WriteInt32 (Sink sink, int value)
+	{
+		scratch [0] = (byte) value;
+		scratch [1] = (byte) (value >> 8);
+		scratch [2] = (byte) (value >> 16);
+		scratch [3] = (byte) (value >> 24);
+		Write (sink, scratch, 0, 4);
+	}
+
+	public void WriteRaw (Sink sink, byte [] value) => Write (sink, value, 0, value.Length);
+
+	public byte [] GetContentFingerprint ()
+	{
+		FlushContent ();
+		return contentHash.GetHashAndReset ();
+	}
+
+	public byte [] GetIncrementalFingerprint ()
+	{
+		if (incrementalHash is null) {
+			throw new InvalidOperationException ("The incremental fingerprint was not requested.");
+		}
+		FlushIncremental ();
+		return incrementalHash.GetHashAndReset ();
+	}
+
+	public void Dispose ()
+	{
+		contentHash.Dispose ();
+		incrementalHash?.Dispose ();
+	}
+
+	void EnsureScratch (int required)
+	{
+		if (scratch.Length < required) {
+			scratch = new byte [Math.Max (required, scratch.Length * 2)];
+		}
+	}
+
+	void Write (Sink sink, byte [] data, int offset, int count)
+	{
+		if ((sink & Sink.Content) != 0) {
+			if (count > contentBuffer.Length - contentPosition) {
+				FlushContent ();
+			}
+			if (count > contentBuffer.Length) {
+				contentHash.AppendData (data, offset, count);
+			} else {
+				Buffer.BlockCopy (data, offset, contentBuffer, contentPosition, count);
+				contentPosition += count;
+			}
+		}
+		if ((sink & Sink.Incremental) == 0) {
+			return;
+		}
+		if (incrementalHash is null || incrementalBuffer is null) {
+			// Silently dropping the write would produce a fingerprint that looks valid but
+			// covers less than it claims to, so fail fast on the caller's mistake instead.
+			throw new InvalidOperationException (
+				$"Cannot write to {nameof (Sink.Incremental)} because the incremental fingerprint was not requested.");
+		}
+		if (count > incrementalBuffer.Length - incrementalPosition) {
+			FlushIncremental ();
+		}
+		if (count > incrementalBuffer.Length) {
+			incrementalHash.AppendData (data, offset, count);
+		} else {
+			Buffer.BlockCopy (data, offset, incrementalBuffer, incrementalPosition, count);
+			incrementalPosition += count;
+		}
+	}
+
+	void FlushContent ()
+	{
+		if (contentPosition > 0) {
+			contentHash.AppendData (contentBuffer, 0, contentPosition);
+			contentPosition = 0;
+		}
+	}
+
+	void FlushIncremental ()
+	{
+		if (incrementalPosition > 0 && incrementalHash is not null && incrementalBuffer is not null) {
+			incrementalHash.AppendData (incrementalBuffer, 0, incrementalPosition);
+			incrementalPosition = 0;
+		}
+	}
+
+	static int Write7BitEncodedInt (byte [] destination, int offset, int value)
+	{
+		uint remaining = (uint) value;
+		while (remaining > 0x7Fu) {
+			destination [offset++] = (byte) (remaining | ~0x7Fu);
+			remaining >>= 7;
+		}
+		destination [offset++] = (byte) remaining;
+		return offset;
+	}
+}