Percent-encode literal "%" when turning a JSON Pointer into a URI fragment

src/core/jsonpointer/jsonpointer.cc

@@ -436,7 +436,12 @@ auto to_uri(const Pointer &pointer) -> URI {
                            std::allocator<JSON::Char>>
       result;
   stringify<JSON::Char, JSON::CharTraits, std::allocator>(pointer, result);
-  return URI::from_fragment(result.str());
+  // RFC 6901 Section 6: "A JSON Pointer can be represented in a URI fragment
+  // identifier by encoding it into octets using UTF-8, while percent-encoding
+  // those characters not allowed by the fragment rule in [RFC3986]". The
+  // stringified pointer is the literal text to encode, so a token that already
+  // reads as an escape sequence must not be taken as one
+  return URI::from_unescaped_fragment(result.str());
 }
 
 auto to_uri(const Pointer &pointer, const URI &base) -> URI {
@@ -448,7 +453,7 @@ auto to_uri(const WeakPointer &pointer) -> URI {
                            std::allocator<JSON::Char>>
       result;
   stringify(pointer, result);
-  return URI::from_fragment(result.str());
+  return URI::from_unescaped_fragment(result.str());
 }
 
 auto to_uri(const WeakPointer &pointer, const URI &base) -> URI {

src/core/uri/include/sourcemeta/core/uri.h

@@ -421,7 +421,8 @@ class SOURCEMETA_CORE_URI_EXPORT URI {
   /// ```
   [[nodiscard]] auto fragment() const -> std::optional<std::string_view>;
 
-  /// Set the fragment part of the URI. For example:
+  /// Set the fragment part of the URI, taking the input as already
+  /// percent-encoded. For example:
   ///
   /// ```cpp
   /// #include <sourcemeta/core/uri.h>
@@ -435,6 +436,19 @@ class SOURCEMETA_CORE_URI_EXPORT URI {
   /// ```
   auto fragment(const std::string_view fragment) -> URI &;
 
+  /// Set the fragment part of the URI, taking the input as literal text that
+  /// still needs to be percent-encoded. For example:
+  ///
+  /// ```cpp
+  /// #include <sourcemeta/core/uri.h>
+  /// #include <cassert>
+  ///
+  /// sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  /// uri.unescaped_fragment("/100%");
+  /// assert(uri.recompose() == "https://www.sourcemeta.com#/100%25");
+  /// ```
+  auto unescaped_fragment(const std::string_view fragment) -> URI &;
+
   /// A non-owning, zero-copy view over the RFC 3986 query
   /// component of a URI. Provides convenience access to query
   /// parameters formatted as `name=value` pairs separated by `&`.
@@ -773,7 +787,8 @@ class SOURCEMETA_CORE_URI_EXPORT URI {
   /// To support ordering of URIs
   auto operator<(const URI &other) const noexcept -> bool;
 
-  /// Create a URI from a fragment. For example:
+  /// Create a URI from a fragment that is already percent-encoded. For
+  /// example:
   ///
   /// ```cpp
   /// #include <sourcemeta/core/uri.h>
@@ -785,6 +800,19 @@ class SOURCEMETA_CORE_URI_EXPORT URI {
   /// ```
   static auto from_fragment(const std::string_view fragment) -> URI;
 
+  /// Create a URI from a fragment given as literal text that still needs to be
+  /// percent-encoded. For example:
+  ///
+  /// ```cpp
+  /// #include <sourcemeta/core/uri.h>
+  /// #include <cassert>
+  ///
+  /// const sourcemeta::core::URI uri{
+  ///   sourcemeta::core::URI::from_unescaped_fragment("/100%")};
+  /// assert(uri.recompose() == "#/100%25");
+  /// ```
+  static auto from_unescaped_fragment(const std::string_view fragment) -> URI;
+
   /// Create a URI from a file system path. For example:
   ///
   /// ```cpp

src/core/uri/setters.cc

@@ -327,6 +327,28 @@ auto URI::fragment(const std::string_view fragment) -> URI & {
   return *this;
 }
 
+auto URI::unescaped_fragment(const std::string_view fragment) -> URI & {
+  std::string value;
+  value.reserve(fragment.size());
+
+  // RFC 3986 Section 2.4: "Because the percent ("%") character serves as the
+  // indicator for percent-encoded octets, it must be percent-encoded as "%25"
+  // for that octet to be used as data within a URI". Escaping it is what makes
+  // the encoding total, as every other character a fragment cannot hold
+  // literally is escaped when recomposing, and none of those escapes can be
+  // mistaken for input the caller wrote
+  for (const auto character : fragment) {
+    if (character == URI_PERCENT) {
+      uri_percent_encode_byte(value, URI_PERCENT);
+    } else {
+      value.push_back(character);
+    }
+  }
+
+  this->fragment_ = std::move(value);
+  return *this;
+}
+
 auto URI::query(const std::string_view query) -> URI & {
   if (query.empty()) {
     this->query_ = std::nullopt;

src/core/uri/uri.cc

@@ -15,6 +15,12 @@ auto URI::from_fragment(const std::string_view fragment) -> URI {
   return result;
 }
 
+auto URI::from_unescaped_fragment(const std::string_view fragment) -> URI {
+  URI result;
+  result.unescaped_fragment(fragment);
+  return result;
+}
+
 auto URI::from_iri(const std::string_view input) -> URI {
   URI result;
   result.iri_ = true;

test/jsonpointer/jsonpointer_fragment_to_pointer_test.cc

@@ -154,6 +154,23 @@ TEST(round_trip_tricky_tokens_with_base) {
   EXPECT_EQ(result.value(), pointer);
 }
 
+TEST(round_trip_token_with_percent_encoded_sequence) {
+  const sourcemeta::core::Pointer pointer{"a%20b"};
+  const auto result{
+      sourcemeta::core::fragment_to_pointer(sourcemeta::core::to_uri(pointer))};
+  EXPECT_TRUE(result.has_value());
+  EXPECT_EQ(result.value(), pointer);
+}
+
+TEST(round_trip_token_with_percent_encoded_sequence_with_base) {
+  const sourcemeta::core::Pointer pointer{"a%20b"};
+  const sourcemeta::core::URI base{"https://www.example.com"};
+  const auto result{sourcemeta::core::fragment_to_pointer(
+      sourcemeta::core::to_uri(pointer, base))};
+  EXPECT_TRUE(result.has_value());
+  EXPECT_EQ(result.value(), pointer);
+}
+
 TEST(round_trip_percent_encoded_colon) {
   const sourcemeta::core::Pointer pointer{"$defs",
                                           "https://example.com/schema/type"};

test/jsonpointer/jsonpointer_to_uri_test.cc

@@ -44,6 +44,26 @@ TEST(escape_uri_percentage) {
   EXPECT_EQ(fragment.recompose(), "#/foo/percent%25field");
 }
 
+TEST(escape_uri_percentage_before_hex_digits) {
+  const sourcemeta::core::Pointer pointer{"foo", "a%20b"};
+  const sourcemeta::core::URI fragment{sourcemeta::core::to_uri(pointer)};
+  EXPECT_EQ(fragment.recompose(), "#/foo/a%2520b");
+}
+
+// RFC 6901 Section 6 lists the fragment "#/c%25d" for the property "c%d"
+TEST(escape_uri_percentage_rfc_6901_example) {
+  const sourcemeta::core::Pointer pointer{"c%d"};
+  const sourcemeta::core::URI fragment{sourcemeta::core::to_uri(pointer)};
+  EXPECT_EQ(fragment.recompose(), "#/c%25d");
+}
+
+TEST(escape_uri_percentage_distinct_from_encoded_space) {
+  const sourcemeta::core::Pointer space{"a b"};
+  const sourcemeta::core::Pointer percentage{"a%20b"};
+  EXPECT_EQ(sourcemeta::core::to_uri(space).recompose(), "#/a%20b");
+  EXPECT_EQ(sourcemeta::core::to_uri(percentage).recompose(), "#/a%2520b");
+}
+
 TEST(escape_uri_quote) {
   const sourcemeta::core::Pointer pointer{"foo", "quote\"field"};
   const sourcemeta::core::URI fragment{sourcemeta::core::to_uri(pointer)};
@@ -297,9 +317,7 @@ TEST(with_absolute_base_percentage) {
   const sourcemeta::core::Pointer pointer{"foo%bar"};
   const sourcemeta::core::URI base{"https://www.example.com"};
   const sourcemeta::core::URI fragment{sourcemeta::core::to_uri(pointer, base)};
-  // The %ba in foo%bar happens to be a valid percent-encoded sequence.
-  // Canonicalize uppercases the hex digits.
-  EXPECT_EQ(fragment.recompose(), "https://www.example.com#/foo%BAr");
+  EXPECT_EQ(fragment.recompose(), "https://www.example.com#/foo%25bar");
 }
 
 TEST(with_relative_base) {

test/uri/uri_fragment_test.cc

@@ -243,6 +243,86 @@ TEST(set_non_ascii_character) {
   EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%C3%A9");
 }
 
+TEST(set_unescaped_simple) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo/bar");
+  EXPECT_EQ(uri.fragment(), "/foo/bar");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo/bar");
+}
+
+TEST(set_unescaped_empty) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("");
+  EXPECT_EQ(uri.fragment(), "");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#");
+}
+
+TEST(set_unescaped_percentage) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo%bar");
+  EXPECT_EQ(uri.fragment(), "/foo%25bar");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%25bar");
+}
+
+TEST(set_unescaped_percentage_at_end) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo%");
+  EXPECT_EQ(uri.fragment(), "/foo%25");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%25");
+}
+
+TEST(set_unescaped_percentage_followed_by_valid_hex_sequence) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo%2Fbar");
+  EXPECT_EQ(uri.fragment(), "/foo%252Fbar");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%252Fbar");
+}
+
+TEST(set_unescaped_multiple_percentages) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo%%bar");
+  EXPECT_EQ(uri.fragment(), "/foo%25%25bar");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%25%25bar");
+}
+
+TEST(set_unescaped_hash) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("#foo");
+  EXPECT_EQ(uri.fragment(), "#foo");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#%23foo");
+}
+
+TEST(set_unescaped_space_character) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo bar");
+  EXPECT_EQ(uri.fragment(), "/foo bar");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%20bar");
+}
+
+TEST(set_unescaped_non_ascii_character) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo\xC3\xA9");
+  EXPECT_EQ(uri.fragment(), "/foo\xC3\xA9");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%C3%A9");
+}
+
+TEST(set_unescaped_percentage_survives_canonicalize) {
+  sourcemeta::core::URI uri{"https://www.sourcemeta.com"};
+  uri.unescaped_fragment("/foo%bar");
+  uri.canonicalize();
+  EXPECT_EQ(uri.fragment(), "/foo%25bar");
+  EXPECT_EQ(uri.recompose(), "https://www.sourcemeta.com#/foo%25bar");
+}
+
+TEST(set_unescaped_distinct_from_encoded_space) {
+  sourcemeta::core::URI space{"https://www.sourcemeta.com"};
+  space.unescaped_fragment("/a b");
+  sourcemeta::core::URI percentage{"https://www.sourcemeta.com"};
+  percentage.unescaped_fragment("/a%20b");
+  EXPECT_EQ(space.recompose(), "https://www.sourcemeta.com#/a%20b");
+  EXPECT_EQ(percentage.recompose(), "https://www.sourcemeta.com#/a%2520b");
+}
+
 TEST(getter_setter_invariant_simple) {
   sourcemeta::core::URI uri{"https://example.com/path#foo"};
   EXPECT_TRUE(uri.fragment().has_value());

test/uri/uri_test.cc

@@ -112,6 +112,46 @@ TEST(from_fragment_with_hash_empty) {
   EXPECT_EQ(uri.recompose(), "#");
 }
 
+TEST(from_unescaped_fragment) {
+  const auto uri{sourcemeta::core::URI::from_unescaped_fragment("foo")};
+  const auto fragment{uri.fragment()};
+  EXPECT_TRUE(fragment.has_value());
+  EXPECT_EQ(fragment.value(), "foo");
+  EXPECT_EQ(uri.recompose(), "#foo");
+}
+
+TEST(from_unescaped_fragment_with_percentage) {
+  const auto uri{sourcemeta::core::URI::from_unescaped_fragment("/100%")};
+  const auto fragment{uri.fragment()};
+  EXPECT_TRUE(fragment.has_value());
+  EXPECT_EQ(fragment.value(), "/100%25");
+  EXPECT_EQ(uri.recompose(), "#/100%25");
+}
+
+TEST(from_unescaped_fragment_with_percentage_sequence) {
+  const auto uri{sourcemeta::core::URI::from_unescaped_fragment("/a%20b")};
+  const auto fragment{uri.fragment()};
+  EXPECT_TRUE(fragment.has_value());
+  EXPECT_EQ(fragment.value(), "/a%2520b");
+  EXPECT_EQ(uri.recompose(), "#/a%2520b");
+}
+
+TEST(from_unescaped_fragment_with_hash) {
+  const auto uri{sourcemeta::core::URI::from_unescaped_fragment("#foo")};
+  const auto fragment{uri.fragment()};
+  EXPECT_TRUE(fragment.has_value());
+  EXPECT_EQ(fragment.value(), "#foo");
+  EXPECT_EQ(uri.recompose(), "#%23foo");
+}
+
+TEST(from_unescaped_fragment_empty) {
+  const auto uri{sourcemeta::core::URI::from_unescaped_fragment("")};
+  const auto fragment{uri.fragment()};
+  EXPECT_TRUE(fragment.has_value());
+  EXPECT_EQ(fragment.value(), "");
+  EXPECT_EQ(uri.recompose(), "#");
+}
+
 TEST(using_istream) {
   std::istringstream input{"https://example.com"};
   const sourcemeta::core::URI uri{input};