Merge pull request #16 from BelfrySCAD/feat/unicode-line-separators

Lexer: Unicode line separators are whitespace that ends a // comment

Author: revarbat

src/grammar/lexer.l

@@ -88,7 +88,17 @@ NUMLIT  0[xX][0-9A-Fa-f]+|[0-9]+([.][0-9]*)?([eE][+-]?[0-9]+)?|[.][0-9]+([eE][+-
     // U+FEFF (EF BB BF, a byte-order mark, anywhere in the file) and a bare
     // A0, Latin-1's no-break space.
 }
-"//"[^\n]*               { /* skip line comment */ }
+\xe2\x80\xa8|\xe2\x80\xa9|\xc2\x85 {
+    // U+2028 LINE SEPARATOR, U+2029 PARAGRAPH SEPARATOR and U+0085 NEXT
+    // LINE: line breaks to look at, so whitespace that also ends a `//`
+    // comment (below). Qt's editor types U+2028 for Shift+Enter, and it
+    // shows as a new line (BelfrySCAD #949). It does NOT count as a line:
+    // the editor numbers its lines by block and a U+2028 sits inside one,
+    // so every line and column after it still matches the editor's.
+    // OpenSCAD's lexer rejects all three.
+}
+    /* A line comment ends at a newline or at any separator above. */
+"//"([^\n\xe2\xc2]|\xe2[^\x80\n]|\xe2\x80[^\xa8\xa9\n]|\xc2[^\x85\n])* { /* skip line comment */ }
 "/*"([^*]|\*+[^*/])*\*+"/" { /* skip block comment */ }
 
 {IDENT} {

tests/test_lexical.cpp

@@ -483,6 +483,22 @@ TEST(LexicalWhitespace, NoBreakSpaceByteOrderMarkAndLatin1A0) {
     }
 }
 
+// U+2028/U+2029/U+0085 are whitespace that ends a line comment, without
+// counting as a line (BelfrySCAD's editor numbers lines by block, and a
+// U+2028 sits inside one).
+TEST(LexicalWhitespace, UnicodeLineSeparatorsEndACommentButNotALine) {
+    for (const char* sep : {"\xe2\x80\xa8", "\xe2\x80\xa9", "\xc2\x85"}) {
+        const std::string src = std::string("a = 1;") + sep + "b = 2; // note" + sep + "c = 3;\nd = 4;";
+        auto ast = parseSrc(src);
+        ASSERT_EQ(ast.size(), 4u) << "the comment swallowed c = 3";
+        EXPECT_EQ(ast[2]->position().line, 1);
+        EXPECT_EQ(size_t(ast[2]->position().start_offset), src.find("c = 3"));
+        EXPECT_EQ(ast[3]->position().line, 2);
+    }
+    // Other multi-byte characters stay inside the comment.
+    EXPECT_EQ(parseSrc("a = 1; // \xe2\x80\x94 \xc2\xa0 \xe2\x82\xac c = 3;\nb = 2;").size(), 2u);
+}
+
 // The message quotes a whole character, and names a byte that starts none,
 // so it is always valid UTF-8.
 TEST(LexicalWhitespace, UnexpectedCharacterMessageIsValidUtf8) {