Python: Fail early on syntax errors

Previously, we would only signal a syntax error when we created a
SyntaxErrorNode from the tsg-python output. However, this meant that we
might accidentally try to construct nodes based on bogus input (since if
the file has a syntax error, then all bets are off).

Instead, we now propagate the syntax error as soon as we encounter it
(immediately after reading the output of tsg-python).

Author: tausbn

python/extractor/semmle/python/parser/tsg_parser.py

@@ -163,6 +163,12 @@ def read_tsg_python_output(path, logger):
         current_node = encoded_node["id"]
         attrs = _decode_tsg_node_attributes(encoded_node["attrs"], path, logger)
         node_attr[current_node] = attrs
+        if attrs.get("_kind") == "SyntaxErrorNode":
+            lineno, offset, _, _ = get_location_info(attrs)
+            exc = SyntaxError("Syntax Error")
+            exc.lineno = lineno
+            exc.offset = offset
+            raise exc
         for encoded_edge in encoded_node["edges"]:
             current_end = encoded_edge["sink"]
             edge_fields = edge_attr.setdefault(current_node, {})
@@ -339,12 +345,6 @@ def parse(path, logger):
         node.lineno, node.col_offset, end_line, end_column = get_location_info(attrs)
         node._end = (end_line, end_column)
 
-        if isinstance(node, SyntaxErrorNode):
-            exc = SyntaxError("Syntax Error")
-            exc.lineno = node.lineno
-            exc.offset = node.col_offset
-            raise exc
-
         # Set up context information, if any
         if "ctx" in expected_fields:
             node.ctx = get_context(id, node_attr, path, logger)

python/extractor/tests/test_tsg_parser.py

@@ -74,6 +74,41 @@ def test_evaluate_string_on_reported_value(self):
         value = '"\u26a0\ufe0f  problem %s: %s"'
         self.assertEqual(evaluate_string(value), "\u26a0\ufe0f  problem %s: %s")
 
+    def test_syntax_error_node_raises(self):
+        output = json.dumps(
+            [
+                {
+                    "id": 0,
+                    "edges": [],
+                    "attrs": {
+                        "_kind": {"type": "string", "string": "SyntaxErrorNode"},
+                        "_location": {
+                            "type": "list",
+                            "values": [
+                                {"type": "int", "int": 4},
+                                {"type": "int", "int": 5},
+                                {"type": "int", "int": 4},
+                                {"type": "int", "int": 6},
+                            ],
+                        },
+                        "source": {"type": "string", "string": "!"},
+                    },
+                }
+            ]
+        ).encode("utf-8")
+
+        process = unittest.mock.Mock()
+        process.communicate.return_value = (output, None)
+        process.returncode = 0
+        with unittest.mock.patch(
+            "semmle.python.parser.tsg_parser.subprocess.Popen", return_value=process
+        ):
+            with self.assertRaises(SyntaxError) as raised:
+                read_tsg_python_output("test.py", unittest.mock.Mock())
+
+        self.assertEqual(raised.exception.lineno, 5)
+        self.assertEqual(raised.exception.offset, 5)
+
 
 class FormatMessageTest(unittest.TestCase):
     """A pre-formatted log message may contain `%` directives coming from the analysed source, and