|
3 | 3 | # Functions and classes used for parsing Python files using `tree-sitter-graph` |
4 | 4 |
|
5 | 5 | from ast import literal_eval |
6 | | -import re |
| 6 | +import json |
7 | 7 | import sys |
8 | 8 | import os |
9 | 9 | import semmle.python.parser |
@@ -114,98 +114,67 @@ def __repr__(self): |
114 | 114 | cargo_file = os.path.join(tsg_python_path, "Cargo.toml") |
115 | 115 | tsg_command = ["cargo", "run", "--quiet", "--release", "--manifest-path="+cargo_file] |
116 | 116 |
|
| 117 | +def _decode_tsg_value(value): |
| 118 | + value_type = value["type"] |
| 119 | + if value_type == "null": |
| 120 | + return None |
| 121 | + if value_type in ("bool", "int", "string"): |
| 122 | + return value[value_type] |
| 123 | + if value_type == "list": |
| 124 | + return [_decode_tsg_value(item) for item in value["values"]] |
| 125 | + if value_type == "graphNode": |
| 126 | + return Node(value["id"]) |
| 127 | + raise ValueError("Unsupported TSG value type '{}'".format(value_type)) |
| 128 | + |
| 129 | +def _decode_tsg_node_attributes(encoded_attrs, path, logger): |
| 130 | + # Decode every attribute first so location information is available if string decoding fails. |
| 131 | + attrs = { |
| 132 | + key: _decode_tsg_value(value) |
| 133 | + for key, value in encoded_attrs.items() |
| 134 | + } |
| 135 | + if "s" not in attrs: |
| 136 | + return attrs |
| 137 | + |
| 138 | + try: |
| 139 | + attrs["s"] = evaluate_string(attrs["s"]) |
| 140 | + except Exception as ex: |
| 141 | + loc = ":".join(str(i) for i in get_location_info(attrs)) |
| 142 | + error = ex.args[0] if ex.args else "unknown" |
| 143 | + logger.warning( |
| 144 | + "Error '{}' while parsing value {} at {}:{}\n".format( |
| 145 | + error, repr(attrs["s"]), path, loc |
| 146 | + ) |
| 147 | + ) |
| 148 | + return attrs |
| 149 | + |
117 | 150 | def read_tsg_python_output(path, logger): |
| 151 | + command_args = tsg_command + [path] |
| 152 | + p = subprocess.Popen(command_args, stdout=subprocess.PIPE) |
| 153 | + stdout, _ = p.communicate() |
| 154 | + if p.returncode: |
| 155 | + raise subprocess.CalledProcessError(p.returncode, command_args, stdout) |
| 156 | + |
118 | 157 | # Mapping from node id (an integer) to a dictionary containing attribute data. |
119 | 158 | node_attr = {} |
120 | 159 | # Mapping a start node to a map from attribute names to lists of (value, end_node) pairs. |
121 | 160 | edge_attr = {} |
122 | 161 |
|
123 | | - command_args = tsg_command + [path] |
124 | | - p = subprocess.Popen(command_args, stdout=subprocess.PIPE) |
125 | | - for line in p.stdout: |
126 | | - line = line.decode(sys.getfilesystemencoding()) |
127 | | - line = line.rstrip() |
128 | | - if line.startswith("node"): # e.g. `node 5` |
129 | | - current_node = int(line.split(" ")[1]) |
130 | | - d = {} |
131 | | - node_attr[current_node] = d |
132 | | - in_node = True |
133 | | - elif line.startswith("edge"): # e.g. `edge 5 -> 6` |
134 | | - current_start, current_end = tuple(map(int, line[4:].split("->"))) |
135 | | - d = edge_attr.setdefault(current_start, {}) |
136 | | - in_node = False |
137 | | - else: # attribute, e.g. `_kind: "Class"` |
138 | | - key, value = line[2:].split(": ", 1) |
139 | | - if value.startswith("[graph node"): # e.g. `_skip_to: [graph node 5]` |
140 | | - value = Node(int(value.split(" ")[2][:-1])) |
141 | | - elif value == "#true": # e.g. `_is_parenthesised: #true` |
142 | | - value = True |
143 | | - elif value == "#false": # e.g. `top: #false` |
144 | | - value = False |
145 | | - elif value == "#null": # e.g. `exc: #null` |
146 | | - value = None |
147 | | - else: # literal values, e.g. `name: "k1.k2"` or `level: 5` |
148 | | - value = rust_to_python_escapes(value) |
149 | | - try: |
150 | | - if key =="s" and value[0] == '"': # e.g. `s: "k1.k2"` |
151 | | - value = evaluate_string(value) |
152 | | - else: |
153 | | - value = literal_eval(value) |
154 | | - if isinstance(value, bytes): |
155 | | - try: |
156 | | - value = value.decode(sys.getfilesystemencoding()) |
157 | | - except UnicodeDecodeError: |
158 | | - # just include the bytes as-is |
159 | | - pass |
160 | | - except Exception as ex: |
161 | | - # We may not know the location at this point -- for instance if we forgot to set |
162 | | - # it -- but `get_location_info` will degrade gracefully in this case. |
163 | | - loc = ":".join(str(i) for i in get_location_info(d)) |
164 | | - error = ex.args[0] if ex.args else "unknown" |
165 | | - logger.warning("Error '{}' while parsing value {} at {}:{}\n".format(error, repr(value), path, loc)) |
166 | | - if in_node: |
167 | | - d[key] = value |
168 | | - else: |
169 | | - d.setdefault(key, []).append((value, current_end)) |
170 | | - p.stdout.close() |
171 | | - p.terminate() |
172 | | - p.wait() |
| 162 | + for encoded_node in json.loads(stdout): |
| 163 | + current_node = encoded_node["id"] |
| 164 | + attrs = _decode_tsg_node_attributes(encoded_node["attrs"], path, logger) |
| 165 | + node_attr[current_node] = attrs |
| 166 | + for encoded_edge in encoded_node["edges"]: |
| 167 | + current_end = encoded_edge["sink"] |
| 168 | + edge_fields = edge_attr.setdefault(current_node, {}) |
| 169 | + for key, value in encoded_edge["attrs"].items(): |
| 170 | + value = _decode_tsg_value(value) |
| 171 | + edge_fields.setdefault(key, []).append((value, current_end)) |
173 | 172 | logger.debug("Read {} nodes and {} edges from TSG output".format(len(node_attr), len(edge_attr))) |
174 | 173 | return node_attr, edge_attr |
175 | 174 |
|
176 | | -# `tsg-python` serialises string values using Rust's `Debug` formatting, which diverges from what |
177 | | -# Python's `literal_eval` accepts in two ways: |
178 | | -# - characters Rust considers non-printable -- including grapheme-extending ones such as the U+FE0F |
179 | | -# variation selector, U+200D zero width joiner and combining accents -- are rendered as `\u{...}`, |
180 | | -# a syntax Python does not know at all; |
181 | | -# - NUL is rendered as `\0`, which Python reads as the start of an *octal* escape, silently |
182 | | -# swallowing up to two more digits (NUL followed by `1` is emitted as `"\01"`, which decodes |
183 | | -# to `\x01`). |
184 | | -# Everything else Rust emits (`\t`, `\r`, `\n`, `\\`, `\"`, and unescaped characters) is read back |
185 | | -# identically by `literal_eval`, as verified exhaustively over every Unicode scalar value. |
186 | | -_RUST_ESCAPE = re.compile(r"\\(?:u\{([0-9a-fA-F]{1,6})\}|.)", re.DOTALL) |
187 | | - |
188 | | -def rust_to_python_escapes(text): |
189 | | - """Rewrites Rust escapes in `text` that Python would reject or misread into their equivalents. |
190 | | -
|
191 | | - Matching every escape sequence (rather than only the offending ones) keeps the scan in step with |
192 | | - the backslashes, so an escaped backslash -- how a literal `\\u{fe0f}` in the source is |
193 | | - serialised -- is left alone.""" |
194 | | - if "\\u{" not in text and "\\0" not in text: |
195 | | - return text |
196 | | - def replace(match): |
197 | | - code_point = match.group(1) |
198 | | - if code_point is None: |
199 | | - return "\\x00" if match.group(0) == "\\0" else match.group(0) |
200 | | - code_point = int(code_point, 16) |
201 | | - if code_point > 0xFFFF: |
202 | | - return "\\U{:08x}".format(code_point) |
203 | | - return "\\u{:04x}".format(code_point) |
204 | | - return _RUST_ESCAPE.sub(replace, text) |
205 | | - |
206 | | -def evaluate_string(s): |
207 | | - s = literal_eval(s) |
208 | | - prefix, quotes, content = split_string(s, None) |
| 175 | +def evaluate_string(source_literal): |
| 176 | + """Evaluates Python string literal text that has already been decoded from the wire format.""" |
| 177 | + prefix, quotes, content = split_string(source_literal, None) |
209 | 178 | ends_with_illegal_character = False |
210 | 179 | # If the string ends with the same quote character as the outer quotes (and/or backslashes) |
211 | 180 | # (e.g. the first string part of `f"""hello"{0}"""`), we must take care to not accidently create |
|
0 commit comments