1import configparser
2from collections.abc import ByteString, Callable
3from pathlib import Path
4from typing import TypedDict
5from urllib.request import pathname2url
6
7from giturlparse import parse
8from tree_sitter import Language, Parser, Point, Query, QueryCursor
9from tree_sitter import Node as TreeSitterNode
10
11from sphinx_codelinks.config import UNIX_NEWLINE, CommentCategory
12from sphinx_codelinks.logger import get_logger
13from sphinx_codelinks.source_discover.config import CommentType
14
15# Language-specific node types for scope detection.
16#
17# YAML and JSONC are intentionally absent. They are data formats, not code, so a
18# comment associates with the surrounding data structure (key/value pair, list
19# item, or scalar) rather than with an enclosing or following declaration. That
20# needs a different algorithm (inline same-row association first, scalar targets,
21# grammar-specific traversal), implemented in find_yaml_associated_structure and
22# find_jsonc_associated_structure and dispatched from find_associated_scope.
23# Those bespoke finders never read this table (only find_next_scope and
24# find_enclosing_scope do), so an entry here would be dead.
25SCOPE_NODE_TYPES = {
[docs] 26 # @Python Scope Node Types, IMPL_PY_2, impl, [FE_PY]
27 CommentType.python: {"function_definition", "class_definition"},
[docs] 28 # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP]
29 CommentType.cpp: {"function_definition", "class_definition"},
30 CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"},
31 # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST];
32 CommentType.rust: {
33 "function_item",
34 "struct_item",
35 "enum_item",
36 "impl_item",
37 "trait_item",
38 "mod_item",
39 },
[docs] 40 # @Go Scope Node Types, IMPL_GO_4, impl, [FE_GO]
41 CommentType.go: {
42 "function_declaration",
43 "method_declaration",
44 "type_declaration",
45 "type_spec",
46 },
[docs] 47 # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH]
48 CommentType.bash: {"function_definition"},
49}
50
51logger = get_logger(__name__)
52
53GIT_HOST_URL_TEMPLATE = {
54 "github": "https://github.com/{owner}/{repo}/blob/{rev}/{path}#L{lineno}",
55 "gitlab": "https://gitlab.com/{owner}/{repo}/-/blob/{rev}/{path}#L{lineno}",
56}
57
58PYTHON_QUERY = """
59 ; Match comments
60 (comment) @comment
61
62 ; Match docstrings inside modules, functions, or classes
63 (module (expression_statement (string)) @comment)
64 (function_definition (block (expression_statement (string)) @comment))
65 (class_definition (block (expression_statement (string)) @comment))
66 """
67CPP_QUERY = """(comment) @comment"""
68C_SHARP_QUERY = """(comment) @comment"""
69YAML_QUERY = """(comment) @comment"""
70RUST_QUERY = """
71 (line_comment) @comment
72 (block_comment) @comment
73"""
[docs] 74# @Go comment query for tree-sitter, IMPL_GO_3, impl, [FE_GO]
75GO_QUERY = """
76 (comment) @comment
77"""
78JSONC_QUERY = """(comment) @comment"""
[docs] 79# @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH]
80BASH_QUERY = """(comment) @comment"""
81
82# JSON value node types that can be associated with a comment.
83JSON_STRUCTURE_TYPES = {
84 "pair",
85 "object",
86 "array",
87 "string",
88 "number",
89 "true",
90 "false",
91 "null",
92}
93
94
95def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
96 """Return True if file is likely text, False if binary."""
97 try:
98 with filepath.open("rb") as f:
99 chunk = f.read(sample_size)
100 # Quick binary heuristic: null byte present
101 if b"\x00" in chunk:
102 return False
103 # Try UTF-8 decode on the sample
104 chunk.decode("utf-8")
105 return True
106 except UnicodeDecodeError:
107 return False
108
109
[docs]110# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH]
111def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
112 if comment_type == CommentType.cpp:
113 import tree_sitter_cpp
114
115 parsed_language = Language(tree_sitter_cpp.language())
116 query = Query(parsed_language, CPP_QUERY)
117 elif comment_type == CommentType.python:
118 import tree_sitter_python
119
120 parsed_language = Language(tree_sitter_python.language())
121 query = Query(parsed_language, PYTHON_QUERY)
122 elif comment_type == CommentType.cs:
123 import tree_sitter_c_sharp
124
125 parsed_language = Language(tree_sitter_c_sharp.language())
126 query = Query(parsed_language, C_SHARP_QUERY)
127 elif comment_type == CommentType.yaml:
128 import tree_sitter_yaml
129
130 parsed_language = Language(tree_sitter_yaml.language())
131 query = Query(parsed_language, YAML_QUERY)
132 elif comment_type == CommentType.rust:
133 import tree_sitter_rust
134
135 parsed_language = Language(tree_sitter_rust.language())
136 query = Query(parsed_language, RUST_QUERY)
137 elif comment_type == CommentType.go:
138 import tree_sitter_go
139
140 parsed_language = Language(tree_sitter_go.language())
141 query = Query(parsed_language, GO_QUERY)
142 elif comment_type == CommentType.jsonc:
143 import tree_sitter_json
144
145 parsed_language = Language(tree_sitter_json.language())
146 query = Query(parsed_language, JSONC_QUERY)
147 elif comment_type == CommentType.bash:
148 import tree_sitter_bash
149
150 parsed_language = Language(tree_sitter_bash.language())
151 query = Query(parsed_language, BASH_QUERY)
152 else:
153 raise ValueError(f"Unsupported comment style: {comment_type}")
154 parser = Parser(parsed_language)
155 return parser, query
156
157
158def wrap_read_callable_point(
159 src_string: ByteString,
160) -> Callable[[int, Point], ByteString]:
161 def read_callable_byte_offset(byte_offset: int, _: Point) -> ByteString:
162 return src_string[byte_offset : byte_offset + 1]
163
164 return read_callable_byte_offset
165
166
[docs]167# @Comment extraction from source code using tree-sitter, IMPL_EXTR_1, impl, [FE_DEF]
168def extract_comments(
169 src_string: ByteString, parser: Parser, query: Query
170) -> list[TreeSitterNode] | None:
171 """Get all comments from source files by tree-sitter."""
172 read_point_fn = wrap_read_callable_point(src_string)
173 tree = parser.parse(read_point_fn)
174 query_cursor = QueryCursor(query)
175 captures: dict[str, list[TreeSitterNode]] = query_cursor.captures(tree.root_node)
176
177 return captures.get("comment")
178
179
180def find_enclosing_scope(
181 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
182) -> TreeSitterNode | None:
183 """Find the enclosing scope of a comment."""
184 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
185 current: TreeSitterNode = node
186 while current:
187 if current.type in scope_types:
188 return current
189 current: TreeSitterNode | None = current.parent # required for node traversal
190 return None
191
192
193def find_next_scope(
194 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
195) -> TreeSitterNode | None:
196 """Find the next scope of a comment."""
197 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
198 current: TreeSitterNode = node
199 while current:
200 if current.type in scope_types:
201 return current
202 current: TreeSitterNode | None = (
203 current.next_named_sibling
204 ) # required for node traversal
205 if current and current.type == "block":
206 for child in current.named_children:
207 if child.type in scope_types:
208 return child
209 return None
210
211
212def _find_yaml_structure_in_block_node(
213 block_node: TreeSitterNode,
214) -> TreeSitterNode | None:
215 """Find YAML structure elements within a block_node."""
216 for grandchild in block_node.named_children:
217 if grandchild.type == "block_mapping":
218 for ggchild in grandchild.named_children:
219 if ggchild.type == "block_mapping_pair":
220 return ggchild
221 elif grandchild.type == "block_sequence":
222 for ggchild in grandchild.named_children:
223 if ggchild.type == "block_sequence_item":
224 return ggchild
225 return None
226
227
228def find_yaml_next_structure(node: TreeSitterNode) -> TreeSitterNode | None:
229 """Find the next YAML structure element after the comment node."""
230 current = node.next_named_sibling
231 while current:
232 if current.type in {
233 "block_mapping_pair",
234 "block_sequence_item",
235 "flow_mapping",
236 "flow_sequence",
237 }:
238 return current
239 if current.type == "document":
240 for child in current.named_children:
241 if child.type == "block_node":
242 result = _find_yaml_structure_in_block_node(child)
243 if result:
244 return result
245 if current.type == "block_node":
246 result = _find_yaml_structure_in_block_node(current)
247 if result:
248 return result
249 current = current.next_named_sibling
250 return None
251
252
253def find_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode | None:
254 """Find a previous named sibling that is on the same row as the comment.
255
256 Grammar-agnostic: used to detect inline comments in both YAML and JSONC.
257 """
258 comment_row = node.start_point.row
259 current = node.prev_named_sibling
260
261 while current:
262 # Check if this sibling ends on the same row as the comment starts
263 # This indicates it's an inline comment
264 if current.end_point.row == comment_row:
265 return current
266 # If we find a sibling that ends before the comment row, we can stop
267 # as we won't find any siblings on the same row going backwards
268 if current.end_point.row < comment_row:
269 break
270 current = current.prev_named_sibling
271
272 return None
273
274
275def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
276 """Find the YAML structure (key-value pair, list item, etc.) associated with a comment."""
277 # First, check if this is an inline comment by looking for a previous sibling on the same row
278 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
279 if prev_sibling_same_row:
280 return prev_sibling_same_row
281
282 # If no previous sibling on same row, try to find the next named sibling (structure after the comment)
283 structure = find_yaml_next_structure(node)
284 if structure:
285 return structure
286
287 # If no next sibling found, traverse up to find parent structure
288 parent = node.parent
289 while parent:
290 if parent.type in {"block_mapping_pair", "block_sequence_item"}:
291 return parent
292 parent = parent.parent
293
294 return None
295
296
[docs]297# @JSONC comment-to-structure association, IMPL_JSONC_2, impl, [FE_JSONC]
298def find_jsonc_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
299 """Find the JSON structure (key/value pair, value, list item) for a comment.
300
301 JSON is data rather than code, so association follows the same intent as YAML:
302 an inline comment belongs to the value on its row, a leading comment belongs to
303 the following structure, otherwise it belongs to the enclosing structure.
304 """
305 # Inline comment: a value/pair on the same row, before the comment
306 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
307 if prev_sibling_same_row:
308 return prev_sibling_same_row
309
310 # Leading comment: the next structure following the comment
311 current = node.next_named_sibling
312 while current:
313 if current.type in JSON_STRUCTURE_TYPES:
314 return current
315 current = current.next_named_sibling
316
317 # Otherwise: the enclosing structure
318 parent = node.parent
319 while parent:
320 if parent.type in {"pair", "object", "array"}:
321 return parent
322 parent = parent.parent
323
324 return None
325
326
327def find_associated_scope(
328 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
329) -> TreeSitterNode | None:
330 """Find the associated scope of a comment."""
331 if comment_type == CommentType.yaml:
332 # YAML uses different structure association logic
333 return find_yaml_associated_structure(node)
334
335 if comment_type == CommentType.jsonc:
336 # JSONC uses data-aware structure association logic
337 return find_jsonc_associated_structure(node)
338
339 if node.type == CommentCategory.docstring:
340 # Only for python's docstring
341 return find_enclosing_scope(node, comment_type)
342 # General comments regardless of comment types
343 associated_scope = find_next_scope(node, comment_type)
344 if not associated_scope:
345 associated_scope = find_enclosing_scope(node, comment_type)
346 return associated_scope
347
348
349def locate_git_root(src_dir: Path) -> Path | None:
350 """Traverse upwards to find git root."""
351 current = src_dir.resolve()
352 parents = list(current.parents)
353 parents.append(current)
354 for parent in parents:
355 if (parent / ".git").exists() and (parent / ".git").is_dir():
356 return parent
357 logger.warning(
358 f"git root is not found in the parent of {src_dir}",
359 subtype="git_root",
360 location=str(src_dir),
361 )
362 return None
363
364
365def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
366 """Get remote url from .git/config."""
367 config_path = git_root / ".git" / "config"
368 if not config_path.exists():
369 logger.warning(
370 f"{config_path} does not exist",
371 subtype="git_config",
372 location=str(config_path),
373 )
374 return None
375
376 config = configparser.ConfigParser(allow_no_value=True, strict=False)
377 config.read(config_path)
378 section = f'remote "{remote_name}"'
379 if section in config and "url" in config[section]:
380 url: str = config[section]["url"]
381 return url
382 logger.warning(
383 f"remote-url is not found in {config_path}",
384 subtype="git_remote",
385 location=str(config_path),
386 )
387 return None
388
389
390def get_current_rev(git_root: Path) -> str | None:
391 """Get current commit rev from .git/HEAD."""
392 head_path = git_root / ".git" / "HEAD"
393 if not head_path.exists():
394 logger.warning(
395 f"{head_path} does not exist",
396 subtype="git_head",
397 location=str(head_path),
398 )
399 return None
400 head_content = head_path.read_text().strip()
401 if not head_content.startswith("ref: "):
402 # Detached HEAD (e.g. CI checkouts): .git/HEAD holds the commit SHA
403 # directly, which is exactly the rev we want.
404 return head_content
405
406 ref_path = git_root / ".git" / head_content.split(":", 1)[1].strip()
407 if not ref_path.exists():
408 logger.warning(
409 f"{ref_path} does not exist",
410 subtype="git_ref",
411 location=str(ref_path),
412 )
413 return None
414 return ref_path.read_text().strip()
415
416
417def form_https_url(
418 git_url: str, rev: str, project_path: Path, filepath: Path, lineno: int
419) -> str | None:
420 parsed_url = parse(git_url)
421 template = GIT_HOST_URL_TEMPLATE.get(parsed_url.platform)
422 if not template:
423 logger.warning(
424 f"Unsupported Git host: {parsed_url.platform}",
425 subtype="git_host",
426 )
427 return git_url
428 https_url = template.format(
429 owner=parsed_url.owner,
430 repo=parsed_url.repo,
431 rev=rev,
432 path=pathname2url(str(filepath.absolute().relative_to(project_path))),
433 lineno=str(lineno),
434 )
435 return https_url
436
437
438def remove_leading_sequences(text: str, leading_sequences: list[str]) -> str:
439 lines = text.splitlines(keepends=True)
440 no_comment_lines = []
441 for line in lines:
442 leading_sequence_exist = False
443 for leading_sequence in leading_sequences:
444 leading_sequence_idx = line.find(leading_sequence)
445 if leading_sequence_idx == -1:
446 continue
447 no_comment_lines.append(
448 line[leading_sequence_idx + len(leading_sequence) :]
449 )
450 leading_sequence_exist = True
451 break
452
453 if not leading_sequence_exist:
454 no_comment_lines.append(line)
455
456 return "".join(no_comment_lines)
457
458
459class ExtractedRstType(TypedDict):
460 rst_text: str
461 row_offset: int
462 start_idx: int
463 end_idx: int
464
465
[docs]466# @Extract reStructuredText blocks embedded in comments, IMPL_RST_1, impl, [FE_RST_EXTRACTION]
467def extract_rst(
468 text: str, start_marker: str, end_marker: str
469) -> ExtractedRstType | None:
470 """Extract rst from a comment.
471
472 Two use cases:
473 1. Start_marker and end_marker one the same line.
474
475 The rst text is wrapped by start and the end markers on the same line,
476 so, there is no need to remove the leading chars.ArithmeticError
477 E.g.
478 @rst .. admonition:: title here @endrst
479
480 2. Start_marker and end_marker in different lines.
481
482 The rst text is expected to start from the next line of the start_marker
483 and ends at he previous line of the end_marker.
484 E.g.
485 @rst
486 .. admonition:: title here
487 :collapsible: open
488
489 This example is collapsible, and initially open.
490 @endrst
491 """
492 start_idx = text.find(start_marker)
493 end_idx = text.rfind(end_marker)
494 if start_idx == -1 or end_idx == -1:
495 return None
496 rst_text = text[start_idx + len(start_marker) : end_idx]
497 row_offset = len(text[:start_idx].splitlines())
498 if not rst_text.strip():
499 # empty string is out of the interest
500 return None
501 if UNIX_NEWLINE not in rst_text:
502 # single line rst text
503 oneline_rst: ExtractedRstType = {
504 "rst_text": rst_text,
505 "row_offset": row_offset,
506 "start_idx": start_idx + len(start_marker),
507 "end_idx": end_idx,
508 }
509 return oneline_rst
510
511 # multiline rst text
512
513 first_newline_idx = rst_text.find(UNIX_NEWLINE)
514 rst_text = rst_text[first_newline_idx + len(UNIX_NEWLINE) :]
515 multiline_rst: ExtractedRstType = {
516 "rst_text": rst_text,
517 "row_offset": row_offset,
518 "start_idx": start_idx
519 + len(start_marker)
520 + first_newline_idx
521 + len(UNIX_NEWLINE),
522 "end_idx": end_idx,
523 }
524
525 return multiline_rst