1from collections.abc import ByteString, Callable
2import configparser
3from pathlib import Path
4from typing import TypedDict
5from urllib.request import pathname2url
6
7from giturlparse import parse # type: ignore[import-untyped]
8from tree_sitter import Language, Parser, Point, Query, QueryCursor
9from tree_sitter import Node as TreeSitterNode
10
11from sphinx_codelinks.config import UNIX_NEWLINE, CommentCategory
12from sphinx_codelinks.logger import get_logger
13from sphinx_codelinks.source_discover.config import CommentType
14
15# Language-specific node types for scope detection.
16#
17# YAML and JSONC are intentionally absent. They are data formats, not code, so a
18# comment associates with the surrounding data structure (key/value pair, list
19# item, or scalar) rather than with an enclosing or following declaration. That
20# needs a different algorithm (inline same-row association first, scalar targets,
21# grammar-specific traversal), implemented in find_yaml_associated_structure and
22# find_jsonc_associated_structure and dispatched from find_associated_scope.
23# Those bespoke finders never read this table (only find_next_scope and
24# find_enclosing_scope do), so an entry here would be dead.
25SCOPE_NODE_TYPES = {
[docs] 26 # @Python Scope Node Types, IMPL_PY_2, impl, [FE_PY]
27 CommentType.python: {"function_definition", "class_definition"},
[docs] 28 # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP]
29 CommentType.cpp: {"function_definition", "class_definition"},
30 CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"},
31 # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST];
32 CommentType.rust: {
33 "function_item",
34 "struct_item",
35 "enum_item",
36 "impl_item",
37 "trait_item",
38 "mod_item",
39 },
[docs] 40 # @Go Scope Node Types, IMPL_GO_4, impl, [FE_GO]
41 CommentType.go: {
42 "function_declaration",
43 "method_declaration",
44 "type_declaration",
45 "type_spec",
46 },
[docs] 47 # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH]
48 CommentType.bash: {"function_definition"},
49}
50
51logger = get_logger(__name__)
52
53GIT_HOST_URL_TEMPLATE = {
54 "github": "https://github.com/{owner}/{repo}/blob/{rev}/{path}#L{lineno}",
55 "gitlab": "https://gitlab.com/{owner}/{repo}/-/blob/{rev}/{path}#L{lineno}",
56}
57
58PYTHON_QUERY = """
59 ; Match comments
60 (comment) @comment
61
62 ; Match docstrings inside modules, functions, or classes
63 (module (expression_statement (string)) @comment)
64 (function_definition (block (expression_statement (string)) @comment))
65 (class_definition (block (expression_statement (string)) @comment))
66 """
67CPP_QUERY = """(comment) @comment"""
68C_SHARP_QUERY = """(comment) @comment"""
69YAML_QUERY = """(comment) @comment"""
70RUST_QUERY = """
71 (line_comment) @comment
72 (block_comment) @comment
73"""
[docs] 74# @Go comment query for tree-sitter, IMPL_GO_3, impl, [FE_GO]
75GO_QUERY = """
76 (comment) @comment
77"""
78JSONC_QUERY = """(comment) @comment"""
[docs] 79# @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH]
80BASH_QUERY = """(comment) @comment"""
81
82# JSON value node types that can be associated with a comment.
83JSON_STRUCTURE_TYPES = {
84 "pair",
85 "object",
86 "array",
87 "string",
88 "number",
89 "true",
90 "false",
91 "null",
92}
93
94
95def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
96 """Return True if file is likely text, False if binary."""
97 try:
98 with filepath.open("rb") as f:
99 chunk = f.read(sample_size)
100 # Quick binary heuristic: null byte present
101 if b"\x00" in chunk:
102 return False
103 # Try UTF-8 decode on the sample
104 chunk.decode("utf-8")
105 return True
106 except UnicodeDecodeError:
107 return False
108
109
[docs]110# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH]
111def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
112 if comment_type == CommentType.cpp:
113 import tree_sitter_cpp # noqa: PLC0415
114
115 parsed_language = Language(tree_sitter_cpp.language())
116 query = Query(parsed_language, CPP_QUERY)
117 elif comment_type == CommentType.python:
118 import tree_sitter_python # noqa: PLC0415
119
120 parsed_language = Language(tree_sitter_python.language())
121 query = Query(parsed_language, PYTHON_QUERY)
122 elif comment_type == CommentType.cs:
123 import tree_sitter_c_sharp # noqa: PLC0415
124
125 parsed_language = Language(tree_sitter_c_sharp.language())
126 query = Query(parsed_language, C_SHARP_QUERY)
127 elif comment_type == CommentType.yaml:
128 import tree_sitter_yaml # noqa: PLC0415
129
130 parsed_language = Language(tree_sitter_yaml.language())
131 query = Query(parsed_language, YAML_QUERY)
132 elif comment_type == CommentType.rust:
133 import tree_sitter_rust # noqa: PLC0415
134
135 parsed_language = Language(tree_sitter_rust.language())
136 query = Query(parsed_language, RUST_QUERY)
137 elif comment_type == CommentType.go:
138 import tree_sitter_go # noqa: PLC0415
139
140 parsed_language = Language(tree_sitter_go.language())
141 query = Query(parsed_language, GO_QUERY)
142 elif comment_type == CommentType.jsonc:
143 import tree_sitter_json # noqa: PLC0415
144
145 parsed_language = Language(tree_sitter_json.language())
146 query = Query(parsed_language, JSONC_QUERY)
147 elif comment_type == CommentType.bash:
148 import tree_sitter_bash # noqa: PLC0415
149
150 parsed_language = Language(tree_sitter_bash.language())
151 query = Query(parsed_language, BASH_QUERY)
152 else:
153 raise ValueError(f"Unsupported comment style: {comment_type}")
154 parser = Parser(parsed_language)
155 return parser, query
156
157
158def wrap_read_callable_point(
159 src_string: ByteString,
160) -> Callable[[int, Point], ByteString]:
161 def read_callable_byte_offset(byte_offset: int, _: Point) -> ByteString:
162 return src_string[byte_offset : byte_offset + 1]
163
164 return read_callable_byte_offset
165
166
[docs]167# @Comment extraction from source code using tree-sitter, IMPL_EXTR_1, impl, [FE_DEF]
168def extract_comments(
169 src_string: ByteString, parser: Parser, query: Query
170) -> list[TreeSitterNode] | None:
171 """Get all comments from source files by tree-sitter."""
172 read_point_fn = wrap_read_callable_point(src_string)
173 tree = parser.parse(read_point_fn)
174 query_cursor = QueryCursor(query)
175 captures: dict[str, list[TreeSitterNode]] = query_cursor.captures(tree.root_node)
176
177 return captures.get("comment")
178
179
180def find_enclosing_scope(
181 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
182) -> TreeSitterNode | None:
183 """Find the enclosing scope of a comment."""
184 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
185 current: TreeSitterNode = node
186 while current:
187 if current.type in scope_types:
188 return current
189 current: TreeSitterNode | None = current.parent # type: ignore[no-redef] # required for node traversal
190 return None
191
192
193def find_next_scope(
194 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
195) -> TreeSitterNode | None:
196 """Find the next scope of a comment."""
197 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
198 current: TreeSitterNode = node
199 while current:
200 if current.type in scope_types:
201 return current
202 current: TreeSitterNode | None = current.next_named_sibling # type: ignore[no-redef] # required for node traversal
203 if current and current.type == "block":
204 for child in current.named_children:
205 if child.type in scope_types:
206 return child
207 return None
208
209
210def _find_yaml_structure_in_block_node(
211 block_node: TreeSitterNode,
212) -> TreeSitterNode | None:
213 """Find YAML structure elements within a block_node."""
214 for grandchild in block_node.named_children:
215 if grandchild.type == "block_mapping":
216 for ggchild in grandchild.named_children:
217 if ggchild.type == "block_mapping_pair":
218 return ggchild
219 elif grandchild.type == "block_sequence":
220 for ggchild in grandchild.named_children:
221 if ggchild.type == "block_sequence_item":
222 return ggchild
223 return None
224
225
226def find_yaml_next_structure(node: TreeSitterNode) -> TreeSitterNode | None:
227 """Find the next YAML structure element after the comment node."""
228 current = node.next_named_sibling
229 while current:
230 if current.type in {
231 "block_mapping_pair",
232 "block_sequence_item",
233 "flow_mapping",
234 "flow_sequence",
235 }:
236 return current
237 if current.type == "document":
238 for child in current.named_children:
239 if child.type == "block_node":
240 result = _find_yaml_structure_in_block_node(child)
241 if result:
242 return result
243 if current.type == "block_node":
244 result = _find_yaml_structure_in_block_node(current)
245 if result:
246 return result
247 current = current.next_named_sibling
248 return None
249
250
251def find_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode | None:
252 """Find a previous named sibling that is on the same row as the comment.
253
254 Grammar-agnostic: used to detect inline comments in both YAML and JSONC.
255 """
256 comment_row = node.start_point.row
257 current = node.prev_named_sibling
258
259 while current:
260 # Check if this sibling ends on the same row as the comment starts
261 # This indicates it's an inline comment
262 if current.end_point.row == comment_row:
263 return current
264 # If we find a sibling that ends before the comment row, we can stop
265 # as we won't find any siblings on the same row going backwards
266 if current.end_point.row < comment_row:
267 break
268 current = current.prev_named_sibling
269
270 return None
271
272
273def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
274 """Find the YAML structure (key-value pair, list item, etc.) associated with a comment."""
275 # First, check if this is an inline comment by looking for a previous sibling on the same row
276 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
277 if prev_sibling_same_row:
278 return prev_sibling_same_row
279
280 # If no previous sibling on same row, try to find the next named sibling (structure after the comment)
281 structure = find_yaml_next_structure(node)
282 if structure:
283 return structure
284
285 # If no next sibling found, traverse up to find parent structure
286 parent = node.parent
287 while parent:
288 if parent.type in {"block_mapping_pair", "block_sequence_item"}:
289 return parent
290 parent = parent.parent
291
292 return None
293
294
[docs]295# @JSONC comment-to-structure association, IMPL_JSONC_2, impl, [FE_JSONC]
296def find_jsonc_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
297 """Find the JSON structure (key/value pair, value, list item) for a comment.
298
299 JSON is data rather than code, so association follows the same intent as YAML:
300 an inline comment belongs to the value on its row, a leading comment belongs to
301 the following structure, otherwise it belongs to the enclosing structure.
302 """
303 # Inline comment: a value/pair on the same row, before the comment
304 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
305 if prev_sibling_same_row:
306 return prev_sibling_same_row
307
308 # Leading comment: the next structure following the comment
309 current = node.next_named_sibling
310 while current:
311 if current.type in JSON_STRUCTURE_TYPES:
312 return current
313 current = current.next_named_sibling
314
315 # Otherwise: the enclosing structure
316 parent = node.parent
317 while parent:
318 if parent.type in {"pair", "object", "array"}:
319 return parent
320 parent = parent.parent
321
322 return None
323
324
325def find_associated_scope(
326 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
327) -> TreeSitterNode | None:
328 """Find the associated scope of a comment."""
329 if comment_type == CommentType.yaml:
330 # YAML uses different structure association logic
331 return find_yaml_associated_structure(node)
332
333 if comment_type == CommentType.jsonc:
334 # JSONC uses data-aware structure association logic
335 return find_jsonc_associated_structure(node)
336
337 if node.type == CommentCategory.docstring:
338 # Only for python's docstring
339 return find_enclosing_scope(node, comment_type)
340 # General comments regardless of comment types
341 associated_scope = find_next_scope(node, comment_type)
342 if not associated_scope:
343 associated_scope = find_enclosing_scope(node, comment_type)
344 return associated_scope
345
346
347def locate_git_root(src_dir: Path) -> Path | None:
348 """Traverse upwards to find git root."""
349 current = src_dir.resolve()
350 parents = list(current.parents)
351 parents.append(current)
352 for parent in parents:
353 if (parent / ".git").exists() and (parent / ".git").is_dir():
354 return parent
355 logger.warning(
356 f"git root is not found in the parent of {src_dir}",
357 subtype="git_root",
358 location=str(src_dir),
359 )
360 return None
361
362
363def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
364 """Get remote url from .git/config."""
365 config_path = git_root / ".git" / "config"
366 if not config_path.exists():
367 logger.warning(
368 f"{config_path} does not exist",
369 subtype="git_config",
370 location=str(config_path),
371 )
372 return None
373
374 config = configparser.ConfigParser(allow_no_value=True, strict=False)
375 config.read(config_path)
376 section = f'remote "{remote_name}"'
377 if section in config and "url" in config[section]:
378 url: str = config[section]["url"]
379 return url
380 logger.warning(
381 f"remote-url is not found in {config_path}",
382 subtype="git_remote",
383 location=str(config_path),
384 )
385 return None
386
387
388def get_current_rev(git_root: Path) -> str | None:
389 """Get current commit rev from .git/HEAD."""
390 head_path = git_root / ".git" / "HEAD"
391 if not head_path.exists():
392 logger.warning(
393 f"{head_path} does not exist",
394 subtype="git_head",
395 location=str(head_path),
396 )
397 return None
398 head_content = head_path.read_text().strip()
399 if not head_content.startswith("ref: "):
400 # Detached HEAD (e.g. CI checkouts): .git/HEAD holds the commit SHA
401 # directly, which is exactly the rev we want.
402 return head_content
403
404 ref_path = git_root / ".git" / head_content.split(":", 1)[1].strip()
405 if not ref_path.exists():
406 logger.warning(
407 f"{ref_path} does not exist",
408 subtype="git_ref",
409 location=str(ref_path),
410 )
411 return None
412 return ref_path.read_text().strip()
413
414
415def form_https_url(
416 git_url: str, rev: str, project_path: Path, filepath: Path, lineno: int
417) -> str | None:
418 parsed_url = parse(git_url)
419 template = GIT_HOST_URL_TEMPLATE.get(parsed_url.platform)
420 if not template:
421 logger.warning(
422 f"Unsupported Git host: {parsed_url.platform}",
423 subtype="git_host",
424 )
425 return git_url
426 https_url = template.format(
427 owner=parsed_url.owner,
428 repo=parsed_url.repo,
429 rev=rev,
430 path=pathname2url(str(filepath.absolute().relative_to(project_path))),
431 lineno=str(lineno),
432 )
433 return https_url
434
435
436def remove_leading_sequences(text: str, leading_sequences: list[str]) -> str:
437 lines = text.splitlines(keepends=True)
438 no_comment_lines = []
439 for line in lines:
440 leading_sequence_exist = False
441 for leading_sequence in leading_sequences:
442 leading_sequence_idx = line.find(leading_sequence)
443 if leading_sequence_idx == -1:
444 continue
445 no_comment_lines.append(
446 line[leading_sequence_idx + len(leading_sequence) :]
447 )
448 leading_sequence_exist = True
449 break
450
451 if not leading_sequence_exist:
452 no_comment_lines.append(line)
453
454 return "".join(no_comment_lines)
455
456
457class ExtractedRstType(TypedDict):
458 rst_text: str
459 row_offset: int
460 start_idx: int
461 end_idx: int
462
463
[docs]464# @Extract reStructuredText blocks embedded in comments, IMPL_RST_1, impl, [FE_RST_EXTRACTION]
465def extract_rst(
466 text: str, start_marker: str, end_marker: str
467) -> ExtractedRstType | None:
468 """Extract rst from a comment.
469
470 Two use cases:
471 1. Start_marker and end_marker one the same line.
472
473 The rst text is wrapped by start and the end markers on the same line,
474 so, there is no need to remove the leading chars.ArithmeticError
475 E.g.
476 @rst .. admonition:: title here @endrst
477
478 2. Start_marker and end_marker in different lines.
479
480 The rst text is expected to start from the next line of the start_marker
481 and ends at he previous line of the end_marker.
482 E.g.
483 @rst
484 .. admonition:: title here
485 :collapsible: open
486
487 This example is collapsible, and initially open.
488 @endrst
489 """
490 start_idx = text.find(start_marker)
491 end_idx = text.rfind(end_marker)
492 if start_idx == -1 or end_idx == -1:
493 return None
494 rst_text = text[start_idx + len(start_marker) : end_idx]
495 row_offset = len(text[:start_idx].splitlines())
496 if not rst_text.strip():
497 # empty string is out of the interest
498 return None
499 if UNIX_NEWLINE not in rst_text:
500 # single line rst text
501 oneline_rst: ExtractedRstType = {
502 "rst_text": rst_text,
503 "row_offset": row_offset,
504 "start_idx": start_idx + len(start_marker),
505 "end_idx": end_idx,
506 }
507 return oneline_rst
508
509 # multiline rst text
510
511 first_newline_idx = rst_text.find(UNIX_NEWLINE)
512 rst_text = rst_text[first_newline_idx + len(UNIX_NEWLINE) :]
513 multiline_rst: ExtractedRstType = {
514 "rst_text": rst_text,
515 "row_offset": row_offset,
516 "start_idx": start_idx
517 + len(start_marker)
518 + first_newline_idx
519 + len(UNIX_NEWLINE),
520 "end_idx": end_idx,
521 }
522
523 return multiline_rst