1from collections.abc import ByteString, Callable
  2import configparser
  3from pathlib import Path
  4from typing import TypedDict
  5from urllib.request import pathname2url
  6
  7from giturlparse import parse  # type: ignore[import-untyped]
  8from tree_sitter import Language, Parser, Point, Query, QueryCursor
  9from tree_sitter import Node as TreeSitterNode
 10
 11from sphinx_codelinks.config import UNIX_NEWLINE, CommentCategory
 12from sphinx_codelinks.logger import get_logger
 13from sphinx_codelinks.source_discover.config import CommentType
 14
 15# Language-specific node types for scope detection.
 16#
 17# YAML and JSONC are intentionally absent. They are data formats, not code, so a
 18# comment associates with the surrounding data structure (key/value pair, list
 19# item, or scalar) rather than with an enclosing or following declaration. That
 20# needs a different algorithm (inline same-row association first, scalar targets,
 21# grammar-specific traversal), implemented in find_yaml_associated_structure and
 22# find_jsonc_associated_structure and dispatched from find_associated_scope.
 23# Those bespoke finders never read this table (only find_next_scope and
 24# find_enclosing_scope do), so an entry here would be dead.
 25SCOPE_NODE_TYPES = {
[docs] 26    # @Python Scope Node Types, IMPL_PY_2, impl, [FE_PY]
 27    CommentType.python: {"function_definition", "class_definition"},
[docs] 28    # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP]
 29    CommentType.cpp: {"function_definition", "class_definition"},
 30    CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"},
 31    # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST];
 32    CommentType.rust: {
 33        "function_item",
 34        "struct_item",
 35        "enum_item",
 36        "impl_item",
 37        "trait_item",
 38        "mod_item",
 39    },
[docs] 40    # @Go Scope Node Types, IMPL_GO_4, impl, [FE_GO]
 41    CommentType.go: {
 42        "function_declaration",
 43        "method_declaration",
 44        "type_declaration",
 45        "type_spec",
 46    },
[docs] 47    # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH]
 48    CommentType.bash: {"function_definition"},
 49}
 50
 51logger = get_logger(__name__)
 52
 53GIT_HOST_URL_TEMPLATE = {
 54    "github": "https://github.com/{owner}/{repo}/blob/{rev}/{path}#L{lineno}",
 55    "gitlab": "https://gitlab.com/{owner}/{repo}/-/blob/{rev}/{path}#L{lineno}",
 56}
 57
 58PYTHON_QUERY = """
 59                ; Match comments
 60                (comment) @comment
 61
 62                ; Match docstrings inside modules, functions, or classes
 63                (module (expression_statement (string)) @comment)
 64                (function_definition (block (expression_statement (string)) @comment))
 65                (class_definition (block (expression_statement (string)) @comment))
 66            """
 67CPP_QUERY = """(comment) @comment"""
 68C_SHARP_QUERY = """(comment) @comment"""
 69YAML_QUERY = """(comment) @comment"""
 70RUST_QUERY = """
 71    (line_comment) @comment
 72    (block_comment) @comment
 73"""
[docs] 74# @Go comment query for tree-sitter, IMPL_GO_3, impl, [FE_GO]
 75GO_QUERY = """
 76    (comment) @comment
 77"""
 78JSONC_QUERY = """(comment) @comment"""
[docs] 79# @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH]
 80BASH_QUERY = """(comment) @comment"""
 81
 82# JSON value node types that can be associated with a comment.
 83JSON_STRUCTURE_TYPES = {
 84    "pair",
 85    "object",
 86    "array",
 87    "string",
 88    "number",
 89    "true",
 90    "false",
 91    "null",
 92}
 93
 94
 95def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
 96    """Return True if file is likely text, False if binary."""
 97    try:
 98        with filepath.open("rb") as f:
 99            chunk = f.read(sample_size)
100        # Quick binary heuristic: null byte present
101        if b"\x00" in chunk:
102            return False
103        # Try UTF-8 decode on the sample
104        chunk.decode("utf-8")
105        return True
106    except UnicodeDecodeError:
107        return False
108
109
[docs]110# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH]
111def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
112    if comment_type == CommentType.cpp:
113        import tree_sitter_cpp  # noqa: PLC0415
114
115        parsed_language = Language(tree_sitter_cpp.language())
116        query = Query(parsed_language, CPP_QUERY)
117    elif comment_type == CommentType.python:
118        import tree_sitter_python  # noqa: PLC0415
119
120        parsed_language = Language(tree_sitter_python.language())
121        query = Query(parsed_language, PYTHON_QUERY)
122    elif comment_type == CommentType.cs:
123        import tree_sitter_c_sharp  # noqa: PLC0415
124
125        parsed_language = Language(tree_sitter_c_sharp.language())
126        query = Query(parsed_language, C_SHARP_QUERY)
127    elif comment_type == CommentType.yaml:
128        import tree_sitter_yaml  # noqa: PLC0415
129
130        parsed_language = Language(tree_sitter_yaml.language())
131        query = Query(parsed_language, YAML_QUERY)
132    elif comment_type == CommentType.rust:
133        import tree_sitter_rust  # noqa: PLC0415
134
135        parsed_language = Language(tree_sitter_rust.language())
136        query = Query(parsed_language, RUST_QUERY)
137    elif comment_type == CommentType.go:
138        import tree_sitter_go  # noqa: PLC0415
139
140        parsed_language = Language(tree_sitter_go.language())
141        query = Query(parsed_language, GO_QUERY)
142    elif comment_type == CommentType.jsonc:
143        import tree_sitter_json  # noqa: PLC0415
144
145        parsed_language = Language(tree_sitter_json.language())
146        query = Query(parsed_language, JSONC_QUERY)
147    elif comment_type == CommentType.bash:
148        import tree_sitter_bash  # noqa: PLC0415
149
150        parsed_language = Language(tree_sitter_bash.language())
151        query = Query(parsed_language, BASH_QUERY)
152    else:
153        raise ValueError(f"Unsupported comment style: {comment_type}")
154    parser = Parser(parsed_language)
155    return parser, query
156
157
158def wrap_read_callable_point(
159    src_string: ByteString,
160) -> Callable[[int, Point], ByteString]:
161    def read_callable_byte_offset(byte_offset: int, _: Point) -> ByteString:
162        return src_string[byte_offset : byte_offset + 1]
163
164    return read_callable_byte_offset
165
166
[docs]167# @Comment extraction from source code using tree-sitter, IMPL_EXTR_1, impl, [FE_DEF]
168def extract_comments(
169    src_string: ByteString, parser: Parser, query: Query
170) -> list[TreeSitterNode] | None:
171    """Get all comments from source files by tree-sitter."""
172    read_point_fn = wrap_read_callable_point(src_string)
173    tree = parser.parse(read_point_fn)
174    query_cursor = QueryCursor(query)
175    captures: dict[str, list[TreeSitterNode]] = query_cursor.captures(tree.root_node)
176
177    return captures.get("comment")
178
179
180def find_enclosing_scope(
181    node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
182) -> TreeSitterNode | None:
183    """Find the enclosing scope of a comment."""
184    scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
185    current: TreeSitterNode = node
186    while current:
187        if current.type in scope_types:
188            return current
189        current: TreeSitterNode | None = current.parent  # type: ignore[no-redef]  # required for node traversal
190    return None
191
192
193def find_next_scope(
194    node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
195) -> TreeSitterNode | None:
196    """Find the next scope of a comment."""
197    scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
198    current: TreeSitterNode = node
199    while current:
200        if current.type in scope_types:
201            return current
202        current: TreeSitterNode | None = current.next_named_sibling  # type: ignore[no-redef]  # required for node traversal
203        if current and current.type == "block":
204            for child in current.named_children:
205                if child.type in scope_types:
206                    return child
207    return None
208
209
210def _find_yaml_structure_in_block_node(
211    block_node: TreeSitterNode,
212) -> TreeSitterNode | None:
213    """Find YAML structure elements within a block_node."""
214    for grandchild in block_node.named_children:
215        if grandchild.type == "block_mapping":
216            for ggchild in grandchild.named_children:
217                if ggchild.type == "block_mapping_pair":
218                    return ggchild
219        elif grandchild.type == "block_sequence":
220            for ggchild in grandchild.named_children:
221                if ggchild.type == "block_sequence_item":
222                    return ggchild
223    return None
224
225
226def find_yaml_next_structure(node: TreeSitterNode) -> TreeSitterNode | None:
227    """Find the next YAML structure element after the comment node."""
228    current = node.next_named_sibling
229    while current:
230        if current.type in {
231            "block_mapping_pair",
232            "block_sequence_item",
233            "flow_mapping",
234            "flow_sequence",
235        }:
236            return current
237        if current.type == "document":
238            for child in current.named_children:
239                if child.type == "block_node":
240                    result = _find_yaml_structure_in_block_node(child)
241                    if result:
242                        return result
243        if current.type == "block_node":
244            result = _find_yaml_structure_in_block_node(current)
245            if result:
246                return result
247        current = current.next_named_sibling
248    return None
249
250
251def find_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode | None:
252    """Find a previous named sibling that is on the same row as the comment.
253
254    Grammar-agnostic: used to detect inline comments in both YAML and JSONC.
255    """
256    comment_row = node.start_point.row
257    current = node.prev_named_sibling
258
259    while current:
260        # Check if this sibling ends on the same row as the comment starts
261        # This indicates it's an inline comment
262        if current.end_point.row == comment_row:
263            return current
264        # If we find a sibling that ends before the comment row, we can stop
265        # as we won't find any siblings on the same row going backwards
266        if current.end_point.row < comment_row:
267            break
268        current = current.prev_named_sibling
269
270    return None
271
272
273def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
274    """Find the YAML structure (key-value pair, list item, etc.) associated with a comment."""
275    # First, check if this is an inline comment by looking for a previous sibling on the same row
276    prev_sibling_same_row = find_prev_sibling_on_same_row(node)
277    if prev_sibling_same_row:
278        return prev_sibling_same_row
279
280    # If no previous sibling on same row, try to find the next named sibling (structure after the comment)
281    structure = find_yaml_next_structure(node)
282    if structure:
283        return structure
284
285    # If no next sibling found, traverse up to find parent structure
286    parent = node.parent
287    while parent:
288        if parent.type in {"block_mapping_pair", "block_sequence_item"}:
289            return parent
290        parent = parent.parent
291
292    return None
293
294
[docs]295# @JSONC comment-to-structure association, IMPL_JSONC_2, impl, [FE_JSONC]
296def find_jsonc_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
297    """Find the JSON structure (key/value pair, value, list item) for a comment.
298
299    JSON is data rather than code, so association follows the same intent as YAML:
300    an inline comment belongs to the value on its row, a leading comment belongs to
301    the following structure, otherwise it belongs to the enclosing structure.
302    """
303    # Inline comment: a value/pair on the same row, before the comment
304    prev_sibling_same_row = find_prev_sibling_on_same_row(node)
305    if prev_sibling_same_row:
306        return prev_sibling_same_row
307
308    # Leading comment: the next structure following the comment
309    current = node.next_named_sibling
310    while current:
311        if current.type in JSON_STRUCTURE_TYPES:
312            return current
313        current = current.next_named_sibling
314
315    # Otherwise: the enclosing structure
316    parent = node.parent
317    while parent:
318        if parent.type in {"pair", "object", "array"}:
319            return parent
320        parent = parent.parent
321
322    return None
323
324
325def find_associated_scope(
326    node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
327) -> TreeSitterNode | None:
328    """Find the associated scope of a comment."""
329    if comment_type == CommentType.yaml:
330        # YAML uses different structure association logic
331        return find_yaml_associated_structure(node)
332
333    if comment_type == CommentType.jsonc:
334        # JSONC uses data-aware structure association logic
335        return find_jsonc_associated_structure(node)
336
337    if node.type == CommentCategory.docstring:
338        # Only for python's docstring
339        return find_enclosing_scope(node, comment_type)
340    # General comments regardless of comment types
341    associated_scope = find_next_scope(node, comment_type)
342    if not associated_scope:
343        associated_scope = find_enclosing_scope(node, comment_type)
344    return associated_scope
345
346
347def locate_git_root(src_dir: Path) -> Path | None:
348    """Traverse upwards to find git root."""
349    current = src_dir.resolve()
350    parents = list(current.parents)
351    parents.append(current)
352    for parent in parents:
353        if (parent / ".git").exists() and (parent / ".git").is_dir():
354            return parent
355    logger.warning(
356        f"git root is not found in the parent of {src_dir}",
357        subtype="git_root",
358        location=str(src_dir),
359    )
360    return None
361
362
363def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
364    """Get remote url from .git/config."""
365    config_path = git_root / ".git" / "config"
366    if not config_path.exists():
367        logger.warning(
368            f"{config_path} does not exist",
369            subtype="git_config",
370            location=str(config_path),
371        )
372        return None
373
374    config = configparser.ConfigParser(allow_no_value=True, strict=False)
375    config.read(config_path)
376    section = f'remote "{remote_name}"'
377    if section in config and "url" in config[section]:
378        url: str = config[section]["url"]
379        return url
380    logger.warning(
381        f"remote-url is not found in {config_path}",
382        subtype="git_remote",
383        location=str(config_path),
384    )
385    return None
386
387
388def get_current_rev(git_root: Path) -> str | None:
389    """Get current commit rev from .git/HEAD."""
390    head_path = git_root / ".git" / "HEAD"
391    if not head_path.exists():
392        logger.warning(
393            f"{head_path} does not exist",
394            subtype="git_head",
395            location=str(head_path),
396        )
397        return None
398    head_content = head_path.read_text().strip()
399    if not head_content.startswith("ref: "):
400        # Detached HEAD (e.g. CI checkouts): .git/HEAD holds the commit SHA
401        # directly, which is exactly the rev we want.
402        return head_content
403
404    ref_path = git_root / ".git" / head_content.split(":", 1)[1].strip()
405    if not ref_path.exists():
406        logger.warning(
407            f"{ref_path} does not exist",
408            subtype="git_ref",
409            location=str(ref_path),
410        )
411        return None
412    return ref_path.read_text().strip()
413
414
415def form_https_url(
416    git_url: str, rev: str, project_path: Path, filepath: Path, lineno: int
417) -> str | None:
418    parsed_url = parse(git_url)
419    template = GIT_HOST_URL_TEMPLATE.get(parsed_url.platform)
420    if not template:
421        logger.warning(
422            f"Unsupported Git host: {parsed_url.platform}",
423            subtype="git_host",
424        )
425        return git_url
426    https_url = template.format(
427        owner=parsed_url.owner,
428        repo=parsed_url.repo,
429        rev=rev,
430        path=pathname2url(str(filepath.absolute().relative_to(project_path))),
431        lineno=str(lineno),
432    )
433    return https_url
434
435
436def remove_leading_sequences(text: str, leading_sequences: list[str]) -> str:
437    lines = text.splitlines(keepends=True)
438    no_comment_lines = []
439    for line in lines:
440        leading_sequence_exist = False
441        for leading_sequence in leading_sequences:
442            leading_sequence_idx = line.find(leading_sequence)
443            if leading_sequence_idx == -1:
444                continue
445            no_comment_lines.append(
446                line[leading_sequence_idx + len(leading_sequence) :]
447            )
448            leading_sequence_exist = True
449            break
450
451        if not leading_sequence_exist:
452            no_comment_lines.append(line)
453
454    return "".join(no_comment_lines)
455
456
457class ExtractedRstType(TypedDict):
458    rst_text: str
459    row_offset: int
460    start_idx: int
461    end_idx: int
462
463
[docs]464# @Extract reStructuredText blocks embedded in comments, IMPL_RST_1, impl, [FE_RST_EXTRACTION]
465def extract_rst(
466    text: str, start_marker: str, end_marker: str
467) -> ExtractedRstType | None:
468    """Extract rst from a comment.
469
470    Two use cases:
471    1. Start_marker and end_marker one the same line.
472
473    The rst text is wrapped by start and the end markers on the same line,
474    so, there is no need to remove the leading chars.ArithmeticError
475    E.g.
476    @rst  .. admonition:: title here @endrst
477
478    2. Start_marker and end_marker in different lines.
479
480    The rst text is expected to start from the next line of the start_marker
481    and ends at he previous line of the end_marker.
482    E.g.
483    @rst
484    .. admonition:: title here
485      :collapsible: open
486
487      This example is collapsible, and initially open.
488    @endrst
489    """
490    start_idx = text.find(start_marker)
491    end_idx = text.rfind(end_marker)
492    if start_idx == -1 or end_idx == -1:
493        return None
494    rst_text = text[start_idx + len(start_marker) : end_idx]
495    row_offset = len(text[:start_idx].splitlines())
496    if not rst_text.strip():
497        # empty string is out of the interest
498        return None
499    if UNIX_NEWLINE not in rst_text:
500        # single line rst text
501        oneline_rst: ExtractedRstType = {
502            "rst_text": rst_text,
503            "row_offset": row_offset,
504            "start_idx": start_idx + len(start_marker),
505            "end_idx": end_idx,
506        }
507        return oneline_rst
508
509    # multiline rst text
510
511    first_newline_idx = rst_text.find(UNIX_NEWLINE)
512    rst_text = rst_text[first_newline_idx + len(UNIX_NEWLINE) :]
513    multiline_rst: ExtractedRstType = {
514        "rst_text": rst_text,
515        "row_offset": row_offset,
516        "start_idx": start_idx
517        + len(start_marker)
518        + first_newline_idx
519        + len(UNIX_NEWLINE),
520        "end_idx": end_idx,
521    }
522
523    return multiline_rst