1import configparser
  2from collections.abc import ByteString, Callable
  3from pathlib import Path
  4from typing import TypedDict
  5from urllib.request import pathname2url
  6
  7from giturlparse import parse
  8from tree_sitter import Language, Parser, Point, Query, QueryCursor
  9from tree_sitter import Node as TreeSitterNode
 10
 11from sphinx_codelinks.config import UNIX_NEWLINE, CommentCategory
 12from sphinx_codelinks.logger import get_logger
 13from sphinx_codelinks.source_discover.config import CommentType
 14
 15# Language-specific node types for scope detection.
 16#
 17# YAML and JSONC are intentionally absent. They are data formats, not code, so a
 18# comment associates with the surrounding data structure (key/value pair, list
 19# item, or scalar) rather than with an enclosing or following declaration. That
 20# needs a different algorithm (inline same-row association first, scalar targets,
 21# grammar-specific traversal), implemented in find_yaml_associated_structure and
 22# find_jsonc_associated_structure and dispatched from find_associated_scope.
 23# Those bespoke finders never read this table (only find_next_scope and
 24# find_enclosing_scope do), so an entry here would be dead.
 25SCOPE_NODE_TYPES = {
[docs] 26    # @Python Scope Node Types, IMPL_PY_2, impl, [FE_PY]
 27    CommentType.python: {"function_definition", "class_definition"},
[docs] 28    # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP]
 29    CommentType.cpp: {"function_definition", "class_definition"},
 30    CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"},
 31    # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST];
 32    CommentType.rust: {
 33        "function_item",
 34        "struct_item",
 35        "enum_item",
 36        "impl_item",
 37        "trait_item",
 38        "mod_item",
 39    },
[docs] 40    # @Go Scope Node Types, IMPL_GO_4, impl, [FE_GO]
 41    CommentType.go: {
 42        "function_declaration",
 43        "method_declaration",
 44        "type_declaration",
 45        "type_spec",
 46    },
[docs] 47    # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH]
 48    CommentType.bash: {"function_definition"},
 49}
 50
 51logger = get_logger(__name__)
 52
 53GIT_HOST_URL_TEMPLATE = {
 54    "github": "https://github.com/{owner}/{repo}/blob/{rev}/{path}#L{lineno}",
 55    "gitlab": "https://gitlab.com/{owner}/{repo}/-/blob/{rev}/{path}#L{lineno}",
 56}
 57
 58PYTHON_QUERY = """
 59                ; Match comments
 60                (comment) @comment
 61
 62                ; Match docstrings inside modules, functions, or classes
 63                (module (expression_statement (string)) @comment)
 64                (function_definition (block (expression_statement (string)) @comment))
 65                (class_definition (block (expression_statement (string)) @comment))
 66            """
 67CPP_QUERY = """(comment) @comment"""
 68C_SHARP_QUERY = """(comment) @comment"""
 69YAML_QUERY = """(comment) @comment"""
 70RUST_QUERY = """
 71    (line_comment) @comment
 72    (block_comment) @comment
 73"""
[docs] 74# @Go comment query for tree-sitter, IMPL_GO_3, impl, [FE_GO]
 75GO_QUERY = """
 76    (comment) @comment
 77"""
 78JSONC_QUERY = """(comment) @comment"""
[docs] 79# @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH]
 80BASH_QUERY = """(comment) @comment"""
 81
 82# JSON value node types that can be associated with a comment.
 83JSON_STRUCTURE_TYPES = {
 84    "pair",
 85    "object",
 86    "array",
 87    "string",
 88    "number",
 89    "true",
 90    "false",
 91    "null",
 92}
 93
 94
 95def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
 96    """Return True if file is likely text, False if binary."""
 97    try:
 98        with filepath.open("rb") as f:
 99            chunk = f.read(sample_size)
100        # Quick binary heuristic: null byte present
101        if b"\x00" in chunk:
102            return False
103        # Try UTF-8 decode on the sample
104        chunk.decode("utf-8")
105        return True
106    except UnicodeDecodeError:
107        return False
108
109
[docs]110# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH]
111def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
112    if comment_type == CommentType.cpp:
113        import tree_sitter_cpp
114
115        parsed_language = Language(tree_sitter_cpp.language())
116        query = Query(parsed_language, CPP_QUERY)
117    elif comment_type == CommentType.python:
118        import tree_sitter_python
119
120        parsed_language = Language(tree_sitter_python.language())
121        query = Query(parsed_language, PYTHON_QUERY)
122    elif comment_type == CommentType.cs:
123        import tree_sitter_c_sharp
124
125        parsed_language = Language(tree_sitter_c_sharp.language())
126        query = Query(parsed_language, C_SHARP_QUERY)
127    elif comment_type == CommentType.yaml:
128        import tree_sitter_yaml
129
130        parsed_language = Language(tree_sitter_yaml.language())
131        query = Query(parsed_language, YAML_QUERY)
132    elif comment_type == CommentType.rust:
133        import tree_sitter_rust
134
135        parsed_language = Language(tree_sitter_rust.language())
136        query = Query(parsed_language, RUST_QUERY)
137    elif comment_type == CommentType.go:
138        import tree_sitter_go
139
140        parsed_language = Language(tree_sitter_go.language())
141        query = Query(parsed_language, GO_QUERY)
142    elif comment_type == CommentType.jsonc:
143        import tree_sitter_json
144
145        parsed_language = Language(tree_sitter_json.language())
146        query = Query(parsed_language, JSONC_QUERY)
147    elif comment_type == CommentType.bash:
148        import tree_sitter_bash
149
150        parsed_language = Language(tree_sitter_bash.language())
151        query = Query(parsed_language, BASH_QUERY)
152    else:
153        raise ValueError(f"Unsupported comment style: {comment_type}")
154    parser = Parser(parsed_language)
155    return parser, query
156
157
158def wrap_read_callable_point(
159    src_string: ByteString,
160) -> Callable[[int, Point], ByteString]:
161    def read_callable_byte_offset(byte_offset: int, _: Point) -> ByteString:
162        return src_string[byte_offset : byte_offset + 1]
163
164    return read_callable_byte_offset
165
166
[docs]167# @Comment extraction from source code using tree-sitter, IMPL_EXTR_1, impl, [FE_DEF]
168def extract_comments(
169    src_string: ByteString, parser: Parser, query: Query
170) -> list[TreeSitterNode] | None:
171    """Get all comments from source files by tree-sitter."""
172    read_point_fn = wrap_read_callable_point(src_string)
173    tree = parser.parse(read_point_fn)
174    query_cursor = QueryCursor(query)
175    captures: dict[str, list[TreeSitterNode]] = query_cursor.captures(tree.root_node)
176
177    return captures.get("comment")
178
179
180def find_enclosing_scope(
181    node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
182) -> TreeSitterNode | None:
183    """Find the enclosing scope of a comment."""
184    scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
185    current: TreeSitterNode = node
186    while current:
187        if current.type in scope_types:
188            return current
189        current: TreeSitterNode | None = current.parent  # required for node traversal
190    return None
191
192
193def find_next_scope(
194    node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
195) -> TreeSitterNode | None:
196    """Find the next scope of a comment."""
197    scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
198    current: TreeSitterNode = node
199    while current:
200        if current.type in scope_types:
201            return current
202        current: TreeSitterNode | None = (
203            current.next_named_sibling
204        )  # required for node traversal
205        if current and current.type == "block":
206            for child in current.named_children:
207                if child.type in scope_types:
208                    return child
209    return None
210
211
212def _find_yaml_structure_in_block_node(
213    block_node: TreeSitterNode,
214) -> TreeSitterNode | None:
215    """Find YAML structure elements within a block_node."""
216    for grandchild in block_node.named_children:
217        if grandchild.type == "block_mapping":
218            for ggchild in grandchild.named_children:
219                if ggchild.type == "block_mapping_pair":
220                    return ggchild
221        elif grandchild.type == "block_sequence":
222            for ggchild in grandchild.named_children:
223                if ggchild.type == "block_sequence_item":
224                    return ggchild
225    return None
226
227
228def find_yaml_next_structure(node: TreeSitterNode) -> TreeSitterNode | None:
229    """Find the next YAML structure element after the comment node."""
230    current = node.next_named_sibling
231    while current:
232        if current.type in {
233            "block_mapping_pair",
234            "block_sequence_item",
235            "flow_mapping",
236            "flow_sequence",
237        }:
238            return current
239        if current.type == "document":
240            for child in current.named_children:
241                if child.type == "block_node":
242                    result = _find_yaml_structure_in_block_node(child)
243                    if result:
244                        return result
245        if current.type == "block_node":
246            result = _find_yaml_structure_in_block_node(current)
247            if result:
248                return result
249        current = current.next_named_sibling
250    return None
251
252
253def find_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode | None:
254    """Find a previous named sibling that is on the same row as the comment.
255
256    Grammar-agnostic: used to detect inline comments in both YAML and JSONC.
257    """
258    comment_row = node.start_point.row
259    current = node.prev_named_sibling
260
261    while current:
262        # Check if this sibling ends on the same row as the comment starts
263        # This indicates it's an inline comment
264        if current.end_point.row == comment_row:
265            return current
266        # If we find a sibling that ends before the comment row, we can stop
267        # as we won't find any siblings on the same row going backwards
268        if current.end_point.row < comment_row:
269            break
270        current = current.prev_named_sibling
271
272    return None
273
274
275def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
276    """Find the YAML structure (key-value pair, list item, etc.) associated with a comment."""
277    # First, check if this is an inline comment by looking for a previous sibling on the same row
278    prev_sibling_same_row = find_prev_sibling_on_same_row(node)
279    if prev_sibling_same_row:
280        return prev_sibling_same_row
281
282    # If no previous sibling on same row, try to find the next named sibling (structure after the comment)
283    structure = find_yaml_next_structure(node)
284    if structure:
285        return structure
286
287    # If no next sibling found, traverse up to find parent structure
288    parent = node.parent
289    while parent:
290        if parent.type in {"block_mapping_pair", "block_sequence_item"}:
291            return parent
292        parent = parent.parent
293
294    return None
295
296
[docs]297# @JSONC comment-to-structure association, IMPL_JSONC_2, impl, [FE_JSONC]
298def find_jsonc_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
299    """Find the JSON structure (key/value pair, value, list item) for a comment.
300
301    JSON is data rather than code, so association follows the same intent as YAML:
302    an inline comment belongs to the value on its row, a leading comment belongs to
303    the following structure, otherwise it belongs to the enclosing structure.
304    """
305    # Inline comment: a value/pair on the same row, before the comment
306    prev_sibling_same_row = find_prev_sibling_on_same_row(node)
307    if prev_sibling_same_row:
308        return prev_sibling_same_row
309
310    # Leading comment: the next structure following the comment
311    current = node.next_named_sibling
312    while current:
313        if current.type in JSON_STRUCTURE_TYPES:
314            return current
315        current = current.next_named_sibling
316
317    # Otherwise: the enclosing structure
318    parent = node.parent
319    while parent:
320        if parent.type in {"pair", "object", "array"}:
321            return parent
322        parent = parent.parent
323
324    return None
325
326
327def find_associated_scope(
328    node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
329) -> TreeSitterNode | None:
330    """Find the associated scope of a comment."""
331    if comment_type == CommentType.yaml:
332        # YAML uses different structure association logic
333        return find_yaml_associated_structure(node)
334
335    if comment_type == CommentType.jsonc:
336        # JSONC uses data-aware structure association logic
337        return find_jsonc_associated_structure(node)
338
339    if node.type == CommentCategory.docstring:
340        # Only for python's docstring
341        return find_enclosing_scope(node, comment_type)
342    # General comments regardless of comment types
343    associated_scope = find_next_scope(node, comment_type)
344    if not associated_scope:
345        associated_scope = find_enclosing_scope(node, comment_type)
346    return associated_scope
347
348
349def locate_git_root(src_dir: Path) -> Path | None:
350    """Traverse upwards to find git root."""
351    current = src_dir.resolve()
352    parents = list(current.parents)
353    parents.append(current)
354    for parent in parents:
355        if (parent / ".git").exists() and (parent / ".git").is_dir():
356            return parent
357    logger.warning(
358        f"git root is not found in the parent of {src_dir}",
359        subtype="git_root",
360        location=str(src_dir),
361    )
362    return None
363
364
365def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
366    """Get remote url from .git/config."""
367    config_path = git_root / ".git" / "config"
368    if not config_path.exists():
369        logger.warning(
370            f"{config_path} does not exist",
371            subtype="git_config",
372            location=str(config_path),
373        )
374        return None
375
376    config = configparser.ConfigParser(allow_no_value=True, strict=False)
377    config.read(config_path)
378    section = f'remote "{remote_name}"'
379    if section in config and "url" in config[section]:
380        url: str = config[section]["url"]
381        return url
382    logger.warning(
383        f"remote-url is not found in {config_path}",
384        subtype="git_remote",
385        location=str(config_path),
386    )
387    return None
388
389
390def get_current_rev(git_root: Path) -> str | None:
391    """Get current commit rev from .git/HEAD."""
392    head_path = git_root / ".git" / "HEAD"
393    if not head_path.exists():
394        logger.warning(
395            f"{head_path} does not exist",
396            subtype="git_head",
397            location=str(head_path),
398        )
399        return None
400    head_content = head_path.read_text().strip()
401    if not head_content.startswith("ref: "):
402        # Detached HEAD (e.g. CI checkouts): .git/HEAD holds the commit SHA
403        # directly, which is exactly the rev we want.
404        return head_content
405
406    ref_path = git_root / ".git" / head_content.split(":", 1)[1].strip()
407    if not ref_path.exists():
408        logger.warning(
409            f"{ref_path} does not exist",
410            subtype="git_ref",
411            location=str(ref_path),
412        )
413        return None
414    return ref_path.read_text().strip()
415
416
417def form_https_url(
418    git_url: str, rev: str, project_path: Path, filepath: Path, lineno: int
419) -> str | None:
420    parsed_url = parse(git_url)
421    template = GIT_HOST_URL_TEMPLATE.get(parsed_url.platform)
422    if not template:
423        logger.warning(
424            f"Unsupported Git host: {parsed_url.platform}",
425            subtype="git_host",
426        )
427        return git_url
428    https_url = template.format(
429        owner=parsed_url.owner,
430        repo=parsed_url.repo,
431        rev=rev,
432        path=pathname2url(str(filepath.absolute().relative_to(project_path))),
433        lineno=str(lineno),
434    )
435    return https_url
436
437
438def remove_leading_sequences(text: str, leading_sequences: list[str]) -> str:
439    lines = text.splitlines(keepends=True)
440    no_comment_lines = []
441    for line in lines:
442        leading_sequence_exist = False
443        for leading_sequence in leading_sequences:
444            leading_sequence_idx = line.find(leading_sequence)
445            if leading_sequence_idx == -1:
446                continue
447            no_comment_lines.append(
448                line[leading_sequence_idx + len(leading_sequence) :]
449            )
450            leading_sequence_exist = True
451            break
452
453        if not leading_sequence_exist:
454            no_comment_lines.append(line)
455
456    return "".join(no_comment_lines)
457
458
459class ExtractedRstType(TypedDict):
460    rst_text: str
461    row_offset: int
462    start_idx: int
463    end_idx: int
464
465
[docs]466# @Extract reStructuredText blocks embedded in comments, IMPL_RST_1, impl, [FE_RST_EXTRACTION]
467def extract_rst(
468    text: str, start_marker: str, end_marker: str
469) -> ExtractedRstType | None:
470    """Extract rst from a comment.
471
472    Two use cases:
473    1. Start_marker and end_marker one the same line.
474
475    The rst text is wrapped by start and the end markers on the same line,
476    so, there is no need to remove the leading chars.ArithmeticError
477    E.g.
478    @rst  .. admonition:: title here @endrst
479
480    2. Start_marker and end_marker in different lines.
481
482    The rst text is expected to start from the next line of the start_marker
483    and ends at he previous line of the end_marker.
484    E.g.
485    @rst
486    .. admonition:: title here
487      :collapsible: open
488
489      This example is collapsible, and initially open.
490    @endrst
491    """
492    start_idx = text.find(start_marker)
493    end_idx = text.rfind(end_marker)
494    if start_idx == -1 or end_idx == -1:
495        return None
496    rst_text = text[start_idx + len(start_marker) : end_idx]
497    row_offset = len(text[:start_idx].splitlines())
498    if not rst_text.strip():
499        # empty string is out of the interest
500        return None
501    if UNIX_NEWLINE not in rst_text:
502        # single line rst text
503        oneline_rst: ExtractedRstType = {
504            "rst_text": rst_text,
505            "row_offset": row_offset,
506            "start_idx": start_idx + len(start_marker),
507            "end_idx": end_idx,
508        }
509        return oneline_rst
510
511    # multiline rst text
512
513    first_newline_idx = rst_text.find(UNIX_NEWLINE)
514    rst_text = rst_text[first_newline_idx + len(UNIX_NEWLINE) :]
515    multiline_rst: ExtractedRstType = {
516        "rst_text": rst_text,
517        "row_offset": row_offset,
518        "start_idx": start_idx
519        + len(start_marker)
520        + first_newline_idx
521        + len(UNIX_NEWLINE),
522        "end_idx": end_idx,
523    }
524
525    return multiline_rst