1from collections.abc import Generator
  2from dataclasses import dataclass
  3import json
  4from pathlib import Path
  5from typing import Any, TypedDict, cast
  6
  7from tree_sitter import Node as TreeSitterNode
  8
  9from sphinx_codelinks.analyse import utils
 10from sphinx_codelinks.analyse.models import (
 11    MarkedContentType,
 12    MarkedRst,
 13    NeedIdRefs,
 14    OneLineNeed,
 15    SourceComment,
 16    SourceFile,
 17    SourceMap,
 18)
 19from sphinx_codelinks.analyse.oneline_parser import (
 20    OnelineParserInvalidWarning,
 21    oneline_parser,
 22)
 23from sphinx_codelinks.config import (
 24    UNIX_NEWLINE,
 25    OneLineCommentStyle,
 26    SourceAnalyseConfig,
 27)
 28from sphinx_codelinks.logger import get_logger
 29from sphinx_codelinks.source_discover.config import CommentType
 30
 31logger = get_logger(__name__)
 32
 33
 34def _count(n: int, noun: str) -> str:
 35    """Format ``n noun`` with a naive (append-s) plural for progress summaries."""
 36    return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
 37
 38
 39class AnalyseWarningType(TypedDict):
 40    file_path: str
 41    lineno: int
 42    msg: str
 43    type: str
 44    sub_type: str
 45
 46
 47@dataclass
 48class AnalyseWarning:
 49    file_path: str
 50    lineno: int
 51    msg: str
 52    type: str
 53    sub_type: str
 54
 55
 56class SourceAnalyse:
 57    def __init__(
 58        self,
 59        analyse_config: SourceAnalyseConfig,
 60        *,
 61        name: str = "",
 62    ) -> None:
 63        self.name = name
 64        self.analyse_config = analyse_config
 65        self.src_files: list[SourceFile] = []
 66        self.src_comments: list[SourceComment] = []
 67        self.need_id_refs: list[NeedIdRefs] = []
 68        self.oneline_needs: list[OneLineNeed] = []
 69        self.marked_rst: list[MarkedRst] = []
 70        self.all_marked_content: list[NeedIdRefs | OneLineNeed | MarkedRst] = []
 71        # Use explicitly configured git_root if provided, otherwise auto-detect
 72        if self.analyse_config.git_root is not None:
 73            self.git_root: Path | None = self.analyse_config.git_root.resolve()
 74        else:
 75            self.git_root = utils.locate_git_root(self.analyse_config.src_dir)
 76        self.git_remote_url: str | None = (
 77            utils.get_remote_url(self.git_root) if self.git_root else None
 78        )
 79        self.git_commit_rev: str | None = (
 80            utils.get_current_rev(self.git_root) if self.git_root else None
 81        )
 82        self.project_path: Path = self.git_root or self.analyse_config.src_dir
 83        self.oneline_warnings: list[AnalyseWarning] = []
 84        # Per-run memo of parsed compile_commands.json, keyed by DB path, so the
 85        # database is read once per run instead of once per source file.
 86        self._flags_map_cache: dict[Path, dict[Path, list[str]] | None] = {}
 87
 88    def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]:  # type: ignore[explicit-any]
 89        """Load source files and extract their content."""
 90        for src_path in self.analyse_config.src_files:
 91            if not utils.is_text_file(src_path):
 92                continue
 93            with src_path.open("r", encoding="utf-8", newline="") as f:
 94                # Normalize all line endings to Unix LF
 95                text = f.read()
 96            text = text.replace("\r\n", "\n").replace("\r", "\n")
 97            yield src_path, text.encode("utf-8")
 98
 99    def create_src_objects(self) -> None:
100        parser, query = utils.init_tree_sitter(self.analyse_config.comment_type)
101
102        for src_path, src_string in self.get_src_strings():
103            comments: list[TreeSitterNode] | None = utils.extract_comments(
104                src_string, parser, query
105            )
106            if not comments:
107                continue
108            src_comments: list[SourceComment] = [
109                SourceComment(node) for node in comments
110            ]
111
112            src_file = SourceFile(src_path.absolute())
113            src_file.add_comments(src_comments)
114            self.src_files.append(src_file)
115            self.src_comments.extend(src_comments)
116
117    def _flags_map_for(self, db_path: Path) -> dict[Path, list[str]] | None:
118        """Return the parsed compile_commands.json for ``db_path``, memoized.
119
120        The database is read and parsed once per run instead of once per source
121        file (O(files x entries) -> O(entries)). A present-but-malformed or
122        unreadable database is warned once and cached as ``None`` so callers fall
123        back to the configured defines without re-reading or re-warning per file.
124        """
125        from sphinx_codelinks.analyse.preproc import compile_db  # noqa: PLC0415
126
127        if db_path in self._flags_map_cache:
128            return self._flags_map_cache[db_path]
129        try:
130            flags = compile_db.load_flags_map(db_path)
131        except (OSError, ValueError, TypeError) as exc:
132            logger.warning(
133                f"codelinks: failed to read {db_path} ({exc}); "
134                f"falling back to the configured defines"
135            )
136            self._flags_map_cache[db_path] = None
137            return None
138        self._flags_map_cache[db_path] = flags
139        return flags
140
141    def _resolve_preproc_args(self, src_path: Path) -> list[str] | None:
142        from sphinx_codelinks.analyse.preproc import compile_db  # noqa: PLC0415
143
144        # `run()` calls this (via create_src_objects_libclang) only when
145        # `preprocessor is not None`, but keep the guard: it narrows the type for
146        # the checker and is a cheap defense (an `assert` would trip bandit S101).
147        preproc = self.analyse_config.preprocessor
148        if preproc is None:
149            return []
150        db_path = preproc.compile_commands
151        if db_path is None:
152            db_path = compile_db.find_compile_db(src_path, self.project_path)
153        if db_path is not None and db_path.is_file():
154            flags = self._flags_map_for(db_path)
155            if flags is None:
156                # Present-but-malformed/unreadable DB (warned once in the helper):
157                # fall back to the configured defines so extraction still runs.
158                return compile_db.defines_to_args(
159                    preproc.defines, preproc.includes, preproc.std
160                )
161            args = flags.get(src_path.absolute().resolve())
162            if args is not None:
163                return args
164            # Absent from the DB. compile_commands.json lists only compiled
165            # translation units, never headers — so a header here is parsed
166            # standalone with the global defines (one run = one variant). A
167            # compiled source absent from the build is skipped (spec §3.3).
168            if compile_db.is_translation_unit_source(src_path):
169                return None
170            return compile_db.defines_to_args(
171                preproc.defines, preproc.includes, preproc.std
172            )
173        # No readable DB. `find_compile_db` only ever returns a real file, so a
174        # non-None `db_path` that reaches here is an explicit `compile_commands`
175        # path that is not a readable file (typo/missing): warn and fall back
176        # rather than skip silently. A genuinely absent DB (db_path is None) just
177        # falls back to the manual defines applied globally.
178        if db_path is not None:
179            logger.warning(
180                f"codelinks: compile_commands path {db_path} is not a readable "
181                f"file; falling back to the configured defines"
182            )
183        return compile_db.defines_to_args(
184            preproc.defines, preproc.includes, preproc.std
185        )
186
[docs]187    # @Extract traceability objects with the preprocessor-aware libclang engine, IMPL_PREPROC_1, impl, [FE_PREPROC]
188    def create_src_objects_libclang(self) -> None:
189        from sphinx_codelinks.analyse.preproc import (  # noqa: PLC0415
190            libclang_parser,
191            loader,
192        )
193
194        # Resolve the exception via the loader (never a direct ``import
195        # clang.cindex``) so a missing ``libclang`` extra still surfaces the
196        # loader's friendly install hint rather than a bare ImportError.
197        translation_unit_load_error = (
198            loader.load_clang_cindex().TranslationUnitLoadError
199        )
200
201        for src_path in self.analyse_config.src_files:
202            if not utils.is_text_file(src_path):
203                continue
204            args = self._resolve_preproc_args(src_path)
205            if args is None:
206                logger.debug(
207                    f"codelinks: skipping {src_path} — not found in compile_commands.json"
208                )
209                continue
210            try:
211                comments = libclang_parser.extract_active_comments(src_path, args)
212            except translation_unit_load_error:
213                # Last-resort guard. Standalone parses pin ``-x`` to match the
214                # ``-std`` (see defines_to_args), so the common case — a ``.h``/
215                # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its
216                # markers extract. This only fires if libclang still cannot load
217                # the file as a translation unit at all; skip it rather than
218                # aborting the whole run. Surfaced as a ``warning`` so the silent
219                # data-loss (a file's markers dropped) is visible — accepting that
220                # this fails ``sphinx-build -W``.
221                logger.warning(
222                    f"codelinks: skipping {src_path} — libclang could not load it "
223                    f"as a translation unit"
224                )
225                continue
226            if not comments:
227                continue
228            # ``c`` is a LibclangComment duck-typing the tree-sitter Node
229            # interface SourceComment reads (``.text`` / ``.start_point.row``);
230            # the Node-only path (find_associated_scope) is guarded by
231            # ``is_libclang`` so it never runs on these.
232            src_comments = [SourceComment(cast("TreeSitterNode", c)) for c in comments]
233            src_file = SourceFile(src_path.absolute())
234            src_file.add_comments(src_comments)
235            self.src_files.append(src_file)
236            self.src_comments.extend(src_comments)
237
238    def extract_marker(
239        self,
240        text: str,
241    ) -> Generator[tuple[str, list[str], int, int, int], None, None]:
242        lines = text.splitlines()
243        row_offset = 0
244        for line in lines:
245            for marker in self.analyse_config.need_id_refs_config.markers:
246                marker_idx = line.find(marker)
247                if marker_idx == -1:
248                    continue
249                markered_text = line[marker_idx + len(marker) :].strip()
250                need_ids = markered_text.replace(",", " ").split()
251                start_column = marker_idx + len(marker)
252                end_column = start_column + len(markered_text)
253                yield marker, need_ids, row_offset, start_column, end_column
254            row_offset += 1
255
[docs]256    # @Extract need ID references from code comments, IMPL_LNK_1, impl, [FE_LNK]
257    def extract_anchors(
258        self,
259        text: str,
260        filepath: Path,
261        tagged_scope: TreeSitterNode | None,
262        src_comment: SourceComment,
263    ) -> list[NeedIdRefs]:
264        """Extract need-ids-refs from a comment."""
265        anchors: list[NeedIdRefs] = []
266        for (
267            marker,
268            need_ids,
269            row_offset,
270            start_column,
271            end_column,
272        ) in self.extract_marker(text):
273            lineno = src_comment.node.start_point.row + row_offset + 1
274            remote_url = self.git_remote_url
275            if self.git_remote_url and self.git_commit_rev:
276                remote_url = utils.form_https_url(
277                    self.git_remote_url,
278                    self.git_commit_rev,
279                    self.project_path,
280                    filepath,
281                    lineno,
282                )
283            source_map: SourceMap = {
284                "start": {
285                    "row": lineno - 1,
286                    "column": start_column,
287                },
288                "end": {
289                    "row": lineno - 1,
290                    "column": end_column,
291                },
292            }
293            anchors.append(
294                NeedIdRefs(
295                    filepath,
296                    remote_url,
297                    source_map,
298                    src_comment,
299                    tagged_scope,
300                    need_ids,
301                    marker,
302                )
303            )
304        return anchors
305
306    def extract_oneline_need(
307        self,
308        text: str,
309        src_comment: SourceComment,
310        oneline_comment_style: OneLineCommentStyle,
311    ) -> Generator[tuple[dict[str, str | list[str] | int], int]]:
312        lines = text.splitlines(keepends=True)
313        row_offset = 0
314        if len(lines) == 1:
315            # single line comment has no newline char in the extracted comment
316            lines[0] = f"{lines[0]}{UNIX_NEWLINE}"
317
318        for line in lines:
319            resolved = oneline_parser(line, oneline_comment_style)
320            if not resolved:
321                row_offset += 1
322                continue
323            if isinstance(resolved, OnelineParserInvalidWarning):
324                if not src_comment.source_file:
325                    row_offset += 1
326                    continue
327                lineno = src_comment.node.start_point.row + row_offset + 1
328                warning = AnalyseWarning(
329                    str(src_comment.source_file.filepath),
330                    lineno,
331                    resolved.msg,
332                    MarkedContentType.need,
333                    resolved.sub_type.value,
334                )
335                self.oneline_warnings.append(warning)
336                row_offset += 1
337                continue
338            yield resolved, row_offset
339            row_offset += 1
340
[docs]341    # @Extract one-line traceability needs from comments, IMPL_ONE_1, impl, [FE_DEF, FE_CMT]
342    def extract_oneline_needs(
343        self,
344        text: str,
345        filepath: Path,
346        tagged_scope: TreeSitterNode | None,
347        src_comment: SourceComment,
348        oneline_comment_style: OneLineCommentStyle,
349    ) -> list[OneLineNeed]:
350        row_offset = 0
351        oneline_needs = []
352        for resolved, row_offset in self.extract_oneline_need(
353            text, src_comment, oneline_comment_style
354        ):
355            lineno = src_comment.node.start_point.row + row_offset + 1
356            remote_url = self.git_remote_url
357            if self.git_remote_url and self.git_commit_rev:
358                remote_url = utils.form_https_url(
359                    self.git_remote_url,
360                    self.git_commit_rev,
361                    self.project_path,
362                    filepath,
363                    lineno,
364                )
365            source_map: SourceMap = {
366                "start": {
367                    "row": lineno - 1,
368                    "column": resolved["start_column"],  # type: ignore[typeddict-item]  # dynamic keys
369                },
370                "end": {
371                    "row": lineno - 1,
372                    "column": resolved["end_column"],  # type: ignore[typeddict-item]  # dynamic keys
373                },
374            }
375            del resolved["start_column"]
376            del resolved["end_column"]
377            oneline_needs.append(
378                OneLineNeed(
379                    filepath,
380                    remote_url,
381                    source_map,
382                    src_comment,
383                    tagged_scope,
384                    resolved,  # type: ignore[arg-type] # int arguments were deleted
385                )
386            )
387        return oneline_needs
388
[docs]389    # @Extract marked reStructuredText blocks from comments, IMPL_MRST_1, impl, [FE_RST_EXTRACTION]
390    def extract_marked_rst(
391        self,
392        text: str,
393        filepath: Path,
394        tagged_scope: TreeSitterNode | None,
395        src_comment: SourceComment,
396    ) -> MarkedRst | None:
397        """Extract marked rst from a comment.
398
399        Presumably, only one marked rst text in a comment.
400        """
401        extracted_rst = utils.extract_rst(
402            text,
403            self.analyse_config.marked_rst_config.start_sequence,
404            self.analyse_config.marked_rst_config.end_sequence,
405        )
406        if not extracted_rst:
407            return None
408        if UNIX_NEWLINE in extracted_rst["rst_text"]:
409            rst_text = utils.remove_leading_sequences(extracted_rst["rst_text"], ["*"])
410        else:
411            rst_text = extracted_rst["rst_text"]
412        lineno = src_comment.node.start_point.row + extracted_rst["row_offset"] + 1
413        remote_url = self.git_remote_url
414        if self.git_remote_url and self.git_commit_rev:
415            remote_url = utils.form_https_url(
416                self.git_remote_url,
417                self.git_commit_rev,
418                self.project_path,
419                filepath,
420                lineno,
421            )
422        source_map: SourceMap = {
423            "start": {
424                "row": lineno - 1,
425                "column": extracted_rst["start_idx"],
426            },
427            "end": {
428                "row": lineno - 1,
429                "column": extracted_rst["end_idx"],
430            },
431        }
432        return MarkedRst(
433            filepath,
434            remote_url,
435            source_map,
436            src_comment,
437            tagged_scope,
438            rst_text,
439        )
440
441    def extract_marked_content(self) -> None:
442        for src_comment in self.src_comments:
443            text = (
444                src_comment.node.text.decode("utf-8") if src_comment.node.text else None
445            )
446            if not text:
447                continue
448            filepath = (
449                src_comment.source_file.filepath if src_comment.source_file else None
450            )
451            if not filepath:
452                continue
453            if getattr(src_comment.node, "is_libclang", False):
454                tagged_scope: TreeSitterNode | None = None
455            else:
456                tagged_scope = utils.find_associated_scope(
457                    src_comment.node, self.analyse_config.comment_type
458                )
459            if self.analyse_config.get_need_id_refs:
460                anchors = self.extract_anchors(
461                    text, filepath, tagged_scope, src_comment
462                )
463                self.need_id_refs.extend(anchors)
464
465            if self.analyse_config.get_oneline_needs:
466                oneline_needs = self.extract_oneline_needs(
467                    text,
468                    filepath,
469                    tagged_scope,
470                    src_comment,
471                    self.analyse_config.oneline_comment_style,
472                )
473                self.oneline_needs.extend(oneline_needs)
474            if self.analyse_config.get_rst:
475                marked_rst = self.extract_marked_rst(
476                    text, filepath, tagged_scope, src_comment
477                )
478                if marked_rst:
479                    self.marked_rst.append(marked_rst)
480
481    def merge_marked_content(self) -> None:
482        self.all_marked_content.extend(self.need_id_refs)
483        self.oneline_needs.sort(key=lambda x: x.source_map["start"]["row"])
484        self.all_marked_content.extend(self.oneline_needs)
485        self.all_marked_content.extend(self.marked_rst)
486        self.all_marked_content.sort(
487            key=lambda x: (x.filepath, x.source_map["start"]["row"])
488        )
489
490    def dump_marked_content(self, outdir: Path) -> None:
491        output_path = outdir / "marked_content.json"
492        if not output_path.parent.exists():
493            output_path.parent.mkdir(parents=True)
494        to_dump = [
495            marked_content.to_dict() for marked_content in self.all_marked_content
496        ]
497        with output_path.open("w") as f:
498            json.dump(to_dump, f)
499
500    def run(self) -> None:
501        if (
502            self.analyse_config.preprocessor is not None
503            and self.analyse_config.comment_type == CommentType.cpp
504        ):
505            self.create_src_objects_libclang()
506        else:
507            self.create_src_objects()
508        self.extract_marked_content()
509        self.merge_marked_content()
510        self._log_summary()
511
512    def _log_summary(self) -> None:
513        """Emit a per-project marker (default-visible) plus a -v breakdown."""
514        label = f"codelinks [{self.name}]" if self.name else "codelinks"
515        logger.info(
516            f"{label}: {_count(len(self.src_files), 'file')}, "
517            f"{_count(len(self.all_marked_content), 'marker')}"
518        )
519        logger.debug(
520            f"{label}: {_count(len(self.src_comments), 'comment')}, "
521            f"{_count(len(self.oneline_needs), 'oneline need')}, "
522            f"{_count(len(self.need_id_refs), 'id-ref')}, "
523            f"{_count(len(self.marked_rst), 'marked-rst block')}"
524        )