1from collections.abc import Generator
2from dataclasses import dataclass
3import json
4from pathlib import Path
5from typing import Any, TypedDict, cast
6
7from tree_sitter import Node as TreeSitterNode
8
9from sphinx_codelinks.analyse import utils
10from sphinx_codelinks.analyse.models import (
11 MarkedContentType,
12 MarkedRst,
13 NeedIdRefs,
14 OneLineNeed,
15 SourceComment,
16 SourceFile,
17 SourceMap,
18)
19from sphinx_codelinks.analyse.oneline_parser import (
20 OnelineParserInvalidWarning,
21 oneline_parser,
22)
23from sphinx_codelinks.config import (
24 UNIX_NEWLINE,
25 OneLineCommentStyle,
26 SourceAnalyseConfig,
27)
28from sphinx_codelinks.logger import get_logger
29from sphinx_codelinks.source_discover.config import CommentType
30
31logger = get_logger(__name__)
32
33
34def _count(n: int, noun: str) -> str:
35 """Format ``n noun`` with a naive (append-s) plural for progress summaries."""
36 return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
37
38
39class AnalyseWarningType(TypedDict):
40 file_path: str
41 lineno: int
42 msg: str
43 type: str
44 sub_type: str
45
46
47@dataclass
48class AnalyseWarning:
49 file_path: str
50 lineno: int
51 msg: str
52 type: str
53 sub_type: str
54
55
56class SourceAnalyse:
57 def __init__(
58 self,
59 analyse_config: SourceAnalyseConfig,
60 *,
61 name: str = "",
62 ) -> None:
63 self.name = name
64 self.analyse_config = analyse_config
65 self.src_files: list[SourceFile] = []
66 self.src_comments: list[SourceComment] = []
67 self.need_id_refs: list[NeedIdRefs] = []
68 self.oneline_needs: list[OneLineNeed] = []
69 self.marked_rst: list[MarkedRst] = []
70 self.all_marked_content: list[NeedIdRefs | OneLineNeed | MarkedRst] = []
71 # Use explicitly configured git_root if provided, otherwise auto-detect
72 if self.analyse_config.git_root is not None:
73 self.git_root: Path | None = self.analyse_config.git_root.resolve()
74 else:
75 self.git_root = utils.locate_git_root(self.analyse_config.src_dir)
76 self.git_remote_url: str | None = (
77 utils.get_remote_url(self.git_root) if self.git_root else None
78 )
79 self.git_commit_rev: str | None = (
80 utils.get_current_rev(self.git_root) if self.git_root else None
81 )
82 self.project_path: Path = self.git_root or self.analyse_config.src_dir
83 self.oneline_warnings: list[AnalyseWarning] = []
84 # Per-run memo of parsed compile_commands.json, keyed by DB path, so the
85 # database is read once per run instead of once per source file.
86 self._flags_map_cache: dict[Path, dict[Path, list[str]] | None] = {}
87
88 def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]: # type: ignore[explicit-any]
89 """Load source files and extract their content."""
90 for src_path in self.analyse_config.src_files:
91 if not utils.is_text_file(src_path):
92 continue
93 with src_path.open("r", encoding="utf-8", newline="") as f:
94 # Normalize all line endings to Unix LF
95 text = f.read()
96 text = text.replace("\r\n", "\n").replace("\r", "\n")
97 yield src_path, text.encode("utf-8")
98
99 def create_src_objects(self) -> None:
100 parser, query = utils.init_tree_sitter(self.analyse_config.comment_type)
101
102 for src_path, src_string in self.get_src_strings():
103 comments: list[TreeSitterNode] | None = utils.extract_comments(
104 src_string, parser, query
105 )
106 if not comments:
107 continue
108 src_comments: list[SourceComment] = [
109 SourceComment(node) for node in comments
110 ]
111
112 src_file = SourceFile(src_path.absolute())
113 src_file.add_comments(src_comments)
114 self.src_files.append(src_file)
115 self.src_comments.extend(src_comments)
116
117 def _flags_map_for(self, db_path: Path) -> dict[Path, list[str]] | None:
118 """Return the parsed compile_commands.json for ``db_path``, memoized.
119
120 The database is read and parsed once per run instead of once per source
121 file (O(files x entries) -> O(entries)). A present-but-malformed or
122 unreadable database is warned once and cached as ``None`` so callers fall
123 back to the configured defines without re-reading or re-warning per file.
124 """
125 from sphinx_codelinks.analyse.preproc import compile_db # noqa: PLC0415
126
127 if db_path in self._flags_map_cache:
128 return self._flags_map_cache[db_path]
129 try:
130 flags = compile_db.load_flags_map(db_path)
131 except (OSError, ValueError, TypeError) as exc:
132 logger.warning(
133 f"codelinks: failed to read {db_path} ({exc}); "
134 f"falling back to the configured defines"
135 )
136 self._flags_map_cache[db_path] = None
137 return None
138 self._flags_map_cache[db_path] = flags
139 return flags
140
141 def _resolve_preproc_args(self, src_path: Path) -> list[str] | None:
142 from sphinx_codelinks.analyse.preproc import compile_db # noqa: PLC0415
143
144 # `run()` calls this (via create_src_objects_libclang) only when
145 # `preprocessor is not None`, but keep the guard: it narrows the type for
146 # the checker and is a cheap defense (an `assert` would trip bandit S101).
147 preproc = self.analyse_config.preprocessor
148 if preproc is None:
149 return []
150 db_path = preproc.compile_commands
151 if db_path is None:
152 db_path = compile_db.find_compile_db(src_path, self.project_path)
153 if db_path is not None and db_path.is_file():
154 flags = self._flags_map_for(db_path)
155 if flags is None:
156 # Present-but-malformed/unreadable DB (warned once in the helper):
157 # fall back to the configured defines so extraction still runs.
158 return compile_db.defines_to_args(
159 preproc.defines, preproc.includes, preproc.std
160 )
161 args = flags.get(src_path.absolute().resolve())
162 if args is not None:
163 return args
164 # Absent from the DB. compile_commands.json lists only compiled
165 # translation units, never headers — so a header here is parsed
166 # standalone with the global defines (one run = one variant). A
167 # compiled source absent from the build is skipped (spec §3.3).
168 if compile_db.is_translation_unit_source(src_path):
169 return None
170 return compile_db.defines_to_args(
171 preproc.defines, preproc.includes, preproc.std
172 )
173 # No readable DB. `find_compile_db` only ever returns a real file, so a
174 # non-None `db_path` that reaches here is an explicit `compile_commands`
175 # path that is not a readable file (typo/missing): warn and fall back
176 # rather than skip silently. A genuinely absent DB (db_path is None) just
177 # falls back to the manual defines applied globally.
178 if db_path is not None:
179 logger.warning(
180 f"codelinks: compile_commands path {db_path} is not a readable "
181 f"file; falling back to the configured defines"
182 )
183 return compile_db.defines_to_args(
184 preproc.defines, preproc.includes, preproc.std
185 )
186
[docs]187 # @Extract traceability objects with the preprocessor-aware libclang engine, IMPL_PREPROC_1, impl, [FE_PREPROC]
188 def create_src_objects_libclang(self) -> None:
189 from sphinx_codelinks.analyse.preproc import ( # noqa: PLC0415
190 libclang_parser,
191 loader,
192 )
193
194 # Resolve the exception via the loader (never a direct ``import
195 # clang.cindex``) so a missing ``libclang`` extra still surfaces the
196 # loader's friendly install hint rather than a bare ImportError.
197 translation_unit_load_error = (
198 loader.load_clang_cindex().TranslationUnitLoadError
199 )
200
201 for src_path in self.analyse_config.src_files:
202 if not utils.is_text_file(src_path):
203 continue
204 args = self._resolve_preproc_args(src_path)
205 if args is None:
206 logger.debug(
207 f"codelinks: skipping {src_path} — not found in compile_commands.json"
208 )
209 continue
210 try:
211 comments = libclang_parser.extract_active_comments(src_path, args)
212 except translation_unit_load_error:
213 # Last-resort guard. Standalone parses pin ``-x`` to match the
214 # ``-std`` (see defines_to_args), so the common case — a ``.h``/
215 # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its
216 # markers extract. This only fires if libclang still cannot load
217 # the file as a translation unit at all; skip it rather than
218 # aborting the whole run. Surfaced as a ``warning`` so the silent
219 # data-loss (a file's markers dropped) is visible — accepting that
220 # this fails ``sphinx-build -W``.
221 logger.warning(
222 f"codelinks: skipping {src_path} — libclang could not load it "
223 f"as a translation unit"
224 )
225 continue
226 if not comments:
227 continue
228 # ``c`` is a LibclangComment duck-typing the tree-sitter Node
229 # interface SourceComment reads (``.text`` / ``.start_point.row``);
230 # the Node-only path (find_associated_scope) is guarded by
231 # ``is_libclang`` so it never runs on these.
232 src_comments = [SourceComment(cast("TreeSitterNode", c)) for c in comments]
233 src_file = SourceFile(src_path.absolute())
234 src_file.add_comments(src_comments)
235 self.src_files.append(src_file)
236 self.src_comments.extend(src_comments)
237
238 def extract_marker(
239 self,
240 text: str,
241 ) -> Generator[tuple[str, list[str], int, int, int], None, None]:
242 lines = text.splitlines()
243 row_offset = 0
244 for line in lines:
245 for marker in self.analyse_config.need_id_refs_config.markers:
246 marker_idx = line.find(marker)
247 if marker_idx == -1:
248 continue
249 markered_text = line[marker_idx + len(marker) :].strip()
250 need_ids = markered_text.replace(",", " ").split()
251 start_column = marker_idx + len(marker)
252 end_column = start_column + len(markered_text)
253 yield marker, need_ids, row_offset, start_column, end_column
254 row_offset += 1
255
[docs]256 # @Extract need ID references from code comments, IMPL_LNK_1, impl, [FE_LNK]
257 def extract_anchors(
258 self,
259 text: str,
260 filepath: Path,
261 tagged_scope: TreeSitterNode | None,
262 src_comment: SourceComment,
263 ) -> list[NeedIdRefs]:
264 """Extract need-ids-refs from a comment."""
265 anchors: list[NeedIdRefs] = []
266 for (
267 marker,
268 need_ids,
269 row_offset,
270 start_column,
271 end_column,
272 ) in self.extract_marker(text):
273 lineno = src_comment.node.start_point.row + row_offset + 1
274 remote_url = self.git_remote_url
275 if self.git_remote_url and self.git_commit_rev:
276 remote_url = utils.form_https_url(
277 self.git_remote_url,
278 self.git_commit_rev,
279 self.project_path,
280 filepath,
281 lineno,
282 )
283 source_map: SourceMap = {
284 "start": {
285 "row": lineno - 1,
286 "column": start_column,
287 },
288 "end": {
289 "row": lineno - 1,
290 "column": end_column,
291 },
292 }
293 anchors.append(
294 NeedIdRefs(
295 filepath,
296 remote_url,
297 source_map,
298 src_comment,
299 tagged_scope,
300 need_ids,
301 marker,
302 )
303 )
304 return anchors
305
306 def extract_oneline_need(
307 self,
308 text: str,
309 src_comment: SourceComment,
310 oneline_comment_style: OneLineCommentStyle,
311 ) -> Generator[tuple[dict[str, str | list[str] | int], int]]:
312 lines = text.splitlines(keepends=True)
313 row_offset = 0
314 if len(lines) == 1:
315 # single line comment has no newline char in the extracted comment
316 lines[0] = f"{lines[0]}{UNIX_NEWLINE}"
317
318 for line in lines:
319 resolved = oneline_parser(line, oneline_comment_style)
320 if not resolved:
321 row_offset += 1
322 continue
323 if isinstance(resolved, OnelineParserInvalidWarning):
324 if not src_comment.source_file:
325 row_offset += 1
326 continue
327 lineno = src_comment.node.start_point.row + row_offset + 1
328 warning = AnalyseWarning(
329 str(src_comment.source_file.filepath),
330 lineno,
331 resolved.msg,
332 MarkedContentType.need,
333 resolved.sub_type.value,
334 )
335 self.oneline_warnings.append(warning)
336 row_offset += 1
337 continue
338 yield resolved, row_offset
339 row_offset += 1
340
[docs]341 # @Extract one-line traceability needs from comments, IMPL_ONE_1, impl, [FE_DEF, FE_CMT]
342 def extract_oneline_needs(
343 self,
344 text: str,
345 filepath: Path,
346 tagged_scope: TreeSitterNode | None,
347 src_comment: SourceComment,
348 oneline_comment_style: OneLineCommentStyle,
349 ) -> list[OneLineNeed]:
350 row_offset = 0
351 oneline_needs = []
352 for resolved, row_offset in self.extract_oneline_need(
353 text, src_comment, oneline_comment_style
354 ):
355 lineno = src_comment.node.start_point.row + row_offset + 1
356 remote_url = self.git_remote_url
357 if self.git_remote_url and self.git_commit_rev:
358 remote_url = utils.form_https_url(
359 self.git_remote_url,
360 self.git_commit_rev,
361 self.project_path,
362 filepath,
363 lineno,
364 )
365 source_map: SourceMap = {
366 "start": {
367 "row": lineno - 1,
368 "column": resolved["start_column"], # type: ignore[typeddict-item] # dynamic keys
369 },
370 "end": {
371 "row": lineno - 1,
372 "column": resolved["end_column"], # type: ignore[typeddict-item] # dynamic keys
373 },
374 }
375 del resolved["start_column"]
376 del resolved["end_column"]
377 oneline_needs.append(
378 OneLineNeed(
379 filepath,
380 remote_url,
381 source_map,
382 src_comment,
383 tagged_scope,
384 resolved, # type: ignore[arg-type] # int arguments were deleted
385 )
386 )
387 return oneline_needs
388
[docs]389 # @Extract marked reStructuredText blocks from comments, IMPL_MRST_1, impl, [FE_RST_EXTRACTION]
390 def extract_marked_rst(
391 self,
392 text: str,
393 filepath: Path,
394 tagged_scope: TreeSitterNode | None,
395 src_comment: SourceComment,
396 ) -> MarkedRst | None:
397 """Extract marked rst from a comment.
398
399 Presumably, only one marked rst text in a comment.
400 """
401 extracted_rst = utils.extract_rst(
402 text,
403 self.analyse_config.marked_rst_config.start_sequence,
404 self.analyse_config.marked_rst_config.end_sequence,
405 )
406 if not extracted_rst:
407 return None
408 if UNIX_NEWLINE in extracted_rst["rst_text"]:
409 rst_text = utils.remove_leading_sequences(extracted_rst["rst_text"], ["*"])
410 else:
411 rst_text = extracted_rst["rst_text"]
412 lineno = src_comment.node.start_point.row + extracted_rst["row_offset"] + 1
413 remote_url = self.git_remote_url
414 if self.git_remote_url and self.git_commit_rev:
415 remote_url = utils.form_https_url(
416 self.git_remote_url,
417 self.git_commit_rev,
418 self.project_path,
419 filepath,
420 lineno,
421 )
422 source_map: SourceMap = {
423 "start": {
424 "row": lineno - 1,
425 "column": extracted_rst["start_idx"],
426 },
427 "end": {
428 "row": lineno - 1,
429 "column": extracted_rst["end_idx"],
430 },
431 }
432 return MarkedRst(
433 filepath,
434 remote_url,
435 source_map,
436 src_comment,
437 tagged_scope,
438 rst_text,
439 )
440
441 def extract_marked_content(self) -> None:
442 for src_comment in self.src_comments:
443 text = (
444 src_comment.node.text.decode("utf-8") if src_comment.node.text else None
445 )
446 if not text:
447 continue
448 filepath = (
449 src_comment.source_file.filepath if src_comment.source_file else None
450 )
451 if not filepath:
452 continue
453 if getattr(src_comment.node, "is_libclang", False):
454 tagged_scope: TreeSitterNode | None = None
455 else:
456 tagged_scope = utils.find_associated_scope(
457 src_comment.node, self.analyse_config.comment_type
458 )
459 if self.analyse_config.get_need_id_refs:
460 anchors = self.extract_anchors(
461 text, filepath, tagged_scope, src_comment
462 )
463 self.need_id_refs.extend(anchors)
464
465 if self.analyse_config.get_oneline_needs:
466 oneline_needs = self.extract_oneline_needs(
467 text,
468 filepath,
469 tagged_scope,
470 src_comment,
471 self.analyse_config.oneline_comment_style,
472 )
473 self.oneline_needs.extend(oneline_needs)
474 if self.analyse_config.get_rst:
475 marked_rst = self.extract_marked_rst(
476 text, filepath, tagged_scope, src_comment
477 )
478 if marked_rst:
479 self.marked_rst.append(marked_rst)
480
481 def merge_marked_content(self) -> None:
482 self.all_marked_content.extend(self.need_id_refs)
483 self.oneline_needs.sort(key=lambda x: x.source_map["start"]["row"])
484 self.all_marked_content.extend(self.oneline_needs)
485 self.all_marked_content.extend(self.marked_rst)
486 self.all_marked_content.sort(
487 key=lambda x: (x.filepath, x.source_map["start"]["row"])
488 )
489
490 def dump_marked_content(self, outdir: Path) -> None:
491 output_path = outdir / "marked_content.json"
492 if not output_path.parent.exists():
493 output_path.parent.mkdir(parents=True)
494 to_dump = [
495 marked_content.to_dict() for marked_content in self.all_marked_content
496 ]
497 with output_path.open("w") as f:
498 json.dump(to_dump, f)
499
500 def run(self) -> None:
501 if (
502 self.analyse_config.preprocessor is not None
503 and self.analyse_config.comment_type == CommentType.cpp
504 ):
505 self.create_src_objects_libclang()
506 else:
507 self.create_src_objects()
508 self.extract_marked_content()
509 self.merge_marked_content()
510 self._log_summary()
511
512 def _log_summary(self) -> None:
513 """Emit a per-project marker (default-visible) plus a -v breakdown."""
514 label = f"codelinks [{self.name}]" if self.name else "codelinks"
515 logger.info(
516 f"{label}: {_count(len(self.src_files), 'file')}, "
517 f"{_count(len(self.all_marked_content), 'marker')}"
518 )
519 logger.debug(
520 f"{label}: {_count(len(self.src_comments), 'comment')}, "
521 f"{_count(len(self.oneline_needs), 'oneline need')}, "
522 f"{_count(len(self.need_id_refs), 'id-ref')}, "
523 f"{_count(len(self.marked_rst), 'marked-rst block')}"
524 )