# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 # For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt """Callback functions and support for sys.monitoring data collection.""" from __future__ import annotations import functools import inspect import os.path import sys import threading import traceback from dataclasses import dataclass from types import CodeType from types import FrameType from typing import Any from typing import Callable from typing import cast from typing import Set from typing import TYPE_CHECKING from coverage.debug import short_filename from coverage.debug import short_stack from coverage.types import AnyCallable from coverage.types import TArc from coverage.types import TFileDisposition from coverage.types import TLineNo from coverage.types import TracerCore from coverage.types import TTraceData from coverage.types import TTraceFileData from coverage.types import TWarnFn # pylint: disable=unused-argument LOG = False # This module will be imported in all versions of Python, but only used in 3.12+ # It will be type-checked for 3.12, but not for earlier versions. sys_monitoring = getattr(sys, 'monitoring', None) if TYPE_CHECKING: assert sys_monitoring is not None # I want to say this but it's not allowed: # MonitorReturn = Literal[sys.monitoring.DISABLE] | None MonitorReturn = Any if LOG: # pragma: debugging class LoggingWrapper: """Wrap a namespace to log all its functions.""" def __init__(self, wrapped: Any, namespace: str) -> None: self.wrapped = wrapped self.namespace = namespace def __getattr__(self, name: str) -> Callable[..., Any]: def _wrapped(*args: Any, **kwargs: Any) -> Any: log(f'{self.namespace}.{name}{args}{kwargs}') return getattr(self.wrapped, name)(*args, **kwargs) return _wrapped sys_monitoring = LoggingWrapper(sys_monitoring, 'sys.monitoring') assert sys_monitoring is not None short_stack = functools.partial( short_stack, full=True, short_filenames=True, frame_ids=True, ) seen_threads: set[int] = set() def log(msg: str) -> None: """Write a message to our detailed debugging log(s).""" # Thread ids are reused across processes? # Make a shorter number more likely to be unique. pid = os.getpid() tid = cast(int, threading.current_thread().ident) tslug = f'{(pid * tid) % 9_999_991:07d}' if tid not in seen_threads: seen_threads.add(tid) log(f'New thread {tid} {tslug}:\n{short_stack()}') # log_seq = int(os.getenv("PANSEQ", "0")) # root = f"/tmp/pan.{log_seq:03d}" for filename in [ '/tmp/foo.out', # f"{root}.out", # f"{root}-{pid}.out", # f"{root}-{pid}-{tslug}.out", ]: with open(filename, 'a') as f: print(f'{pid}:{tslug}: {msg}', file=f, flush=True) def arg_repr(arg: Any) -> str: """Make a customized repr for logged values.""" if isinstance(arg, CodeType): return ( f'' ) return repr(arg) def panopticon(*names: str | None) -> AnyCallable: """Decorate a function to log its calls.""" def _decorator(method: AnyCallable) -> AnyCallable: @functools.wraps(method) def _wrapped(self: Any, *args: Any) -> Any: try: # log(f"{method.__name__}() stack:\n{short_stack()}") args_reprs = [] for name, arg in zip(names, args): if name is None: continue args_reprs.append(f'{name}={arg_repr(arg)}') log(f"{id(self):#x}:{method.__name__}({', '.join(args_reprs)})") ret = method(self, *args) # log(f" end {id(self):#x}:{method.__name__}({', '.join(args_reprs)})") return ret except Exception as exc: log(f'!!{exc.__class__.__name__}: {exc}') log(''.join(traceback.format_exception(exc))) # pylint: disable=[no-value-for-parameter] try: assert sys_monitoring is not None sys_monitoring.set_events(sys.monitoring.COVERAGE_ID, 0) except ValueError: # We might have already shut off monitoring. log('oops, shutting off events with disabled tool id') raise return _wrapped return _decorator else: def log(msg: str) -> None: """Write a message to our detailed debugging log(s), but not really.""" def panopticon(*names: str | None) -> AnyCallable: """Decorate a function to log its calls, but not really.""" def _decorator(meth: AnyCallable) -> AnyCallable: return meth return _decorator @dataclass class CodeInfo: """The information we want about each code object.""" tracing: bool file_data: TTraceFileData | None # TODO: what is byte_to_line for? byte_to_line: dict[int, int] | None def bytes_to_lines(code: CodeType) -> dict[int, int]: """Make a dict mapping byte code offsets to line numbers.""" b2l = {} for bstart, bend, lineno in code.co_lines(): if lineno is not None: for boffset in range(bstart, bend, 2): b2l[boffset] = lineno return b2l class SysMonitor(TracerCore): """Python implementation of the raw data tracer for PEP669 implementations.""" # One of these will be used across threads. Be careful. def __init__(self, tool_id: int) -> None: # Attributes set from the collector: self.data: TTraceData self.trace_arcs = False self.should_trace: Callable[[str, FrameType], TFileDisposition] self.should_trace_cache: dict[str, TFileDisposition | None] # TODO: should_start_context and switch_context are unused! # Change tests/testenv.py:DYN_CONTEXTS when this is updated. self.should_start_context: Callable[[FrameType], str | None] | None = None self.switch_context: Callable[[str | None], None] | None = None # TODO: warn is unused. self.warn: TWarnFn self.myid = tool_id # Map id(code_object) -> CodeInfo self.code_infos: dict[int, CodeInfo] = {} # A list of code_objects, just to keep them alive so that id's are # useful as identity. self.code_objects: list[CodeType] = [] self.last_lines: dict[FrameType, int] = {} # Map id(code_object) -> code_object self.local_event_codes: dict[int, CodeType] = {} self.sysmon_on = False self.stats = { 'starts': 0, } self.stopped = False self._activity = False def __repr__(self) -> str: points = sum(len(v) for v in self.data.values()) files = len(self.data) return f'' @panopticon() def start(self) -> None: """Start this Tracer.""" self.stopped = False assert sys_monitoring is not None sys_monitoring.use_tool_id(self.myid, 'coverage.py') register = functools.partial(sys_monitoring.register_callback, self.myid) events = sys_monitoring.events if self.trace_arcs: sys_monitoring.set_events( self.myid, events.PY_START | events.PY_UNWIND, ) register(events.PY_START, self.sysmon_py_start) register(events.PY_RESUME, self.sysmon_py_resume_arcs) register(events.PY_RETURN, self.sysmon_py_return_arcs) register(events.PY_UNWIND, self.sysmon_py_unwind_arcs) register(events.LINE, self.sysmon_line_arcs) else: sys_monitoring.set_events(self.myid, events.PY_START) register(events.PY_START, self.sysmon_py_start) register(events.LINE, self.sysmon_line_lines) sys_monitoring.restart_events() self.sysmon_on = True @panopticon() def stop(self) -> None: """Stop this Tracer.""" if not self.sysmon_on: # In forking situations, we might try to stop when we are not # started. Do nothing in that case. return assert sys_monitoring is not None sys_monitoring.set_events(self.myid, 0) self.sysmon_on = False for code in self.local_event_codes.values(): sys_monitoring.set_local_events(self.myid, code, 0) self.local_event_codes = {} sys_monitoring.free_tool_id(self.myid) @panopticon() def post_fork(self) -> None: """The process has forked, clean up as needed.""" self.stop() def activity(self) -> bool: """Has there been any activity?""" return self._activity def reset_activity(self) -> None: """Reset the activity() flag.""" self._activity = False def get_stats(self) -> dict[str, int] | None: """Return a dictionary of statistics, or None.""" return None # The number of frames in callers_frame takes @panopticon into account. if LOG: def callers_frame(self) -> FrameType: """Get the frame of the Python code we're monitoring.""" return ( inspect.currentframe().f_back.f_back.f_back # type: ignore[union-attr,return-value] ) else: def callers_frame(self) -> FrameType: """Get the frame of the Python code we're monitoring.""" return inspect.currentframe().f_back.f_back # type: ignore[union-attr,return-value] @panopticon('code', '@') def sysmon_py_start(self, code: CodeType, instruction_offset: int) -> MonitorReturn: """Handle sys.monitoring.events.PY_START events.""" # Entering a new frame. Decide if we should trace in this file. self._activity = True self.stats['starts'] += 1 code_info = self.code_infos.get(id(code)) tracing_code: bool | None = None file_data: TTraceFileData | None = None if code_info is not None: tracing_code = code_info.tracing file_data = code_info.file_data if tracing_code is None: filename = code.co_filename disp = self.should_trace_cache.get(filename) if disp is None: frame = inspect.currentframe().f_back # type: ignore[union-attr] if LOG: # @panopticon adds a frame. frame = frame.f_back # type: ignore[union-attr] disp = self.should_trace(filename, frame) # type: ignore[arg-type] self.should_trace_cache[filename] = disp tracing_code = disp.trace if tracing_code: tracename = disp.source_filename assert tracename is not None if tracename not in self.data: self.data[tracename] = set() file_data = self.data[tracename] b2l = bytes_to_lines(code) else: file_data = None b2l = None self.code_infos[id(code)] = CodeInfo( tracing=tracing_code, file_data=file_data, byte_to_line=b2l, ) self.code_objects.append(code) if tracing_code: events = sys.monitoring.events if self.sysmon_on: assert sys_monitoring is not None sys_monitoring.set_local_events( self.myid, code, events.PY_RETURN | # events.PY_RESUME | # | events.PY_YIELD events.LINE, # | events.BRANCH # | events.JUMP ) self.local_event_codes[id(code)] = code if tracing_code and self.trace_arcs: frame = self.callers_frame() self.last_lines[frame] = -code.co_firstlineno return None else: return sys.monitoring.DISABLE @panopticon('code', '@') def sysmon_py_resume_arcs( self, code: CodeType, instruction_offset: int, ) -> MonitorReturn: """Handle sys.monitoring.events.PY_RESUME events for branch coverage.""" frame = self.callers_frame() self.last_lines[frame] = frame.f_lineno @panopticon('code', '@', None) def sysmon_py_return_arcs( self, code: CodeType, instruction_offset: int, retval: object, ) -> MonitorReturn: """Handle sys.monitoring.events.PY_RETURN events for branch coverage.""" frame = self.callers_frame() code_info = self.code_infos.get(id(code)) if code_info is not None and code_info.file_data is not None: last_line = self.last_lines.get(frame) if last_line is not None: arc = (last_line, -code.co_firstlineno) # log(f"adding {arc=}") cast(Set[TArc], code_info.file_data).add(arc) # Leaving this function, no need for the frame any more. self.last_lines.pop(frame, None) @panopticon('code', '@', 'exc') def sysmon_py_unwind_arcs( self, code: CodeType, instruction_offset: int, exception: BaseException, ) -> MonitorReturn: """Handle sys.monitoring.events.PY_UNWIND events for branch coverage.""" frame = self.callers_frame() # Leaving this function. last_line = self.last_lines.pop(frame, None) if isinstance(exception, GeneratorExit): # We don't want to count generator exits as arcs. return code_info = self.code_infos.get(id(code)) if code_info is not None and code_info.file_data is not None: if last_line is not None: arc = (last_line, -code.co_firstlineno) # log(f"adding {arc=}") cast(Set[TArc], code_info.file_data).add(arc) @panopticon('code', 'line') def sysmon_line_lines(self, code: CodeType, line_number: int) -> MonitorReturn: """Handle sys.monitoring.events.LINE events for line coverage.""" code_info = self.code_infos[id(code)] if code_info.file_data is not None: cast(Set[TLineNo], code_info.file_data).add(line_number) # log(f"adding {line_number=}") return sys.monitoring.DISABLE @panopticon('code', 'line') def sysmon_line_arcs(self, code: CodeType, line_number: int) -> MonitorReturn: """Handle sys.monitoring.events.LINE events for branch coverage.""" code_info = self.code_infos[id(code)] ret = None if code_info.file_data is not None: frame = self.callers_frame() last_line = self.last_lines.get(frame) if last_line is not None: arc = (last_line, line_number) cast(Set[TArc], code_info.file_data).add(arc) # log(f"adding {arc=}") self.last_lines[frame] = line_number return ret