Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Doc/c-api/type.rst
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,7 @@ but need extra remarks for use as slots:
in the following situations:

- The base is not variable-sized (its
:c:member:`~PyTypeObject.tp_itemsize`).
:c:member:`~PyTypeObject.tp_itemsize` is zero).
- The requested :c:member:`PyType_Spec.basicsize` is positive,
suggesting that the memory layout of the base class is known.
- The requested :c:member:`PyType_Spec.basicsize` is zero,
Expand Down
11 changes: 7 additions & 4 deletions Doc/library/curses.rst
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,13 @@ The module :mod:`!curses` defines the following functions:
.. function:: color_pair(pair_number)

Return the attribute value for displaying text in the specified color pair.
Only the first 256 color pairs are supported. This
attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`,
and the other :const:`!A_\*` attributes. :func:`pair_number` is the counterpart
to this function.
Only color pairs that fit in the color-pair field of the returned value can
be represented (usually the first 256); a larger *pair_number* raises
:exc:`OverflowError` rather than being silently masked to a different pair.
Use :meth:`~window.color_set` or :meth:`~window.attr_set` to display higher
pairs. This attribute value can be combined with :const:`A_STANDOUT`,
:const:`A_REVERSE`, and the other :const:`!A_\*` attributes.
:func:`pair_number` is the counterpart to this function.


.. function:: curs_set(visibility)
Expand Down
14 changes: 14 additions & 0 deletions Doc/library/tkinter.rst
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,20 @@ they are denoted in Tk, which can be useful when referring to the Tk man pages.
| %d | detail | %D | delta |
+----+---------------------+----+---------------------+

The ``add`` parameter above only affects the bindings you make yourself.
Every widget also inherits *class bindings*
that implement its standard behavior --
for example a :class:`Text` widget binds :kbd:`Control-t`
to transpose two characters.
These are described in the bindings section of the widget's Tk man page
(such as :manpage:`text(3tk)` or :manpage:`entry(3tk)`).

Class bindings are processed separately from your own,
so binding an event yourself does not replace the default; both run.
To suppress an unwanted default binding,
bind the event on the widget
and return the string ``"break"`` from your callback.


The index parameter
^^^^^^^^^^^^^^^^^^^
Expand Down
16 changes: 16 additions & 0 deletions Grammar/python.gram
Original file line number Diff line number Diff line change
Expand Up @@ -819,6 +819,7 @@ is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, I

bitwise_or[expr_ty]:
| a=bitwise_or '|' b=bitwise_xor { _PyAST_BinOp(a, BitOr, b, EXTRA) }
| invalid_bitwise_or
| bitwise_xor

bitwise_xor[expr_ty]:
Expand All @@ -827,6 +828,7 @@ bitwise_xor[expr_ty]:

bitwise_and[expr_ty]:
| a=bitwise_and '&' b=shift_expr { _PyAST_BinOp(a, BitAnd, b, EXTRA) }
| invalid_bitwise_and
| shift_expr

shift_expr[expr_ty]:
Expand Down Expand Up @@ -1638,3 +1640,17 @@ invalid_type_params:
RAISE_SYNTAX_ERROR_STARTING_FROM(
token,
"Type parameter list cannot be empty")}

invalid_bitwise_and:
| a=bitwise_and b='&' c='&' {
_PyPegen_tokens_are_adjacent(b, c)
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'and' or '&' instead of '&&'?")
: NULL
}

invalid_bitwise_or:
| a=bitwise_or b='|' c='|' {
_PyPegen_tokens_are_adjacent(b, c)
? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'or' or '|' instead of '||'?")
: NULL
}
1 change: 1 addition & 0 deletions Include/cpython/pystate.h
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ struct _ts {
struct _PyInterpreterFrame *base_frame;

struct _PyInterpreterFrame *last_profiled_frame;
uintptr_t last_profiled_frame_seq;

Py_tracefunc c_profilefunc;
Py_tracefunc c_tracefunc;
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_debug_offsets.h
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ typedef struct _Py_DebugOffsets {
uint64_t current_frame;
uint64_t base_frame;
uint64_t last_profiled_frame;
uint64_t last_profiled_frame_seq;
uint64_t thread_id;
uint64_t native_thread_id;
uint64_t datastack_chunk;
Expand Down Expand Up @@ -294,6 +295,7 @@ typedef struct _Py_DebugOffsets {
.current_frame = offsetof(PyThreadState, current_frame), \
.base_frame = offsetof(PyThreadState, base_frame), \
.last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \
.last_profiled_frame_seq = offsetof(PyThreadState, last_profiled_frame_seq), \
.thread_id = offsetof(PyThreadState, thread_id), \
.native_thread_id = offsetof(PyThreadState, native_thread_id), \
.datastack_chunk = offsetof(PyThreadState, datastack_chunk), \
Expand Down
3 changes: 3 additions & 0 deletions Include/internal/pycore_interpframe.h
Original file line number Diff line number Diff line change
Expand Up @@ -319,12 +319,15 @@ _PyThreadState_GetFrame(PyThreadState *tstate)
// This avoids corrupting the cache when transient frames (called and returned
// between profiler samples) update last_profiled_frame to addresses the
// profiler never saw.
// The sequence distinguishes this anchor from a later frame that reuses the
// same _PyInterpreterFrame address.
#define _PyThreadState_UpdateLastProfiledFrame(tstate, frame, previous) \
do { \
PyThreadState *tstate_ = (tstate); \
_PyInterpreterFrame *frame_ = (frame); \
if (tstate_->last_profiled_frame == frame_) { \
tstate_->last_profiled_frame = (previous); \
tstate_->last_profiled_frame_seq++; \
} \
} while (0)

Expand Down
30 changes: 17 additions & 13 deletions InternalDocs/frames.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,22 +142,26 @@ since the frame chain may have been in an inconsistent state due to concurrent u

### Remote Profiling Frame Cache

The `last_profiled_frame` field in `PyThreadState` supports an optimization for
remote profilers that sample call stacks from external processes. When a remote
profiler reads the call stack, it writes the current frame address to this field.
The eval loop then keeps this pointer valid by updating it to the parent frame
whenever a frame returns (in `_PyEval_FrameClearAndPop`).
The `last_profiled_frame` and `last_profiled_frame_seq` fields in
`PyThreadState` support an optimization for remote profilers that sample call
stacks from external processes. When a remote profiler reads the call stack, it
writes the current frame address to `last_profiled_frame`. The eval loop then
keeps this pointer valid by updating it to the parent frame whenever a frame
returns (in `_PyEval_FrameClearAndPop`) and increments the sequence.

This creates a "high-water mark" that always points to a frame still on the stack.
On subsequent samples, the profiler can walk from `current_frame` until it reaches
`last_profiled_frame`, knowing that frames from that point downward are unchanged
and can be retrieved from a cache. This significantly reduces the amount of remote
memory reads needed when call stacks are deep and stable at their base.

The update in `_PyEval_FrameClearAndPop` is guarded: it only writes when
`last_profiled_frame` is non-NULL AND matches the frame being popped. This
prevents transient frames (called and returned between profiler samples) from
corrupting the cache pointer, while avoiding any overhead when profiling is inactive.
`last_profiled_frame`, then validate the pointer and sequence before using cached
callers. This prevents a later frame that reuses the same `_PyInterpreterFrame`
address from being mistaken for the sampled frame. The cache significantly
reduces the amount of remote memory reads needed when call stacks are deep and
stable at their base.

The update in `_PyEval_FrameClearAndPop` is guarded: it only advances the
pointer and sequence when `last_profiled_frame` is non-NULL AND matches the
frame being popped. This prevents transient frames (called and returned between
profiler samples) from corrupting the cache anchor, while avoiding any overhead
when profiling is inactive.


### The Instruction Pointer
Expand Down
6 changes: 3 additions & 3 deletions Lib/_pydatetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,12 +272,12 @@ def _wrap_strftime(object, format, timetuple):
newformat.append(Zreplace)
# Note that datetime(1000, 1, 1).strftime('%G') == '1000' so
# year 1000 for %G can go on the fast path.
elif ((ch in 'YG' or ch in 'FC') and
object.year < 1000 and _need_normalize_century()):
elif (ch in 'YGFC' and timetuple[0] < 1000 and
_need_normalize_century()):
if ch == 'G':
year = int(_time.strftime("%G", timetuple))
else:
year = object.year
year = timetuple[0]
if ch == 'C':
push('{:02}'.format(year // 100))
else:
Expand Down
1 change: 1 addition & 0 deletions Lib/profiling/sampling/binary_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ def export(self, filename=None):
filename: Ignored (binary files are written incrementally)
"""
self._writer.finalize()
return True

@property
def total_samples(self):
Expand Down
30 changes: 21 additions & 9 deletions Lib/profiling/sampling/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,9 @@ def __call__(self, parser, namespace, values, option_string=None):
"binary": BinaryCollector,
}

BROWSER_COMPATIBLE_FORMATS = ("flamegraph", "diff_flamegraph", "heatmap")


def _setup_child_monitor(args, parent_pid):
# Build CLI args for child profilers (excluding --subprocesses to avoid recursion)
child_cli_args = _build_child_profiler_args(args)
Expand Down Expand Up @@ -528,8 +531,12 @@ def _add_format_options(parser, include_compression=True, include_binary=True):
output_group.add_argument(
"--browser",
action="store_true",
help="Automatically open HTML output (flamegraph, heatmap) in browser. "
"When using `--subprocesses`, only the main process opens the browser",
help=(
"Automatically open HTML output "
f"({', '.join('--' + f.replace('_', '-') for f in BROWSER_COMPATIBLE_FORMATS)}) "
"in browser. "
"When using `--subprocesses`, only the main process opens the browser"
),
)


Expand Down Expand Up @@ -789,13 +796,12 @@ def progress_callback(current, total):
args.outfile
or _generate_output_filename(args.format, os.getpid())
)
collector.export(filename)
export_ok = collector.export(filename)

# Auto-open browser for HTML output if --browser flag is set
if (
args.format in (
'flamegraph', 'diff_flamegraph', 'heatmap'
)
export_ok
and args.format in BROWSER_COMPATIBLE_FORMATS
and getattr(args, 'browser', False)
):
_open_in_browser(filename)
Expand Down Expand Up @@ -840,10 +846,14 @@ def _handle_output(collector, args, pid, mode):
filename = os.path.join(args.outfile, _generate_output_filename(args.format, pid))
else:
filename = args.outfile or _generate_output_filename(args.format, pid)
collector.export(filename)
export_ok = collector.export(filename)

# Auto-open browser for HTML output if --browser flag is set
if args.format in ('flamegraph', 'diff_flamegraph', 'heatmap') and getattr(args, 'browser', False):
if (
export_ok
and args.format in BROWSER_COMPATIBLE_FORMATS
and getattr(args, 'browser', False)
):
_open_in_browser(filename)


Expand Down Expand Up @@ -875,13 +885,15 @@ def _validate_args(args, parser):
if hasattr(args, 'live') and args.live:
parser.error("--subprocesses is incompatible with --live mode.")

# Async-aware mode is incompatible with --native, --no-gc, --mode, and --all-threads
# Async-aware mode is incompatible with options that need thread data.
if getattr(args, 'async_aware', False):
issues = []
if getattr(args, 'native', False):
issues.append("--native")
if not getattr(args, 'gc', True):
issues.append("--no-gc")
if getattr(args, 'format', None) == "binary":
issues.append("--binary")
if hasattr(args, 'mode') and args.mode != "wall":
issues.append(f"--mode={args.mode}")
if hasattr(args, 'all_threads') and args.all_threads:
Expand Down
6 changes: 5 additions & 1 deletion Lib/profiling/sampling/collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,11 @@ def collect_failed_sample(self):

@abstractmethod
def export(self, filename):
"""Export collected data to a file."""
"""Export collected data.

Returns:
bool: True if output was generated, False if there was no data to export.
"""

@staticmethod
def _filter_internal_frames(frames):
Expand Down
1 change: 1 addition & 0 deletions Lib/profiling/sampling/gecko_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -756,6 +756,7 @@ def spin():
print(
f"Open in Firefox Profiler: https://profiler.firefox.com/"
)
return True

def _build_marker_schema(self):
"""Build marker schema definitions for Firefox Profiler."""
Expand Down
3 changes: 2 additions & 1 deletion Lib/profiling/sampling/heatmap_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,7 @@ def export(self, output_path):
"""
if not self.file_samples:
print("Warning: No heatmap data to export")
return
return False

try:
output_dir = self._prepare_output_directory(output_path)
Expand All @@ -610,6 +610,7 @@ def export(self, output_path):
self._generate_index_html(output_dir / 'index.html', file_stats)

self._print_export_summary(output_dir, file_stats)
return True

except Exception as e:
print(f"Error: Failed to export heatmap: {e}")
Expand Down
1 change: 1 addition & 0 deletions Lib/profiling/sampling/jsonl_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ def export(self, filename):
)
self._write_message(output, self._build_end_record())
print(f"JSONL profile written to {filename}")
return True

def _build_meta_record(self):
record = {
Expand Down
1 change: 1 addition & 0 deletions Lib/profiling/sampling/pstats_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ def collect(self, stack_frames, timestamps_us=None):
def export(self, filename):
self.create_stats()
self._dump_stats(filename)
return True

def _dump_stats(self, file):
stats_with_marker = dict(self.stats)
Expand Down
4 changes: 3 additions & 1 deletion Lib/profiling/sampling/stack_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def export(self, filename):
for stack, count in lines:
f.write(f"{stack} {count}\n")
print(f"Collapsed stack output written to {filename}")
return True


class FlamegraphCollector(StackTraceCollector):
Expand Down Expand Up @@ -161,14 +162,15 @@ def export(self, filename):
print(
"Warning: No functions found in profiling data. Check if sampling captured any data."
)
return
return False

html_content = self._create_flamegraph_html(flamegraph_data)

with open(filename, "w", encoding="utf-8") as f:
f.write(html_content)

print(f"Flamegraph saved to: {filename}")
return True

@staticmethod
@functools.lru_cache(maxsize=None)
Expand Down
5 changes: 5 additions & 0 deletions Lib/test/datetimetester.py
Original file line number Diff line number Diff line change
Expand Up @@ -4119,6 +4119,11 @@ def test_strftime_special(self):
self.assertEqual(t.strftime('\0'*1000), '\0'*1000)
self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}')
self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0')
# gh-152305: the year directives must not raise on a time.
for directive, expected in (('%Y', '1900'), ('%G', '1900'),
('%C', '19'), ('%F', '1900-01-01')):
with self.subTest(directive=directive):
self.assertEqual(t.strftime(directive), expected)

def test_format(self):
t = self.theclass(1, 2, 3, 4)
Expand Down
Loading
Loading