Skip to content

Release free and stale pages from the page heap. #418

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Release free and stale pages from the page heap. #418

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions tcmalloc/experiment_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ enum class Experiment : int {
TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING, // TODO: b/328301906 - Complete experiment.
TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, // TODO: b/446814339 - Complete experiment.
TEST_ONLY_TCMALLOC_POW2_SIZECLASS,
TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES, // TODO: b/527473378 - Complete experiment.
TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE,
TEST_ONLY_TCMALLOC_SUBRELEASE_UNBACKED_PAGES, // TODO: b/525422238 - Complete experiment.
// go/keep-sorted end
Expand Down Expand Up @@ -60,6 +61,7 @@ inline constexpr ExperimentConfig experiments[] = {
{Experiment::TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING, "TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING", /*brittle=*/true},
{Experiment::TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, "TEST_ONLY_TCMALLOC_HEAP_PARTITIONING"},
{Experiment::TEST_ONLY_TCMALLOC_POW2_SIZECLASS, "TEST_ONLY_TCMALLOC_POW2_SIZECLASS", /*brittle=*/true},
{Experiment::TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES, "TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES"},
{Experiment::TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, "TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE", /*brittle=*/true},
{Experiment::TEST_ONLY_TCMALLOC_SUBRELEASE_UNBACKED_PAGES, "TEST_ONLY_TCMALLOC_SUBRELEASE_UNBACKED_PAGES"},
// go/keep-sorted end
Expand Down
8 changes: 8 additions & 0 deletions tcmalloc/global_stats.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -688,6 +688,10 @@ void DumpStats(Printer& out, int level) {
EnableUnfilteredCollapse::kEnabled
? 1
: 0);
out.printf("PARAMETER tcmalloc_release_stale_pages %d\n",
Parameters::release_stale_pages() == ReleaseStalePages::kEnabled
? 1
: 0);
}
}

Expand Down Expand Up @@ -952,6 +956,10 @@ void DumpStatsInPbtxt(Printer& out, int level) {
region.PrintBool("tcmalloc_enable_unfiltered_collapse",
Parameters::enable_unfiltered_collapse() ==
EnableUnfilteredCollapse::kEnabled);

region.PrintBool(
"tcmalloc_release_stale_pages",
Parameters::release_stale_pages() == ReleaseStalePages::kEnabled);
}

bool GetNumericProperty(const char* name_data, size_t name_size,
Expand Down
9 changes: 8 additions & 1 deletion tcmalloc/huge_page_aware_allocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ class StaticForwarder {
return Parameters::release_max_cold_pages();
}

static ReleaseStalePages release_stale_pages() {
return Parameters::release_stale_pages();
}

// Arena state.
static Arena& arena();

Expand Down Expand Up @@ -1063,8 +1067,11 @@ inline void HugePageAwareAllocator<Forwarder>::TreatHugepageTrackers(
EnableCollapse enable_collapse) {
const EnableUnfilteredCollapse enable_unfiltered_collapse =
forwarder_.enable_unfiltered_collapse();
const ReleaseStalePages release_stale_pages =
forwarder_.release_stale_pages();
PageHeapSpinLockHolder l;
filler_.TreatHugepageTrackers(enable_collapse, enable_unfiltered_collapse);
filler_.TreatHugepageTrackers(enable_collapse, enable_unfiltered_collapse,
release_stale_pages);
FillerType::Tracker* pt;
while ((pt = filler_.FetchFullyFreedTracker()) != nullptr) {
ReleaseHugepage(pt);
Expand Down
82 changes: 75 additions & 7 deletions tcmalloc/huge_page_filler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ namespace tcmalloc_internal {
constexpr absl::Duration kMaxCollapseLatencyThreshold = absl::Milliseconds(30);
constexpr absl::Duration kMinCollapseLatencyThreshold = absl::Milliseconds(15);

// Interval for Page Tracker treatment.
constexpr absl::Duration kRecordInterval = absl::Minutes(5);

enum class HugePageTreatmentType : uint8_t {
kSampled = 1 << 0,
kCollapse = 1 << 1,
Expand Down Expand Up @@ -260,6 +263,9 @@ class PageTracker : public TList<PageTracker>::Elem {
return released_by_page_;
}

Length ReleaseStaleFree(PageBitmap stale, MemoryModifyFunction& unback)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

// Attempts to collapse memory tracked by this tracker. Returns true if the
// collapse was successful.
MemoryModifyStatus Collapse(MemoryModifyFunction& collapse);
Expand Down Expand Up @@ -1153,6 +1159,11 @@ enum class EnableUnfilteredCollapse : bool {
kEnabled = true,
};

enum class ReleaseStalePages : bool {
kDisabled = false,
kEnabled = true,
};

// This tracks a set of unfilled hugepages, and fulfills allocations
// with a goal of filling some hugepages as tightly as possible and emptying
// out the remainder.
Expand Down Expand Up @@ -1341,7 +1352,8 @@ class HugePageFiller {
void TreatHugepageTrackers(
EnableCollapse enable_collapse,
EnableUnfilteredCollapse enable_unfiltered_collapse,
PageFlagsBase* pageflags = nullptr, Residency* residency = nullptr)
ReleaseStalePages release_stale_pages, PageFlagsBase* pageflags = nullptr,
Residency* residency = nullptr)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

// Utility function to release free pages from a given `page_tracker`
Expand All @@ -1357,6 +1369,11 @@ class HugePageFiller {
Length HandleUnbackedHugePage(PageTracker* page_tracker, PageBitmap unbacked)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

// Utility function to release stale free pages in a given `page_tracker`
// and handle accounting.
Length HandleReleaseStaleFreePages(PageTracker* tracker, PageBitmap stale)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock);

private:
// This class wraps an array of N TrackerLists and a Bitmap storing which
// elements are non-empty.
Expand Down Expand Up @@ -1595,6 +1612,33 @@ inline void PageTracker::Put(Range r, SpanAllocInfo span_alloc_info) {
num_objects_ -= span_alloc_info.objects_per_span;
}

inline Length PageTracker::ReleaseStaleFree(PageBitmap stale,
MemoryModifyFunction& unback) {
// We want to release pages that are BACKED (not released), FREE (not used in
// free_), and STALE.
PageBitmap not_to_release = released_by_page_ | free_.bits() | ~stale;

size_t count = 0;
size_t index = 0;
size_t n;
while (not_to_release.NextFreeRange(index, &index, &n)) {
TC_ASSERT_EQ(released_by_page_.CountBits(index, n), 0);
PageId p = location_.first_page() + Length(index);

if (ABSL_PREDICT_TRUE(ReleasePages(Range(p, Length(n)), unback))) {
// Mark pages as released. Amortize the update to released_count_.
released_by_page_.SetRange(index, n);
count += n;
}
index += n;
}

released_count_ += count;
TC_ASSERT_LE(Length(released_count_), kPagesPerHugePage);
TC_ASSERT_EQ(released_by_page_.CountBits(), released_count_);
return Length(count);
}

inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) {
size_t count = 0;
size_t index = 0;
Expand Down Expand Up @@ -1624,7 +1668,7 @@ inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) {
PageId p = location_.first_page() + Length(free_index);

if (ABSL_PREDICT_TRUE(ReleasePages(Range(p, Length(length)), unback))) {
// Mark pages as released. Amortize the update to release_count_.
// Mark pages as released. Amortize the update to released_count_.
released_by_page_.SetRange(free_index, length);
count += length;
}
Expand Down Expand Up @@ -2584,15 +2628,17 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment {
MemoryModifyFunction& collapse, HugePageFiller<TrackerType>& page_filler,
EnableCollapse enable_collapse,
SubreleaseUnbackedMode subrelease_unbacked_mode,
EnableUnfilteredCollapse enable_unfiltered_collapse)
EnableUnfilteredCollapse enable_unfiltered_collapse,
ReleaseStalePages release_stale_pages)
: clock_(clock),
pageflags_(pageflags),
residency_(residency),
collapse_(collapse),
page_filler_(page_filler),
enable_collapse_(enable_collapse),
subrelease_unbacked_mode_(subrelease_unbacked_mode),
enable_unfiltered_collapse_(enable_unfiltered_collapse) {}
enable_unfiltered_collapse_(enable_unfiltered_collapse),
release_stale_pages_(release_stale_pages) {}
~HugePageUnbackedTrackerTreatment() override = default;

static void operator delete(void*) { __builtin_trap(); }
Expand Down Expand Up @@ -2777,6 +2823,14 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment {
treatment_stats_.treated_pages_subreleased +=
released_length.raw_num();
}
} else if (release_stale_pages_ == ReleaseStalePages::kEnabled &&
!residency_states_[i].tracker_state.stale.IsZero()) {
Length released_length = page_filler_.HandleReleaseStaleFreePages(
tracker, residency_states_[i].tracker_state.stale);
if (released_length > Length(0)) {
treatment_stats_.treated_pages_subreleased +=
released_length.raw_num();
}
}

if (subrelease_unbacked_mode_ == SubreleaseUnbackedMode::kEnabled) {
Expand Down Expand Up @@ -2825,7 +2879,6 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment {
}

static constexpr size_t kTotalTrackersToScan = 64;
static constexpr absl::Duration kRecordInterval = absl::Minutes(5);
static constexpr size_t kMaxSwappedPagesForCollapse = 128;
static constexpr size_t kMaxUnbackedPagesForCollapse = 64;

Expand All @@ -2849,6 +2902,7 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment {
SubreleaseUnbackedMode subrelease_unbacked_mode_;

EnableUnfilteredCollapse enable_unfiltered_collapse_;
ReleaseStalePages release_stale_pages_;
};

// Returns true if backoff delay has reached the maximum threshold.
Expand Down Expand Up @@ -2881,7 +2935,8 @@ template <class TrackerType>
inline void HugePageFiller<TrackerType>::TreatHugepageTrackers(
EnableCollapse enable_collapse,
EnableUnfilteredCollapse enable_unfiltered_collapse,
PageFlagsBase* pageflags, Residency* residency) {
ReleaseStalePages release_stale_pages, PageFlagsBase* pageflags,
Residency* residency) {
if (enable_collapse == EnableCollapse::kEnabled &&
ShouldBackoffFromCollapse()) {
enable_collapse = EnableCollapse::kDisabled;
Expand All @@ -2894,7 +2949,8 @@ inline void HugePageFiller<TrackerType>::TreatHugepageTrackers(
set_anon_vma_name_);
HugePageUnbackedTrackerTreatment<TrackerType> unbacked_tracker_treatment(
clock_, pageflags, residency, collapse_, *this, enable_collapse,
subrelease_unbacked_mode_, enable_unfiltered_collapse);
subrelease_unbacked_mode_, enable_unfiltered_collapse,
release_stale_pages);

// Collect up to kTotalTrackersToScan trackers from our lists.
regular_alloc_partial_released_[AccessDensityPrediction::kSparse].Iter(
Expand Down Expand Up @@ -3006,6 +3062,18 @@ inline Length HugePageFiller<TrackerType>::HandleUnbackedHugePage(
return unmapped_length;
}

template <class TrackerType>
inline Length HugePageFiller<TrackerType>::HandleReleaseStaleFreePages(
PageTracker* tracker, PageBitmap stale) {
RemoveFromFillerList(tracker);
Length released_length = tracker->ReleaseStaleFree(stale, unback_);
subrelease_stats_.total_pages_subreleased += released_length;
unmapped_ += released_length;
unmapping_unaccounted_ += released_length;
AddToFillerList(tracker);
return released_length;
}

template <class TrackerType>
inline void HugePageFiller<TrackerType>::Print(Printer& out, bool everything,
PageFlagsBase& pageflags) {
Expand Down
8 changes: 5 additions & 3 deletions tcmalloc/huge_page_filler_fuzz.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,15 +275,17 @@ struct TreatTrackers {
bool enable_collapse;
bool use_userspace_collapse_heuristics;
bool enable_unfiltered_collapse;
bool enable_release_stale_pages;

template <typename Sink>
friend void AbslStringify(Sink& sink, const TreatTrackers& t) {
absl::Format(&sink,
"TreatTrackers{.enable_collapse=%v, "
".use_userspace_collapse_heuristics=%v, "
".enable_unfiltered_collapse=%v}",
".enable_unfiltered_collapse=%v, "
".enable_release_stale_pages=%v}",
t.enable_collapse, t.use_userspace_collapse_heuristics,
t.enable_unfiltered_collapse);
t.enable_unfiltered_collapse, t.enable_release_stale_pages);
}
};

Expand Down Expand Up @@ -669,7 +671,7 @@ void FuzzFiller(const std::vector<Instruction>& instructions,
arg.enable_unfiltered_collapse
? EnableUnfilteredCollapse::kEnabled
: EnableUnfilteredCollapse::kDisabled,
&pageflags, &residency);
ReleaseStalePages::kDisabled, &pageflags, &residency);
treating_trackers = false;
absl::flat_hash_set<PageId>& released_set = ReleasedPages();
while (PageTracker* pt = filler.FetchFullyFreedTracker()) {
Expand Down
Loading
Loading