From ef0e0e0b032e7f60803857208c6e811844b74071 Mon Sep 17 00:00:00 2001 From: Ben Reisner Date: Mon, 29 Jun 2026 10:48:56 -0700 Subject: [PATCH] Release free and stale pages from the page heap. When this feature is enabled, we periodically inspect non-hugepage-backed PageTrackers. We subrelease any pages that are both stale and free. PiperOrigin-RevId: 939916698 --- tcmalloc/experiment_config.h | 2 + tcmalloc/global_stats.cc | 8 ++ tcmalloc/huge_page_aware_allocator.h | 9 +- tcmalloc/huge_page_filler.h | 82 ++++++++++- tcmalloc/huge_page_filler_fuzz.cc | 8 +- tcmalloc/huge_page_filler_test.cc | 154 ++++++++++++++++++++- tcmalloc/mock_huge_page_static_forwarder.h | 5 + tcmalloc/parameters.cc | 12 ++ tcmalloc/parameters.h | 2 + tcmalloc/testing/get_stats_test.cc | 15 ++ tcmalloc/variants.bzl | 6 + 11 files changed, 286 insertions(+), 17 deletions(-) diff --git a/tcmalloc/experiment_config.h b/tcmalloc/experiment_config.h index 6af66b2d2..3651b7671 100644 --- a/tcmalloc/experiment_config.h +++ b/tcmalloc/experiment_config.h @@ -33,6 +33,7 @@ enum class Experiment : int { TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING, // TODO: b/328301906 - Complete experiment. TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, // TODO: b/446814339 - Complete experiment. TEST_ONLY_TCMALLOC_POW2_SIZECLASS, + TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES, // TODO: b/527473378 - Complete experiment. TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, TEST_ONLY_TCMALLOC_SUBRELEASE_UNBACKED_PAGES, // TODO: b/525422238 - Complete experiment. // go/keep-sorted end @@ -60,6 +61,7 @@ inline constexpr ExperimentConfig experiments[] = { {Experiment::TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING, "TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING", /*brittle=*/true}, {Experiment::TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, "TEST_ONLY_TCMALLOC_HEAP_PARTITIONING"}, {Experiment::TEST_ONLY_TCMALLOC_POW2_SIZECLASS, "TEST_ONLY_TCMALLOC_POW2_SIZECLASS", /*brittle=*/true}, + {Experiment::TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES, "TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES"}, {Experiment::TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, "TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE", /*brittle=*/true}, {Experiment::TEST_ONLY_TCMALLOC_SUBRELEASE_UNBACKED_PAGES, "TEST_ONLY_TCMALLOC_SUBRELEASE_UNBACKED_PAGES"}, // go/keep-sorted end diff --git a/tcmalloc/global_stats.cc b/tcmalloc/global_stats.cc index ef22d1637..9a9604970 100644 --- a/tcmalloc/global_stats.cc +++ b/tcmalloc/global_stats.cc @@ -688,6 +688,10 @@ void DumpStats(Printer& out, int level) { EnableUnfilteredCollapse::kEnabled ? 1 : 0); + out.printf("PARAMETER tcmalloc_release_stale_pages %d\n", + Parameters::release_stale_pages() == ReleaseStalePages::kEnabled + ? 1 + : 0); } } @@ -952,6 +956,10 @@ void DumpStatsInPbtxt(Printer& out, int level) { region.PrintBool("tcmalloc_enable_unfiltered_collapse", Parameters::enable_unfiltered_collapse() == EnableUnfilteredCollapse::kEnabled); + + region.PrintBool( + "tcmalloc_release_stale_pages", + Parameters::release_stale_pages() == ReleaseStalePages::kEnabled); } bool GetNumericProperty(const char* name_data, size_t name_size, diff --git a/tcmalloc/huge_page_aware_allocator.h b/tcmalloc/huge_page_aware_allocator.h index 9df38183a..4f0c3c711 100644 --- a/tcmalloc/huge_page_aware_allocator.h +++ b/tcmalloc/huge_page_aware_allocator.h @@ -90,6 +90,10 @@ class StaticForwarder { return Parameters::release_max_cold_pages(); } + static ReleaseStalePages release_stale_pages() { + return Parameters::release_stale_pages(); + } + // Arena state. static Arena& arena(); @@ -1063,8 +1067,11 @@ inline void HugePageAwareAllocator::TreatHugepageTrackers( EnableCollapse enable_collapse) { const EnableUnfilteredCollapse enable_unfiltered_collapse = forwarder_.enable_unfiltered_collapse(); + const ReleaseStalePages release_stale_pages = + forwarder_.release_stale_pages(); PageHeapSpinLockHolder l; - filler_.TreatHugepageTrackers(enable_collapse, enable_unfiltered_collapse); + filler_.TreatHugepageTrackers(enable_collapse, enable_unfiltered_collapse, + release_stale_pages); FillerType::Tracker* pt; while ((pt = filler_.FetchFullyFreedTracker()) != nullptr) { ReleaseHugepage(pt); diff --git a/tcmalloc/huge_page_filler.h b/tcmalloc/huge_page_filler.h index 0cdfe6cc5..199aa824f 100644 --- a/tcmalloc/huge_page_filler.h +++ b/tcmalloc/huge_page_filler.h @@ -67,6 +67,9 @@ namespace tcmalloc_internal { constexpr absl::Duration kMaxCollapseLatencyThreshold = absl::Milliseconds(30); constexpr absl::Duration kMinCollapseLatencyThreshold = absl::Milliseconds(15); +// Interval for Page Tracker treatment. +constexpr absl::Duration kRecordInterval = absl::Minutes(5); + enum class HugePageTreatmentType : uint8_t { kSampled = 1 << 0, kCollapse = 1 << 1, @@ -260,6 +263,9 @@ class PageTracker : public TList::Elem { return released_by_page_; } + Length ReleaseStaleFree(PageBitmap stale, MemoryModifyFunction& unback) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); + // Attempts to collapse memory tracked by this tracker. Returns true if the // collapse was successful. MemoryModifyStatus Collapse(MemoryModifyFunction& collapse); @@ -1153,6 +1159,11 @@ enum class EnableUnfilteredCollapse : bool { kEnabled = true, }; +enum class ReleaseStalePages : bool { + kDisabled = false, + kEnabled = true, +}; + // This tracks a set of unfilled hugepages, and fulfills allocations // with a goal of filling some hugepages as tightly as possible and emptying // out the remainder. @@ -1341,7 +1352,8 @@ class HugePageFiller { void TreatHugepageTrackers( EnableCollapse enable_collapse, EnableUnfilteredCollapse enable_unfiltered_collapse, - PageFlagsBase* pageflags = nullptr, Residency* residency = nullptr) + ReleaseStalePages release_stale_pages, PageFlagsBase* pageflags = nullptr, + Residency* residency = nullptr) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); // Utility function to release free pages from a given `page_tracker` @@ -1357,6 +1369,11 @@ class HugePageFiller { Length HandleUnbackedHugePage(PageTracker* page_tracker, PageBitmap unbacked) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); + // Utility function to release stale free pages in a given `page_tracker` + // and handle accounting. + Length HandleReleaseStaleFreePages(PageTracker* tracker, PageBitmap stale) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); + private: // This class wraps an array of N TrackerLists and a Bitmap storing which // elements are non-empty. @@ -1595,6 +1612,33 @@ inline void PageTracker::Put(Range r, SpanAllocInfo span_alloc_info) { num_objects_ -= span_alloc_info.objects_per_span; } +inline Length PageTracker::ReleaseStaleFree(PageBitmap stale, + MemoryModifyFunction& unback) { + // We want to release pages that are BACKED (not released), FREE (not used in + // free_), and STALE. + PageBitmap not_to_release = released_by_page_ | free_.bits() | ~stale; + + size_t count = 0; + size_t index = 0; + size_t n; + while (not_to_release.NextFreeRange(index, &index, &n)) { + TC_ASSERT_EQ(released_by_page_.CountBits(index, n), 0); + PageId p = location_.first_page() + Length(index); + + if (ABSL_PREDICT_TRUE(ReleasePages(Range(p, Length(n)), unback))) { + // Mark pages as released. Amortize the update to released_count_. + released_by_page_.SetRange(index, n); + count += n; + } + index += n; + } + + released_count_ += count; + TC_ASSERT_LE(Length(released_count_), kPagesPerHugePage); + TC_ASSERT_EQ(released_by_page_.CountBits(), released_count_); + return Length(count); +} + inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) { size_t count = 0; size_t index = 0; @@ -1624,7 +1668,7 @@ inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) { PageId p = location_.first_page() + Length(free_index); if (ABSL_PREDICT_TRUE(ReleasePages(Range(p, Length(length)), unback))) { - // Mark pages as released. Amortize the update to release_count_. + // Mark pages as released. Amortize the update to released_count_. released_by_page_.SetRange(free_index, length); count += length; } @@ -2584,7 +2628,8 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { MemoryModifyFunction& collapse, HugePageFiller& page_filler, EnableCollapse enable_collapse, SubreleaseUnbackedMode subrelease_unbacked_mode, - EnableUnfilteredCollapse enable_unfiltered_collapse) + EnableUnfilteredCollapse enable_unfiltered_collapse, + ReleaseStalePages release_stale_pages) : clock_(clock), pageflags_(pageflags), residency_(residency), @@ -2592,7 +2637,8 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { page_filler_(page_filler), enable_collapse_(enable_collapse), subrelease_unbacked_mode_(subrelease_unbacked_mode), - enable_unfiltered_collapse_(enable_unfiltered_collapse) {} + enable_unfiltered_collapse_(enable_unfiltered_collapse), + release_stale_pages_(release_stale_pages) {} ~HugePageUnbackedTrackerTreatment() override = default; static void operator delete(void*) { __builtin_trap(); } @@ -2777,6 +2823,14 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { treatment_stats_.treated_pages_subreleased += released_length.raw_num(); } + } else if (release_stale_pages_ == ReleaseStalePages::kEnabled && + !residency_states_[i].tracker_state.stale.IsZero()) { + Length released_length = page_filler_.HandleReleaseStaleFreePages( + tracker, residency_states_[i].tracker_state.stale); + if (released_length > Length(0)) { + treatment_stats_.treated_pages_subreleased += + released_length.raw_num(); + } } if (subrelease_unbacked_mode_ == SubreleaseUnbackedMode::kEnabled) { @@ -2825,7 +2879,6 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { } static constexpr size_t kTotalTrackersToScan = 64; - static constexpr absl::Duration kRecordInterval = absl::Minutes(5); static constexpr size_t kMaxSwappedPagesForCollapse = 128; static constexpr size_t kMaxUnbackedPagesForCollapse = 64; @@ -2849,6 +2902,7 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { SubreleaseUnbackedMode subrelease_unbacked_mode_; EnableUnfilteredCollapse enable_unfiltered_collapse_; + ReleaseStalePages release_stale_pages_; }; // Returns true if backoff delay has reached the maximum threshold. @@ -2881,7 +2935,8 @@ template inline void HugePageFiller::TreatHugepageTrackers( EnableCollapse enable_collapse, EnableUnfilteredCollapse enable_unfiltered_collapse, - PageFlagsBase* pageflags, Residency* residency) { + ReleaseStalePages release_stale_pages, PageFlagsBase* pageflags, + Residency* residency) { if (enable_collapse == EnableCollapse::kEnabled && ShouldBackoffFromCollapse()) { enable_collapse = EnableCollapse::kDisabled; @@ -2894,7 +2949,8 @@ inline void HugePageFiller::TreatHugepageTrackers( set_anon_vma_name_); HugePageUnbackedTrackerTreatment unbacked_tracker_treatment( clock_, pageflags, residency, collapse_, *this, enable_collapse, - subrelease_unbacked_mode_, enable_unfiltered_collapse); + subrelease_unbacked_mode_, enable_unfiltered_collapse, + release_stale_pages); // Collect up to kTotalTrackersToScan trackers from our lists. regular_alloc_partial_released_[AccessDensityPrediction::kSparse].Iter( @@ -3006,6 +3062,18 @@ inline Length HugePageFiller::HandleUnbackedHugePage( return unmapped_length; } +template +inline Length HugePageFiller::HandleReleaseStaleFreePages( + PageTracker* tracker, PageBitmap stale) { + RemoveFromFillerList(tracker); + Length released_length = tracker->ReleaseStaleFree(stale, unback_); + subrelease_stats_.total_pages_subreleased += released_length; + unmapped_ += released_length; + unmapping_unaccounted_ += released_length; + AddToFillerList(tracker); + return released_length; +} + template inline void HugePageFiller::Print(Printer& out, bool everything, PageFlagsBase& pageflags) { diff --git a/tcmalloc/huge_page_filler_fuzz.cc b/tcmalloc/huge_page_filler_fuzz.cc index e10fad487..d40f106f7 100644 --- a/tcmalloc/huge_page_filler_fuzz.cc +++ b/tcmalloc/huge_page_filler_fuzz.cc @@ -275,15 +275,17 @@ struct TreatTrackers { bool enable_collapse; bool use_userspace_collapse_heuristics; bool enable_unfiltered_collapse; + bool enable_release_stale_pages; template friend void AbslStringify(Sink& sink, const TreatTrackers& t) { absl::Format(&sink, "TreatTrackers{.enable_collapse=%v, " ".use_userspace_collapse_heuristics=%v, " - ".enable_unfiltered_collapse=%v}", + ".enable_unfiltered_collapse=%v, " + ".enable_release_stale_pages=%v}", t.enable_collapse, t.use_userspace_collapse_heuristics, - t.enable_unfiltered_collapse); + t.enable_unfiltered_collapse, t.enable_release_stale_pages); } }; @@ -669,7 +671,7 @@ void FuzzFiller(const std::vector& instructions, arg.enable_unfiltered_collapse ? EnableUnfilteredCollapse::kEnabled : EnableUnfilteredCollapse::kDisabled, - &pageflags, &residency); + ReleaseStalePages::kDisabled, &pageflags, &residency); treating_trackers = false; absl::flat_hash_set& released_set = ReleasedPages(); while (PageTracker* pt = filler.FetchFullyFreedTracker()) { diff --git a/tcmalloc/huge_page_filler_test.cc b/tcmalloc/huge_page_filler_test.cc index a776056d5..3276dc587 100644 --- a/tcmalloc/huge_page_filler_test.cc +++ b/tcmalloc/huge_page_filler_test.cc @@ -272,6 +272,11 @@ class PageTrackerTest : public testing::Test { return tracker_.ReleaseFree(mock_unback_); } + Length ReleaseStaleFree(PageBitmap stale) { + PageHeapSpinLockHolder l; + return tracker_.ReleaseStaleFree(stale, mock_unback_); + } + // strict because release calls should only happen when we ask MockMemoryInterface mock_unback_; @@ -290,10 +295,16 @@ class FakePageFlags : public PageFlagsBase { } PageFlagsBitmaps GetSinglePageBitmaps(const void* addr) override { - PageFlagsBitmaps ret; - ret.stale.SetBit(0); - ret.status = absl::StatusCode::kOk; - return ret; + PageId p = PageIdContaining(addr); + HugePage hp = HugePageContaining(p); + EXPECT_TRUE(page_flags_bitmaps_.contains(hp.start_addr())); + return page_flags_bitmaps_[hp.start_addr()]; + } + + void SetStaleBitmap(const void* addr, const ResidencyBitmap& stale) { + PageId p = PageIdContaining(addr); + HugePage hp = HugePageContaining(p); + page_flags_bitmaps_[hp.start_addr()] = {stale, absl::StatusCode::kOk}; } void MarkHugePageBacked(void* addr, bool is_hugepage_backed) { @@ -320,6 +331,7 @@ class FakePageFlags : public PageFlagsBase { private: absl::flat_hash_map> is_hugepage_backed_; + absl::flat_hash_map page_flags_bitmaps_; }; class FakeResidency : public Residency { @@ -554,6 +566,42 @@ TEST_F(PageTrackerTest, ReleasingRetainFailure) { EXPECT_EQ(tracker_.free_pages(), a1.n + a2.n + a3.n + a4.n); } +TEST_F(PageTrackerTest, ReleaseStaleFree) { + static const Length kAllocSize = kPagesPerHugePage / 4; + SpanAllocInfo info = {1, AccessDensityPrediction::kSparse}; + PAlloc a1 = Get(kAllocSize, info); + PAlloc a2 = Get(kAllocSize, info); + PAlloc a3 = Get(kAllocSize, info); + PAlloc a4 = Get(kAllocSize, info); + + // Free a1. + Put(a1); + + PageBitmap stale; + stale.Clear(); + + // Initially stale is empty, so nothing should be released. + EXPECT_EQ(ReleaseStaleFree(stale), Length(0)); + mock_unback_.VerifyAndClear(); + + // Manually set some of the stale bits that are not part of a1 to true. These + // don't overlap with the pages freed from a1, so nothing will be freed. + stale.SetRange(64, 2); + EXPECT_EQ(ReleaseStaleFree(stale), Length(0)); + mock_unback_.VerifyAndClear(); + + stale.SetRange(0, 2); + + ExpectUnbackPages(PAlloc(a1.p, Length(2), info), /*success=*/true); + EXPECT_EQ(ReleaseStaleFree(stale), Length(2)); + mock_unback_.VerifyAndClear(); + + // Back to normal. + Put(a2); + Put(a3); + Put(a4); +} + TEST_F(PageTrackerTest, Defrag) { absl::BitGen rng; const Length N = absl::GetFlag(FLAGS_page_tracker_defrag_lim); @@ -1113,14 +1161,15 @@ class FillerTest : public testing::Test { void TreatHugepageTrackers( EnableCollapse enable_collapse, EnableUnfilteredCollapse enable_unfiltered_collapse, - PageFlagsBase* pageflags, Residency* residency) { + PageFlagsBase* pageflags, Residency* residency, + ReleaseStalePages release_stale_pages = ReleaseStalePages::kDisabled) { // Note that scoped pageheap lock isn't used here. This is because the // pageheap lock is manually unlocked before the collapse operation, and the // scoped lock doesn't recognize the manual unlock. In tests, collapse // allocates, so we use manual lock and unlock here. pageheap_lock.lock(); filler_.TreatHugepageTrackers(enable_collapse, enable_unfiltered_collapse, - pageflags, residency); + release_stale_pages, pageflags, residency); pageheap_lock.unlock(); } @@ -1248,6 +1297,99 @@ TEST_F(FillerTest, Density) { } } +TEST_F(FillerTest, ReleaseStaleFree) { + // Disable randomization for predictable layout + randomize_density_ = false; + + static const Length kAllocSize = kPagesPerHugePage / 4; + SpanAllocInfo info = {1, AccessDensityPrediction::kSparse}; + + PAlloc a1 = AllocateWithSpanAllocInfo(kAllocSize, info); + PAlloc a2 = AllocateWithSpanAllocInfo(kAllocSize, info); + PAlloc a3 = AllocateWithSpanAllocInfo(kAllocSize, info); + PAlloc a4 = AllocateWithSpanAllocInfo(kAllocSize, info); + + ASSERT_EQ(a1.pt, a2.pt); + ASSERT_EQ(a1.pt, a3.pt); + ASSERT_EQ(a1.pt, a4.pt); + + // Assert locations to ensure layout assumptions hold + ASSERT_EQ((a1.p - a1.pt->location().first_page()).raw_num(), 0); + + Delete(a1); // Free a1 + + Bitmap empty_bitmap; + + FakePageFlags pageflags; + pageflags.SetStaleBitmap(a1.pt->location().start_addr(), empty_bitmap); + pageflags.MarkHugePageBacked(a1.pt->location().start_addr(), false); + + FakeResidency residency; + empty_bitmap.Clear(); + residency.SetUnbackedAndSwappedBitmaps(a1.pt->location().start_addr(), + empty_bitmap, empty_bitmap); + + // Measure initial stats + HugePageTreatmentStats stats_baseline = GetHugePageTreatmentStats(); + + // 1. Initially stale is empty, so nothing should be released. + TreatHugepageTrackers(EnableCollapse::kDisabled, + EnableUnfilteredCollapse::kDisabled, &pageflags, + &residency, ReleaseStalePages::kEnabled); + + HugePageTreatmentStats stats_after = GetHugePageTreatmentStats(); + EXPECT_EQ(stats_after.treated_pages_subreleased - + stats_baseline.treated_pages_subreleased, + 0); + + // Advance clock to bypass the record interval. + FakeClock::Advance(kRecordInterval + absl::Minutes(1)); + + // 2. Manually set some of the stale bits that are not part of a1 to true. + // These don't overlap with the pages freed from a1, so nothing will be freed. + // SetRange(128, 4) corresponds to TCMalloc pages 64-65 (assuming ratio=2). + // TCMalloc pages 64-65 belong to a2, which is USED. + ResidencyBitmap stale; + stale.Clear(); + stale.SetRange(128, 4); + pageflags.SetStaleBitmap(a1.pt->location().start_addr(), stale); + + stats_baseline = GetHugePageTreatmentStats(); + TreatHugepageTrackers(EnableCollapse::kDisabled, + EnableUnfilteredCollapse::kDisabled, &pageflags, + &residency, ReleaseStalePages::kEnabled); + + stats_after = GetHugePageTreatmentStats(); + EXPECT_EQ(stats_after.treated_pages_subreleased - + stats_baseline.treated_pages_subreleased, + 0); + + // Advance clock to bypass the record interval. + FakeClock::Advance(kRecordInterval + absl::Minutes(1)); + + // 3. Now set some bits stale corresponding to a1 (which is FREE). + // SetRange(0, 3) sets OS pages 0, 1, 2. + // They correspond to TCMalloc pages 0 and 1. + stale.Clear(); + stale.SetRange(0, 3); + pageflags.SetStaleBitmap(a1.pt->location().start_addr(), stale); + + stats_baseline = GetHugePageTreatmentStats(); + TreatHugepageTrackers(EnableCollapse::kDisabled, + EnableUnfilteredCollapse::kDisabled, &pageflags, + &residency, ReleaseStalePages::kEnabled); + + stats_after = GetHugePageTreatmentStats(); + EXPECT_EQ(stats_after.treated_pages_subreleased - + stats_baseline.treated_pages_subreleased, + 2); + + // Clean up + Delete(a2); + Delete(a3); + Delete(a4); +} + TEST_F(FillerTest, ReleaseFreePagesWhenAnyPageIsSwappedRespectsClock) { const Length kAlloc = kPagesPerHugePage; std::vector p1 = AllocateVector(kAlloc - Length(1)); diff --git a/tcmalloc/mock_huge_page_static_forwarder.h b/tcmalloc/mock_huge_page_static_forwarder.h index 15e995cf7..b17a5116a 100644 --- a/tcmalloc/mock_huge_page_static_forwarder.h +++ b/tcmalloc/mock_huge_page_static_forwarder.h @@ -95,6 +95,10 @@ class FakeStaticForwarder { void set_release_max_cold_pages(bool value) { release_max_cold_pages_ = value; } + ReleaseStalePages release_stale_pages() const { return release_stale_pages_; } + void set_release_stale_pages(ReleaseStalePages value) { + release_stale_pages_ = value; + } bool BackAllocations() const { return back_allocations_; } void SetBackAllocations(bool value) { back_allocations_ = value; } @@ -239,6 +243,7 @@ class FakeStaticForwarder { EnableUnfilteredCollapse enable_unfiltered_collapse_ = EnableUnfilteredCollapse::kDisabled; Arena arena_; + ReleaseStalePages release_stale_pages_ = ReleaseStalePages::kDisabled; std::atomic fake_allocation_ = 0x1000; diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index 2b1ab13bd..c9db24918 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -356,6 +356,18 @@ want_span_lifetime_tracking() { return central_freelist_internal::LifetimeTracking::kDisabled; } +ReleaseStalePages Parameters::release_stale_pages() { + ABSL_CONST_INIT static absl::once_flag flag; + ABSL_CONST_INIT static std::atomic v{ + ReleaseStalePages::kDisabled}; + absl::base_internal::LowLevelCallOnce(&flag, [&]() { + v.store(ReleaseStalePages{IsExperimentActive( + Experiment::TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES)}, + std::memory_order_relaxed); + }); + return v.load(std::memory_order_relaxed); +} + central_freelist_internal::LifetimeTracking Parameters::span_lifetime_tracking() { ABSL_CONST_INIT static absl::once_flag flag; diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index 65918df6f..0a80bd1e7 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -196,6 +196,8 @@ class Parameters { static central_freelist_internal::LifetimeTracking span_lifetime_tracking(); + static ReleaseStalePages release_stale_pages(); + private: friend void ::TCMalloc_Internal_SetBackgroundReleaseRate(size_t v); friend void ::TCMalloc_Internal_SetGuardedSamplingInterval(int64_t v); diff --git a/tcmalloc/testing/get_stats_test.cc b/tcmalloc/testing/get_stats_test.cc index f8662e932..0ea3bfad3 100644 --- a/tcmalloc/testing/get_stats_test.cc +++ b/tcmalloc/testing/get_stats_test.cc @@ -164,6 +164,12 @@ TEST_F(GetStatsTest, Pbtxt) { EXPECT_THAT(buf, ContainsRegex("max_cpu_cache_touched: [0-9]+")); } + if (IsExperimentActive(Experiment::TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES)) { + EXPECT_THAT(buf, HasSubstr("tcmalloc_release_stale_pages: true")); + } else { + EXPECT_THAT(buf, HasSubstr("tcmalloc_release_stale_pages: false")); + } + sized_delete(alloc, kSize); } @@ -283,6 +289,15 @@ TEST_F(GetStatsTest, Parameters) { } else { EXPECT_THAT(pbtxt, HasSubstr(R"(min_hot_access_hint: 1)")); } + + if (IsExperimentActive( + Experiment::TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES)) { + EXPECT_THAT(buf, + HasSubstr(R"(PARAMETER tcmalloc_release_stale_pages 1)")); + } else { + EXPECT_THAT(buf, + HasSubstr(R"(PARAMETER tcmalloc_release_stale_pages 0)")); + } } Parameters::set_hpaa_subrelease(true); diff --git a/tcmalloc/variants.bzl b/tcmalloc/variants.bzl index 736c36dad..91b719d81 100644 --- a/tcmalloc/variants.bzl +++ b/tcmalloc/variants.bzl @@ -283,6 +283,12 @@ test_variants = [ "deps": ["//tcmalloc:common_8k_pages"], "env": {"BORG_EXPERIMENTS": "TCMALLOC_HUGE_REGION_ADAPTIVE_RELEASE"}, }, + { + "name": "tcmalloc_release_stale_pages", + "malloc": "//tcmalloc", + "deps": ["//tcmalloc:common_8k_pages"], + "env": {"BORG_EXPERIMENTS": "TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES"}, + }, ] def create_tcmalloc_library(