diff --git a/tcmalloc/experiment_config.h b/tcmalloc/experiment_config.h index 593f78eba..8acac6c07 100644 --- a/tcmalloc/experiment_config.h +++ b/tcmalloc/experiment_config.h @@ -28,6 +28,7 @@ enum class Experiment : int { TCMALLOC_PER_CPU_CACHE_SIZE_1MB, // TODO: b/514747820 - Complete experiment. TCMALLOC_PGHO_EXPERIMENT, // TODO: b/460486507 - Complete experiment. TCMALLOC_REUSE_SIZE_CLASSES_ABLATION, // TODO: b/524296402 - Complete experiment. + TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS, // TODO: b/527907199 - Complete experiment. TEST_ONLY_L3_AWARE, // TODO: b/239977380 - Complete experiment. TEST_ONLY_MM_VCPU, // TODO: b/245776120 - Complete experiment. TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, // TODO: b/446814339 - Complete experiment. @@ -58,6 +59,7 @@ inline constexpr ExperimentConfig experiments[] = { {Experiment::TCMALLOC_PER_CPU_CACHE_SIZE_1MB, "TCMALLOC_PER_CPU_CACHE_SIZE_1MB"}, {Experiment::TCMALLOC_PGHO_EXPERIMENT, "TCMALLOC_PGHO_EXPERIMENT"}, {Experiment::TCMALLOC_REUSE_SIZE_CLASSES_ABLATION, "TCMALLOC_REUSE_SIZE_CLASSES_ABLATION"}, + {Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS, "TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS"}, {Experiment::TEST_ONLY_L3_AWARE, "TEST_ONLY_L3_AWARE"}, {Experiment::TEST_ONLY_MM_VCPU, "TEST_ONLY_MM_VCPU"}, {Experiment::TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, "TEST_ONLY_TCMALLOC_HEAP_PARTITIONING"}, diff --git a/tcmalloc/global_stats.cc b/tcmalloc/global_stats.cc index 9a9604970..f7f8be821 100644 --- a/tcmalloc/global_stats.cc +++ b/tcmalloc/global_stats.cc @@ -33,6 +33,7 @@ #include "tcmalloc/experiment_config.h" #include "tcmalloc/guarded_page_allocator.h" #include "tcmalloc/huge_page_filler.h" +#include "tcmalloc/huge_page_options.h" #include "tcmalloc/huge_pages.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/cpu_utils.h" @@ -636,6 +637,11 @@ void DumpStats(Printer& out, int level) { Parameters::release_pages_from_huge_region() ? 1 : 0); out.printf("PARAMETER tcmalloc_huge_region_adaptive_release %d\n", Parameters::huge_region_adaptive_release() ? 1 : 0); + out.printf("PARAMETER madvise_cold_regions_nohugepage %d\n", + Parameters::madvise_cold_regions_nohugepage() == + MadviseRegionsNoHugepage::kEnabled + ? 1 + : 0); out.printf("PARAMETER tcmalloc_use_wider_slabs %d\n", tc_globals.cpu_cache().UseWiderSlabs() ? 1 : 0); out.printf("PARAMETER heap_partitioning %d\n", @@ -916,6 +922,9 @@ void DumpStatsInPbtxt(Printer& out, int level) { Parameters::release_pages_from_huge_region()); region.PrintBool("tcmalloc_huge_region_adaptive_release", Parameters::huge_region_adaptive_release()); + region.PrintBool("madvise_cold_regions_nohugepage", + Parameters::madvise_cold_regions_nohugepage() == + MadviseRegionsNoHugepage::kEnabled); region.PrintI64("profile_sampling_interval", Parameters::profile_sampling_interval()); region.PrintRaw("percpu_vcpu_type", @@ -963,7 +972,7 @@ void DumpStatsInPbtxt(Printer& out, int level) { } bool GetNumericProperty(const char* name_data, size_t name_size, - size_t* value) { + size_t* absl_nonnull value) { TC_ASSERT(name_data != nullptr || name_size == 0); TC_ASSERT_NE(value, nullptr); const absl::string_view name(name_data, name_size); diff --git a/tcmalloc/huge_page_aware_allocator.h b/tcmalloc/huge_page_aware_allocator.h index 4f0c3c711..abfdaac0e 100644 --- a/tcmalloc/huge_page_aware_allocator.h +++ b/tcmalloc/huge_page_aware_allocator.h @@ -15,7 +15,9 @@ #ifndef TCMALLOC_HUGE_PAGE_AWARE_ALLOCATOR_H_ #define TCMALLOC_HUGE_PAGE_AWARE_ALLOCATOR_H_ +#include #include +#include #include #include @@ -94,6 +96,10 @@ class StaticForwarder { return Parameters::release_stale_pages(); } + static MadviseRegionsNoHugepage madvise_cold_regions_nohugepage() { + return Parameters::madvise_cold_regions_nohugepage(); + } + // Arena state. static Arena& arena(); @@ -844,6 +850,16 @@ template inline bool HugePageAwareAllocator::AddRegion() { HugeRange r = alloc_.Get(HugeRegion::size()); if (!r.valid()) return false; + + if (forwarder_.madvise_cold_regions_nohugepage() == + MadviseRegionsNoHugepage::kEnabled) { + bool madvise_failed = false; + do { + madvise_failed = + madvise(r.start_addr(), r.len().in_bytes(), MADV_NOHUGEPAGE) != 0; + } while (madvise_failed && errno == EAGAIN); + } + HugeRegion* region = region_allocator_.New(r, unback_, set_anon_vma_name_); regions_.Contribute(region); return true; diff --git a/tcmalloc/huge_page_aware_allocator_fuzz.cc b/tcmalloc/huge_page_aware_allocator_fuzz.cc index f25c0ad63..027ba6238 100644 --- a/tcmalloc/huge_page_aware_allocator_fuzz.cc +++ b/tcmalloc/huge_page_aware_allocator_fuzz.cc @@ -299,6 +299,15 @@ struct SetEnableReleaseStalePages { } }; +struct SetMadvNoHugepageHugeRegions { + bool value; + + template + friend void AbslStringify(Sink& sink, const SetMadvNoHugepageHugeRegions& s) { + absl::Format(&sink, "SetMadvNoHugepageHugeRegions{.value=%v}", s.value); + } +}; + struct Instruction; template @@ -314,7 +323,7 @@ using ParamOp = std::variant< SetHpaaSubrelease, SetReleaseSucceeds, SetHugeRegionDemandBasedRelease, SetHugeRegionAdaptiveRelease, SetBackAllocations, SetBackSizeThresholdBytes, ReentrantSubprogram, SetEnableUnfilteredCollapse, SetReleaseMaxColdPages, - SetEnableReleaseStalePages>; + SetEnableReleaseStalePages, SetMadvNoHugepageHugeRegions>; template void AbslStringify(Sink& sink, const ParamOp& p) { @@ -615,6 +624,12 @@ void FuzzHPAA(FuzzHugePageAwareAllocatorOptions fuzz_options, forwarder.set_release_stale_pages( param_arg.value ? ReleaseStalePages::kEnabled : ReleaseStalePages::kDisabled); + } else if constexpr (std::is_same_v< + P, SetMadvNoHugepageHugeRegions>) { + forwarder.set_madvise_cold_regions_nohugepage( + param_arg.value + ? MadviseRegionsNoHugepage::kEnabled + : MadviseRegionsNoHugepage::kDisabled); } }, arg.op); diff --git a/tcmalloc/huge_page_options.h b/tcmalloc/huge_page_options.h index a45c5cd97..9d448678a 100644 --- a/tcmalloc/huge_page_options.h +++ b/tcmalloc/huge_page_options.h @@ -42,6 +42,11 @@ enum class ReleaseStalePages : bool { kEnabled = true, }; +enum class MadviseRegionsNoHugepage : bool { + kDisabled = false, + kEnabled = true, +}; + } // namespace tcmalloc::tcmalloc_internal GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/mock_huge_page_static_forwarder.h b/tcmalloc/mock_huge_page_static_forwarder.h index b17a5116a..bb8606503 100644 --- a/tcmalloc/mock_huge_page_static_forwarder.h +++ b/tcmalloc/mock_huge_page_static_forwarder.h @@ -33,6 +33,7 @@ #include "tcmalloc/arena.h" #include "tcmalloc/common.h" #include "tcmalloc/huge_page_filler.h" +#include "tcmalloc/huge_page_options.h" #include "tcmalloc/huge_pages.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/logging.h" @@ -100,6 +101,14 @@ class FakeStaticForwarder { release_stale_pages_ = value; } + MadviseRegionsNoHugepage madvise_cold_regions_nohugepage() const { + return madvise_cold_regions_nohugepage_; + } + + void set_madvise_cold_regions_nohugepage(MadviseRegionsNoHugepage value) { + madvise_cold_regions_nohugepage_ = value; + } + bool BackAllocations() const { return back_allocations_; } void SetBackAllocations(bool value) { back_allocations_ = value; } int32_t BackSizeThresholdBytes() const { return back_size_threshold_bytes_; } @@ -244,6 +253,8 @@ class FakeStaticForwarder { EnableUnfilteredCollapse::kDisabled; Arena arena_; ReleaseStalePages release_stale_pages_ = ReleaseStalePages::kDisabled; + MadviseRegionsNoHugepage madvise_cold_regions_nohugepage_ = + MadviseRegionsNoHugepage::kDisabled; std::atomic fake_allocation_ = 0x1000; diff --git a/tcmalloc/parameters.cc b/tcmalloc/parameters.cc index 49ae6840f..4ac693bf5 100644 --- a/tcmalloc/parameters.cc +++ b/tcmalloc/parameters.cc @@ -21,7 +21,6 @@ #include "absl/base/attributes.h" #include "absl/base/call_once.h" -#include "absl/base/const_init.h" #include "absl/base/internal/spinlock.h" #include "absl/time/time.h" #include "tcmalloc/central_freelist.h" @@ -240,6 +239,20 @@ ABSL_CONST_INIT std::atomic Parameters::back_size_threshold_bytes_( ABSL_CONST_INIT std::atomic Parameters::enable_unfiltered_collapse_( false); ABSL_CONST_INIT std::atomic Parameters::release_max_cold_pages_(false); +static std::atomic& +madvise_cold_regions_nohugepage_enabled() { + ABSL_CONST_INIT static absl::once_flag flag; + ABSL_CONST_INIT static std::atomic v{ + MadviseRegionsNoHugepage::kDisabled}; + absl::base_internal::LowLevelCallOnce(&flag, [&]() { + if (IsExperimentActive( + Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS)) { + v.store(MadviseRegionsNoHugepage::kEnabled, std::memory_order_relaxed); + } + }); + return v; +} + static std::atomic& huge_region_adaptive_release_enabled() { ABSL_CONST_INIT static absl::once_flag flag; ABSL_CONST_INIT static std::atomic v{false}; @@ -321,6 +334,11 @@ bool Parameters::huge_region_adaptive_release() { return huge_region_adaptive_release_enabled().load(std::memory_order_relaxed); } +MadviseRegionsNoHugepage Parameters::madvise_cold_regions_nohugepage() { + return madvise_cold_regions_nohugepage_enabled().load( + std::memory_order_relaxed); +} + HeapPartitioningMode Parameters::heap_partitioning_mode() { return heap_partitioning_mode_ptr().load(std::memory_order_relaxed); } @@ -674,6 +692,18 @@ void TCMalloc_Internal_SetReleaseMaxColdPages(bool v) { Parameters::release_max_cold_pages_.store(v, std::memory_order_relaxed); } +bool TCMalloc_Internal_GetMadviseColdRegionsNoHugepage() { + return Parameters::madvise_cold_regions_nohugepage() == + tcmalloc::tcmalloc_internal::MadviseRegionsNoHugepage::kEnabled; +} + +void TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(bool v) { + tcmalloc::tcmalloc_internal::madvise_cold_regions_nohugepage_enabled().store( + v ? tcmalloc::tcmalloc_internal::MadviseRegionsNoHugepage::kEnabled + : tcmalloc::tcmalloc_internal::MadviseRegionsNoHugepage::kDisabled, + std::memory_order_relaxed); +} + } // extern "C" GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/parameters.h b/tcmalloc/parameters.h index 32a24a1e3..6b035302a 100644 --- a/tcmalloc/parameters.h +++ b/tcmalloc/parameters.h @@ -24,6 +24,7 @@ #include "absl/time/time.h" #include "tcmalloc/central_freelist.h" #include "tcmalloc/huge_page_filler.h" +#include "tcmalloc/huge_page_options.h" #include "tcmalloc/internal/config.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/parameter_accessors.h" @@ -139,6 +140,12 @@ class Parameters { TCMalloc_Internal_SetReleaseMaxColdPages(value); } + static MadviseRegionsNoHugepage madvise_cold_regions_nohugepage(); + + static void set_madvise_cold_regions_nohugepage(bool value) { + TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(value); + } + static void set_per_cpu_caches(bool value) { #if !defined(TCMALLOC_DEPRECATED_PERTHREAD) if (!value) { diff --git a/tcmalloc/testing/get_stats_test.cc b/tcmalloc/testing/get_stats_test.cc index 0ea3bfad3..614b2548b 100644 --- a/tcmalloc/testing/get_stats_test.cc +++ b/tcmalloc/testing/get_stats_test.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -21,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -33,12 +31,9 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" -#include "absl/types/optional.h" -#include "tcmalloc/common.h" #include "tcmalloc/experiment.h" #include "tcmalloc/experiment_config.h" #include "tcmalloc/global_stats.h" -#include "tcmalloc/internal/config.h" #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/memory_stats.h" #include "tcmalloc/malloc_extension.h" @@ -158,6 +153,12 @@ TEST_F(GetStatsTest, Pbtxt) { EXPECT_THAT(buf, HasSubstr("min_hot_access_hint: 1")); } + if (IsExperimentActive(Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS)) { + EXPECT_THAT(buf, HasSubstr("madvise_cold_regions_nohugepage: true")); + } else { + EXPECT_THAT(buf, HasSubstr("madvise_cold_regions_nohugepage: false")); + } + EXPECT_THAT(buf, HasSubstr("tcmalloc_enable_unfiltered_collapse: false")); if (MallocExtension::PerCpuCachesActive()) { EXPECT_THAT(buf, ContainsRegex("cpu_caches_touched: [0-9]+")); @@ -267,6 +268,14 @@ TEST_F(GetStatsTest, Parameters) { buf, HasSubstr(R"(PARAMETER tcmalloc_huge_region_adaptive_release 0)")); } + if (IsExperimentActive( + Experiment::TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS)) { + EXPECT_THAT(buf, + HasSubstr(R"(PARAMETER madvise_cold_regions_nohugepage 1)")); + } else { + EXPECT_THAT(buf, + HasSubstr(R"(PARAMETER madvise_cold_regions_nohugepage 0)")); + } if (using_hpaa(buf)) { EXPECT_THAT(buf, HasSubstr(R"(using_hpaa_subrelease: false)")); } diff --git a/tcmalloc/variants.bzl b/tcmalloc/variants.bzl index 91b719d81..4bc977f27 100644 --- a/tcmalloc/variants.bzl +++ b/tcmalloc/variants.bzl @@ -289,6 +289,12 @@ test_variants = [ "deps": ["//tcmalloc:common_8k_pages"], "env": {"BORG_EXPERIMENTS": "TEST_ONLY_TCMALLOC_RELEASE_STALE_PAGES"}, }, + { + "name": "tcmalloc_madv_nohugepage_regions", + "malloc": "//tcmalloc", + "deps": ["//tcmalloc:common_8k_pages"], + "env": {"BORG_EXPERIMENTS": "TCMALLOC_SONIC_MADV_NOHUGEPAGE_REGIONS"}, + }, ] def create_tcmalloc_library(