stats: make it's possible to enable the new stats API

changelogs/current/minor_behavior_changes/stats__explicit-tags.rst

@@ -0,0 +1,8 @@
+Added the runtime guard ``envoy.reloadable_features.enable_stats_explicit_tags`` (default
+``false``). When set to ``true`` and the stats configuration carries no custom tags (empty
+:ref:`stats_tags <envoy_v3_api_field_config.metrics.v3.StatsConfig.stats_tags>` and
+:ref:`use_all_default_tags <envoy_v3_api_field_config.metrics.v3.StatsConfig.use_all_default_tags>`
+left at its default of ``true``), the stats store uses the tags supplied by the calling code (the
+explicit-tags logic) and propagates scope-level tags onto every stat, instead of re-parsing the
+flat stat name. The guard is evaluated once at startup. There is no visible change to users while
+the guard remains ``false``.

envoy/stats/scope.h

@@ -107,7 +107,7 @@ class Scope : public std::enable_shared_from_this<Scope> {
   /**
    * Allocate a new scope, optionally supplying tags and a pre-built tagged name (with tag values
    * interleaved). NOTE: The behavior is implementation-defined for now because both legacy and
-   * tag-aware scopes are still supported.
+   * explicit-tags scopes are still supported.
    *
    * @param base_name supplies the scope's tag-extracted name.
    * @param name_tags tags to associate with (and propagate from) the scope, as string_view pairs.

envoy/stats/store.h

@@ -251,6 +251,14 @@ class StoreRoot : public Store {
    */
   virtual void setHistogramSettings(HistogramSettingsConstPtr&& histogram_settings) PURE;
 
+  /**
+   * Selects whether scopes created by this store use the explicit-tags logic, which propagates
+   * scope-level tags onto every stat. Must be called during single-threaded startup, before
+   * scopes other than the root scope are created and before initializeThreading(); flipping it
+   * afterwards is unsupported.
+   */
+  virtual void setUseExplicitTags(bool use_explicit_tags) PURE;
+
   /**
    * Initialize the store for threading. This will be called once after all worker threads have
    * been initialized. At this point the store can initialize itself for multi-threaded operation.

source/common/runtime/runtime_features.cc

@@ -146,6 +146,10 @@ RUNTIME_GUARD(envoy_restart_features_worker_threads_watchdog_fix);
 
 // Sentinel and test flag.
 FALSE_RUNTIME_GUARD(envoy_reloadable_features_test_feature_false);
+// When true (and the stats config carries no custom tags), the stats store uses the explicit-tags
+// scope logic that propagates scope-level tags onto every stat. Evaluated once at startup.
+// TODO: flip to true after sufficient testing.
+FALSE_RUNTIME_GUARD(envoy_reloadable_features_enable_stats_explicit_tags);
 // TODO: Flip to true after sufficient testing to enable formatter support for rate limit action
 // descriptor_value fields by default.
 FALSE_RUNTIME_GUARD(

source/common/stats/isolated_store_impl.cc

@@ -66,20 +66,23 @@ ScopeSharedPtr IsolatedScopeImpl::createScopeWithTaggedName(
     bool evictable, const ScopeStatsLimitSettings& limits, StatsMatcherSharedPtr matcher) {
   // Intern the string-based tag-extracted name, name_tags and (optional) tagged name into a
   // temporary pool, then delegate to the StatName-based variant.
+  std::string sanitize_buffer;
   StatNamePool tag_pool(symbolTable());
-  StatName stat_name = tag_pool.add(Utility::sanitizeStatsName(base_name));
+  StatName stat_name = tag_pool.add(Utility::sanitizeStatsName(base_name, sanitize_buffer));
   StatName stat_tagged_name;
   if (!name_tags.empty()) {
     // The tagged name is only meaningful when there are tags to interleave; otherwise it is
     // ignored and the TagStatNameJoiner will use the tag-extracted name as the flat tagged name.
-    stat_tagged_name =
-        tagged_name.empty() ? StatName() : tag_pool.add(Utility::sanitizeStatsName(tagged_name));
+    stat_tagged_name = tagged_name.empty()
+                           ? StatName()
+                           : tag_pool.add(Utility::sanitizeStatsName(tagged_name, sanitize_buffer));
   }
 
   StatNameTagVec stat_name_tags;
   stat_name_tags.reserve(name_tags.size());
   for (const auto& [tag, value] : name_tags) {
-    stat_name_tags.emplace_back(tag_pool.add(tag), tag_pool.add(value));
+    stat_name_tags.emplace_back(tag_pool.add(Utility::sanitizeStatsName(tag, sanitize_buffer)),
+                                tag_pool.add(Utility::sanitizeStatsName(value, sanitize_buffer)));
   }
   return scopeFromTaggedName(stat_name, stat_name_tags, stat_tagged_name, evictable, limits,
                              std::move(matcher));

source/common/stats/symbol_table.cc

@@ -718,6 +718,40 @@ void SymbolTable::populateList(const StatName* names, uint32_t num_names, StatNa
   list.moveStorageIntoList(mem_block.release());
 }
 
+void SymbolTable::populateList(StatName tagged_name, StatName base_name, StatNameTagSpan name_tags,
+                               StatNameList& list) {
+  const size_t stat_name_count = name_tags.size() * 2 + 2;
+
+  RELEASE_ASSERT(stat_name_count < 256, "Maximum number elements in a StatNameList exceeded");
+
+  // One byte for the number of names, plus the size of each name.
+  size_t total_size_bytes = 1;
+  total_size_bytes += tagged_name.size() + base_name.size();
+  for (const auto& tag : name_tags) {
+    total_size_bytes += tag.first.size() + tag.second.size();
+  }
+
+  // Now allocate the exact number of bytes required and move the encodings into storage.
+  MemBlockBuilder<uint8_t> mem_block(total_size_bytes);
+  mem_block.appendOne(stat_name_count);
+  Encoding::appendToMemBlock(tagged_name, mem_block);
+  Encoding::appendToMemBlock(base_name, mem_block);
+  incRefCount(tagged_name);
+  incRefCount(base_name);
+  for (const auto& tag : name_tags) {
+    Encoding::appendToMemBlock(tag.first, mem_block);
+    Encoding::appendToMemBlock(tag.second, mem_block);
+    incRefCount(tag.first);
+    incRefCount(tag.second);
+  }
+
+  // This assertion double-checks the arithmetic where we computed total_size_bytes. After appending
+  // all the encoded data into the allocated byte array, we should have exhausted all the memory we
+  // thought we needed.
+  ASSERT(mem_block.capacityRemaining() == 0);
+  list.moveStorageIntoList(mem_block.release());
+}
+
 StatNameList::~StatNameList() { ASSERT(!populated()); }
 
 void StatNameList::iterate(const std::function<bool(StatName)>& f) const {

source/common/stats/symbol_table.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <algorithm>
+#include <cstddef>
 #include <memory>
 #include <stack>
 #include <string>
@@ -28,6 +29,8 @@ using StatNameVec = absl::InlinedVector<StatName, 8>;
 class StatNameList;
 class StatNameSet;
 using StatNameSetPtr = std::unique_ptr<StatNameSet>;
+using StatNameTag = std::pair<StatName, StatName>;
+using StatNameTagSpan = absl::Span<const StatNameTag>;
 
 /**
  * Holds a range of indexes indicating which parts of a stat-name are
@@ -314,6 +317,19 @@ class SymbolTable final {
    */
   void populateList(const StatName* names, uint32_t num_names, StatNameList& list);
 
+  /**
+   * Populates a StatNameList from a name, base-name, and tags. This is not done at
+   * construction time to enable StatNameList to be instantiated directly in
+   * a class that doesn't have a live SymbolTable when it is constructed.
+   *
+   * @param tagged_name The tagged name of the stat.
+   * @param base_name The base name of the stat.
+   * @param name_tags The tags associated with the stat.
+   * @param list The StatNameList representing the stat names.
+   */
+  void populateList(StatName tagged_name, StatName base_name, StatNameTagSpan name_tags,
+                    StatNameList& list);
+
 #ifndef ENVOY_CONFIG_COVERAGE
   void debugPrint() const;
 #endif
@@ -906,6 +922,24 @@ class StatNameList {
    * @param f The function to call on each stat.
    */
   void iterate(const std::function<bool(StatName)>& f) const;
+  /**
+   * Iterates over each StatName in the list, calling f(StatName, index). f()
+   * should return true to keep iterating, or false to end the iteration.
+   *
+   * @param f The function to call on each stat.
+   */
+  template <typename ConsumeStatNameCb> void iterateWithIndex(const ConsumeStatNameCb& f) const {
+    ASSERT(populated());
+    const uint8_t* p = storage_.get();
+    const uint32_t num_elements = *p++;
+    for (uint32_t i = 0; i < num_elements; ++i) {
+      const StatName stat_name(p);
+      p += stat_name.size();
+      if (!f(stat_name, i)) {
+        break;
+      }
+    }
+  }
 
   /**
    * Frees each StatName in the list. Failure to call this before destruction

source/common/stats/tag_utility.h

@@ -27,7 +27,7 @@ class TagStatNameJoiner {
   /**
    * Combines a scope's tag-extracted prefix, prefix tags, and tagged (flat) prefix, together with
    * a stat's tag-extracted name, name tags and optional tagged (flat) name, into a single joined
-   * name. This is used by the tag-aware scope implementations to avoid needing to re-parse the
+   * name. This is used by the explicit-tags scope logic to avoid needing to re-parse the
    * final flat stat name in order to extract tags from it.
    *
    * @param prefix StatName the tag-extracted prefix for this scope (no tag values).
@@ -61,7 +61,7 @@ class TagStatNameJoiner {
    *
    * Tags come from one of two mutually-exclusive sources depending on the constructor used:
    * the legacy constructor stores the caller-owned span directly (`stat_name_tags_`); the
-   * tag-aware constructor derives and owns them in `effective_tags_`.
+   * explicit-tags constructor derives and owns them in `effective_tags_`.
    */
   std::optional<StatNameTagSpan> effectiveTags() const {
     if (stat_name_tags_.has_value()) {
@@ -86,7 +86,7 @@ class TagStatNameJoiner {
   // lifetime. effectiveTags() returns this directly when set.
   std::optional<StatNameTagSpan> stat_name_tags_;
 
-  // Set only by the tag-aware constructor: the inherited (prefix) tags followed by this
+  // Set only by the explicit-tags constructor: the inherited (prefix) tags followed by this
   // element's own (name) tags, owned (copied) by this joiner. effectiveTags() returns a span over
   // this when it is non-empty.
   StatNameTagVec effective_tags_;