Merge branch 'master' into blobsidecar-archive-writer-noop

CHANGELOG.md

@@ -15,6 +15,7 @@
 - Added support for [Ephemery Testnet](https://github.com/ephemery.dev) `--network=ephemery`
 - Renamed metrics `validator_attestation_publication_delay`,`validator_block_publication_delay` and `beacon_block_import_delay_counter` to include the suffix `_total` added by the current version of prometheus.
 - Updated bootnodes for Holesky network
+- Added new `--p2p-flood-publish-enabled` parameter to control whenever flood publishing behaviour is enabled (applies to all subnets). Previous teku versions always had this behaviour enabled. Default is `true`.
 
 ### Bug Fixes
  - removed a warning from logs about non blinded blocks being requested (#8562)

ethereum/spec/src/main/java/tech/pegasys/teku/spec/SpecMilestone.java

@@ -46,6 +46,14 @@ public boolean isGreaterThanOrEqualTo(final SpecMilestone other) {
     return compareTo(other) >= 0;
   }
 
+  public boolean isGreaterThan(final SpecMilestone other) {
+    return compareTo(other) > 0;
+  }
+
+  public boolean isLessThanOrEqualTo(final SpecMilestone other) {
+    return compareTo(other) <= 0;
+  }
+
   /** Returns the milestone prior to this milestone */
   public SpecMilestone getPreviousMilestone() {
     if (equals(PHASE0)) {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/AbstractSchemaProvider.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.spec.schemas.registry;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.config.SpecConfig;
+import tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SchemaId;
+
+abstract class AbstractSchemaProvider<T> implements SchemaProvider<T> {
+  private final NavigableMap<SpecMilestone, SpecMilestone> milestoneToEffectiveMilestone =
+      new TreeMap<>();
+  private final SchemaId<T> schemaId;
+
+  protected AbstractSchemaProvider(final SchemaId<T> schemaId) {
+    this.schemaId = schemaId;
+  }
+
+  protected void addMilestoneMapping(
+      final SpecMilestone milestone, final SpecMilestone untilMilestone) {
+    checkArgument(
+        untilMilestone.isGreaterThan(milestone),
+        "%s must be earlier than %s",
+        milestone,
+        untilMilestone);
+
+    checkOverlappingVersionMappings(milestone, untilMilestone);
+
+    SpecMilestone currentMilestone = untilMilestone;
+    while (currentMilestone.isGreaterThan(milestone)) {
+      milestoneToEffectiveMilestone.put(currentMilestone, milestone);
+      currentMilestone = currentMilestone.getPreviousMilestone();
+    }
+  }
+
+  private void checkOverlappingVersionMappings(
+      final SpecMilestone milestone, final SpecMilestone untilMilestone) {
+    final Map.Entry<SpecMilestone, SpecMilestone> floorEntry =
+        milestoneToEffectiveMilestone.floorEntry(untilMilestone);
+    if (floorEntry != null && floorEntry.getValue().isGreaterThanOrEqualTo(milestone)) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Milestone %s is already mapped to %s",
+              floorEntry.getKey(), getEffectiveMilestone(floorEntry.getValue())));
+    }
+    final Map.Entry<SpecMilestone, SpecMilestone> ceilingEntry =
+        milestoneToEffectiveMilestone.ceilingEntry(milestone);
+    if (ceilingEntry != null && ceilingEntry.getKey().isLessThanOrEqualTo(untilMilestone)) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Milestone %s is already mapped to %s",
+              ceilingEntry.getKey(), getEffectiveMilestone(ceilingEntry.getValue())));
+    }
+  }
+
+  @Override
+  public SpecMilestone getEffectiveMilestone(final SpecMilestone milestone) {
+    return milestoneToEffectiveMilestone.getOrDefault(milestone, milestone);
+  }
+
+  @Override
+  public T getSchema(final SchemaRegistry registry) {
+    final SpecMilestone milestone = registry.getMilestone();
+    final SpecMilestone effectiveMilestone = getEffectiveMilestone(milestone);
+    return createSchema(registry, effectiveMilestone, registry.getSpecConfig());
+  }
+
+  @Override
+  public SchemaId<T> getSchemaId() {
+    return schemaId;
+  }
+
+  protected abstract T createSchema(
+      SchemaRegistry registry, SpecMilestone effectiveMilestone, SpecConfig specConfig);
+}

ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaCache.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.spec.schemas.registry;
+
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.Map;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SchemaId;
+
+interface SchemaCache {
+  static SchemaCache createDefault() {
+    return new SchemaCache() {
+      private final Map<SpecMilestone, Map<SchemaId<?>, Object>> cache =
+          new EnumMap<>(SpecMilestone.class);
+
+      @SuppressWarnings("unchecked")
+      @Override
+      public <T> T get(final SpecMilestone milestone, final SchemaId<T> schemaId) {
+        final Map<?, ?> milestoneSchemaIds = cache.get(milestone);
+        if (milestoneSchemaIds == null) {
+          return null;
+        }
+        return (T) milestoneSchemaIds.get(schemaId);
+      }
+
+      @Override
+      public <T> void put(
+          final SpecMilestone milestone, final SchemaId<T> schemaId, final T schema) {
+        cache.computeIfAbsent(milestone, __ -> new HashMap<>()).put(schemaId, schema);
+      }
+    };
+  }
+
+  <T> T get(SpecMilestone milestone, SchemaId<T> schemaId);
+
+  <T> void put(SpecMilestone milestone, SchemaId<T> schemaId, T schema);
+}

ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaProvider.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.spec.schemas.registry;
+
+import static tech.pegasys.teku.spec.SpecMilestone.BELLATRIX;
+import static tech.pegasys.teku.spec.SpecMilestone.CAPELLA;
+import static tech.pegasys.teku.spec.SpecMilestone.DENEB;
+import static tech.pegasys.teku.spec.SpecMilestone.ELECTRA;
+
+import java.util.EnumSet;
+import java.util.Set;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SchemaId;
+
+interface SchemaProvider<T> {
+  Set<SpecMilestone> ALL_MILESTONES = EnumSet.allOf(SpecMilestone.class);
+  Set<SpecMilestone> FROM_BELLATRIX = from(BELLATRIX);
+  Set<SpecMilestone> FROM_CAPELLA = from(CAPELLA);
+  Set<SpecMilestone> FROM_DENEB = from(DENEB);
+  Set<SpecMilestone> FROM_ELECTRA = from(ELECTRA);
+
+  static Set<SpecMilestone> from(final SpecMilestone milestone) {
+    return EnumSet.copyOf(SpecMilestone.getAllMilestonesFrom(milestone));
+  }
+
+  static Set<SpecMilestone> fromTo(
+      final SpecMilestone fromMilestone, final SpecMilestone toMilestone) {
+    return EnumSet.copyOf(
+        SpecMilestone.getAllMilestonesFrom(fromMilestone).stream()
+            .filter(toMilestone::isLessThanOrEqualTo)
+            .toList());
+  }
+
+  T getSchema(SchemaRegistry registry);
+
+  Set<SpecMilestone> getSupportedMilestones();
+
+  SpecMilestone getEffectiveMilestone(SpecMilestone version);
+
+  SchemaId<T> getSchemaId();
+}

ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaRegistry.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.spec.schemas.registry;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.config.SpecConfig;
+import tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SchemaId;
+
+public class SchemaRegistry {
+  // this is used for dependency loop detection during priming
+  private static final Set<SchemaProvider<?>> INFLIGHT_PROVIDERS = new HashSet<>();
+
+  private final Map<SchemaId<?>, SchemaProvider<?>> providers = new HashMap<>();
+  private final SpecMilestone milestone;
+  private final SchemaCache cache;
+  private final SpecConfig specConfig;
+  private boolean primed;
+
+  SchemaRegistry(
+      final SpecMilestone milestone, final SpecConfig specConfig, final SchemaCache cache) {
+    this.milestone = milestone;
+    this.specConfig = specConfig;
+    this.cache = cache;
+    this.primed = false;
+  }
+
+  /**
+   * This is supposed to be called only by {@link SchemaRegistryBuilder#build(SpecMilestone,
+   * SpecConfig)} which is synchronized
+   */
+  void registerProvider(final SchemaProvider<?> provider) {
+    if (primed) {
+      throw new IllegalStateException("Cannot add a provider to a primed registry");
+    }
+    if (providers.put(provider.getSchemaId(), provider) != null) {
+      throw new IllegalStateException(
+          "Cannot add provider "
+              + provider.getClass().getSimpleName()
+              + " referencing "
+              + provider.getSchemaId()
+              + " which has been already added via another provider");
+    }
+  }
+
+  @VisibleForTesting
+  boolean isProviderRegistered(final SchemaProvider<?> provider) {
+    return provider.equals(providers.get(provider.getSchemaId()));
+  }
+
+  @SuppressWarnings("unchecked")
+  public <T> T get(final SchemaId<T> schemaId) {
+    SchemaProvider<T> provider = (SchemaProvider<T>) providers.get(schemaId);
+    if (provider == null) {
+      throw new IllegalArgumentException(
+          "No provider registered for schema "
+              + schemaId
+              + " or it does not support milestone "
+              + milestone);
+    }
+    T schema = cache.get(milestone, schemaId);
+    if (schema != null) {
+      return schema;
+    }
+
+    // let's check if the schema is stored associated to the effective milestone
+    final SpecMilestone effectiveMilestone = provider.getEffectiveMilestone(milestone);
+    if (effectiveMilestone != milestone) {
+      schema = cache.get(effectiveMilestone, schemaId);
+      if (schema != null) {
+        // let's cache the schema for current milestone as well
+        cache.put(milestone, schemaId, schema);
+        return schema;
+      }
+    }
+
+    // The schema was not found.
+    // we reach this point only during priming when we actually ask providers to generate schemas
+    checkState(!primed, "Registry is primed but schema not found for %s", schemaId);
+
+    // save the provider as "inflight"
+    if (!INFLIGHT_PROVIDERS.add(provider)) {
+      throw new IllegalStateException("loop detected creating schema for " + schemaId);
+    }
+
+    // actual schema creation (may trigger recursive registry lookups)
+    schema = provider.getSchema(this);
+
+    // release the provider
+    INFLIGHT_PROVIDERS.remove(provider);
+
+    // cache the schema
+    cache.put(effectiveMilestone, schemaId, schema);
+    if (effectiveMilestone != milestone) {
+      cache.put(milestone, schemaId, schema);
+    }
+    return schema;
+  }
+
+  public SpecMilestone getMilestone() {
+    return milestone;
+  }
+
+  public SpecConfig getSpecConfig() {
+    return specConfig;
+  }
+
+  /**
+   * This is supposed to be called only by {@link SchemaRegistryBuilder#build(SpecMilestone,
+   * SpecConfig)} which is synchronized
+   */
+  void primeRegistry() {
+    if (primed) {
+      throw new IllegalStateException("Registry already primed");
+    }
+    for (final SchemaId<?> schemaClass : providers.keySet()) {
+      get(schemaClass);
+    }
+    primed = true;
+  }
+}