diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 87ab6fae0a3..95a38c4b479 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -14,6 +14,10 @@ public static class Counter { public static final String TXS = "tron:txs"; public static final String MINER = "tron:miner"; public static final String BLOCK_FORK = "tron:block_fork"; + // witness label: bounded cardinality -- SR candidate pool is finite, rotation is + // infrequent (at most once per maintenance interval); kept for at-a-glance SR + // identification in dashboards rather than requiring log cross-referencing. + public static final String SR_SET_CHANGE = "tron:sr_set_change"; public static final String P2P_ERROR = "tron:p2p_error"; public static final String P2P_DISCONNECT = "tron:p2p_disconnect"; public static final String INTERNAL_SERVICE_FAIL = "tron:internal_service_fail"; @@ -62,6 +66,7 @@ public static class Histogram { public static final String MESSAGE_PROCESS_LATENCY = "tron:message_process_latency_seconds"; public static final String BLOCK_FETCH_LATENCY = "tron:block_fetch_latency_seconds"; public static final String BLOCK_RECEIVE_DELAY = "tron:block_receive_delay_seconds"; + public static final String BLOCK_TRANSACTION_COUNT = "tron:block_transaction_count"; private Histogram() { throw new IllegalStateException("Histogram"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java index 2aa3c1e3378..1f0da214085 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricLabels.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricLabels.java @@ -31,6 +31,8 @@ public static class Counter { public static final String TXS_FAIL_SIG = "sig"; public static final String TXS_FAIL_TAPOS = "tapos"; public static final String TXS_FAIL_DUP = "dup"; + public static final String SR_ADD = "add"; + public static final String SR_REMOVE = "remove"; private Counter() { throw new IllegalStateException("Counter"); @@ -66,6 +68,7 @@ private Gauge() { // Histogram public static class Histogram { + public static final String MINER = "miner"; public static final String TRAFFIC_IN = "in"; public static final String TRAFFIC_OUT = "out"; diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsCounter.java b/common/src/main/java/org/tron/common/prometheus/MetricsCounter.java index 6acdf23b3bc..7231baaba8f 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsCounter.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsCounter.java @@ -14,6 +14,7 @@ class MetricsCounter { init(MetricKeys.Counter.TXS, "tron txs info .", "type", "detail"); init(MetricKeys.Counter.MINER, "tron miner info .", "miner", "type"); init(MetricKeys.Counter.BLOCK_FORK, "tron block fork info .", "type"); + init(MetricKeys.Counter.SR_SET_CHANGE, "tron sr set change .", "action", "witness"); init(MetricKeys.Counter.P2P_ERROR, "tron p2p error info .", "type"); init(MetricKeys.Counter.P2P_DISCONNECT, "tron p2p disconnect .", "type"); init(MetricKeys.Counter.INTERNAL_SERVICE_FAIL, "internal Service fail.", diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java index 556db10feb5..6a66dc76bb3 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java @@ -20,7 +20,7 @@ public class MetricsHistogram { init(MetricKeys.Histogram.JSONRPC_SERVICE_LATENCY, "JsonRpc Service latency.", "method"); init(MetricKeys.Histogram.MINER_LATENCY, "miner latency.", - "miner"); + MetricLabels.Histogram.MINER); init(MetricKeys.Histogram.PING_PONG_LATENCY, "node ping pong latency."); init(MetricKeys.Histogram.VERIFY_SIGN_LATENCY, "verify sign latency for trx , block.", "type"); @@ -36,7 +36,7 @@ public class MetricsHistogram { init(MetricKeys.Histogram.PROCESS_TRANSACTION_LATENCY, "process transaction latency.", "type", "contract"); init(MetricKeys.Histogram.MINER_DELAY, "miner delay time, actualTime - planTime.", - "miner"); + MetricLabels.Histogram.MINER); init(MetricKeys.Histogram.UDP_BYTES, "udp_bytes traffic.", "type"); init(MetricKeys.Histogram.TCP_BYTES, "tcp_bytes traffic.", @@ -48,6 +48,11 @@ public class MetricsHistogram { init(MetricKeys.Histogram.BLOCK_FETCH_LATENCY, "fetch block latency."); init(MetricKeys.Histogram.BLOCK_RECEIVE_DELAY, "receive block delay time, receiveTime - blockTime."); + + init(MetricKeys.Histogram.BLOCK_TRANSACTION_COUNT, + "Distribution of transaction counts per block.", + new double[]{0, 10, 50, 100, 200, 500, 1000, 2000, 5000, 10000}, + MetricLabels.Histogram.MINER); } private MetricsHistogram() { @@ -62,6 +67,17 @@ private static void init(String name, String help, String... labels) { .register()); } + private static void init(String name, String help, double[] buckets, String... labels) { + Histogram.Builder builder = Histogram.build() + .name(name) + .help(help) + .labelNames(labels); + if (buckets != null && buckets.length > 0) { + builder.buckets(buckets); + } + container.put(name, builder.register()); + } + static Histogram.Timer startTimer(String key, String... labels) { if (Metrics.enabled()) { Histogram histogram = container.get(key); diff --git a/common/src/main/java/org/tron/common/prometheus/SRMetrics.java b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java new file mode 100644 index 00000000000..0c547a38e2c --- /dev/null +++ b/common/src/main/java/org/tron/common/prometheus/SRMetrics.java @@ -0,0 +1,26 @@ +package org.tron.common.prometheus; + +import com.google.protobuf.ByteString; +import java.util.List; +import org.tron.common.utils.StringUtil; + +public class SRMetrics { + + private SRMetrics() { + throw new IllegalStateException("SRMetrics"); + } + + public static void recordSrSetChange(List currentWits, List newWits) { + if (!Metrics.enabled()) { + return; + } + newWits.stream() + .filter(w -> !currentWits.contains(w)) + .forEach(w -> Metrics.counterInc(MetricKeys.Counter.SR_SET_CHANGE, 1, + MetricLabels.Counter.SR_ADD, StringUtil.encode58Check(w.toByteArray()))); + currentWits.stream() + .filter(w -> !newWits.contains(w)) + .forEach(w -> Metrics.counterInc(MetricKeys.Counter.SR_SET_CHANGE, 1, + MetricLabels.Counter.SR_REMOVE, StringUtil.encode58Check(w.toByteArray()))); + } +} diff --git a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java index 012169bdb87..fd5e4364d0d 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/MaintenanceManager.java @@ -16,6 +16,7 @@ import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.prometheus.SRMetrics; import org.tron.consensus.ConsensusDelegate; import org.tron.consensus.pbft.PbftManager; import org.tron.core.capsule.AccountCapsule; @@ -141,6 +142,8 @@ public void doMaintenance() { witnessCapsule.setIsJobs(true); consensusDelegate.saveWitness(witnessCapsule); }); + + SRMetrics.recordSrSetChange(currentWits, newWits); } logger.info("Update witness success. \nbefore: {} \nafter: {}", diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index cd1a61c01fe..6ccd024091d 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -1270,6 +1270,11 @@ public void pushBlock(final BlockCapsule block) synchronized (this) { Metrics.histogramObserve(blockedTimer.get()); blockedTimer.remove(); + if (Metrics.enabled()) { + Metrics.histogramObserve(MetricKeys.Histogram.BLOCK_TRANSACTION_COUNT, + block.getTransactions().size(), + StringUtil.encode58Check(block.getWitnessAddress().toByteArray())); + } long headerNumber = getDynamicPropertiesStore().getLatestBlockHeaderNumber(); if (block.getNum() <= headerNumber && khaosDb.containBlockInMiniStore(block.getBlockId())) { logger.info("Block {} is already exist.", block.getBlockId().getString()); diff --git a/framework/src/main/java/org/tron/core/metrics/blockchain/BlockChainMetricManager.java b/framework/src/main/java/org/tron/core/metrics/blockchain/BlockChainMetricManager.java index 384f1d8add1..f39cf66a8ad 100644 --- a/framework/src/main/java/org/tron/core/metrics/blockchain/BlockChainMetricManager.java +++ b/framework/src/main/java/org/tron/core/metrics/blockchain/BlockChainMetricManager.java @@ -164,9 +164,10 @@ public void applyBlock(BlockCapsule block) { } //TPS - if (block.getTransactions().size() > 0) { - MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_TPS, block.getTransactions().size()); - Metrics.counterInc(MetricKeys.Counter.TXS, block.getTransactions().size(), + int txCount = block.getTransactions().size(); + if (txCount > 0) { + MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_TPS, txCount); + Metrics.counterInc(MetricKeys.Counter.TXS, txCount, MetricLabels.Counter.TXS_SUCCESS, MetricLabels.Counter.TXS_SUCCESS); } } diff --git a/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java new file mode 100644 index 00000000000..4c2e9292d29 --- /dev/null +++ b/framework/src/test/java/org/tron/common/prometheus/SRMetricsTest.java @@ -0,0 +1,206 @@ +package org.tron.common.prometheus; + +import com.google.protobuf.ByteString; +import io.prometheus.client.CollectorRegistry; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.BaseTest; +import org.tron.common.TestConstants; +import org.tron.common.utils.StringUtil; +import org.tron.consensus.dpos.MaintenanceManager; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.VotesCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.consensus.ConsensusService; +import org.tron.protos.Protocol; +import org.tron.protos.Protocol.Vote; + +@Slf4j(topic = "metric") +public class SRMetricsTest extends BaseTest { + + private static final AtomicInteger PORT = new AtomicInteger(0); + private static final AtomicInteger UNIQUE = new AtomicInteger(0); + + @Resource + private MaintenanceManager maintenanceManager; + @Resource + private ConsensusService consensusService; + + static { + Args.setParam(new String[]{"-d", dbPath()}, TestConstants.TEST_CONF); + Args.getInstance().setNodeListenPort(20000 + PORT.incrementAndGet()); + Args.getInstance().setMetricsPrometheusEnable(true); + Metrics.init(); + } + + @Before + public void setUp() { + Args.getInstance().setMetricsPrometheusEnable(true); + consensusService.start(); + } + + @After + public void tearDown() { + Args.getInstance().setMetricsPrometheusEnable(true); + } + + /** + * Drive the full maintenance flow: starting with a single active witness while WitnessStore + * contains additional ones, doMaintenance() should expand active witnesses to the full set and + * emit SR_ADD for each newly active witness. + */ + @Test + public void testSrAddViaMaintenance() { + ByteString stableWit = registerWitness(); + ByteString newWit1 = registerWitness(); + ByteString newWit2 = registerWitness(); + + chainBaseManager.getWitnessScheduleStore() + .saveActiveWitnesses(Collections.singletonList(stableWit)); + + seedVote(stableWit); + + maintenanceManager.doMaintenance(); + + Assert.assertEquals(1, sample(MetricLabels.Counter.SR_ADD, newWit1).intValue()); + Assert.assertEquals(1, sample(MetricLabels.Counter.SR_ADD, newWit2).intValue()); + Assert.assertNull(sample(MetricLabels.Counter.SR_ADD, stableWit)); + Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, stableWit)); + } + + /** + * Active witness set already matches WitnessStore → no metric emitted. + */ + @Test + public void testNoMetricWhenSetUnchanged() { + ByteString witA = registerWitness(); + ByteString witB = registerWitness(); + + chainBaseManager.getWitnessScheduleStore() + .saveActiveWitnesses(Arrays.asList(witA, witB)); + + seedVote(witA); + + maintenanceManager.doMaintenance(); + + Assert.assertNull(sample(MetricLabels.Counter.SR_ADD, witA)); + Assert.assertNull(sample(MetricLabels.Counter.SR_ADD, witB)); + Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, witA)); + Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, witB)); + } + + /** + * Empty VotesStore → countVote() is empty → SR change check is skipped, even when the active + * set differs from the full witness store. + */ + @Test + public void testNoMetricWhenNoVotes() { + ByteString stableWit = registerWitness(); + ByteString newWit = registerWitness(); + + chainBaseManager.getWitnessScheduleStore() + .saveActiveWitnesses(Collections.singletonList(stableWit)); + + maintenanceManager.doMaintenance(); + + Assert.assertNull(sample(MetricLabels.Counter.SR_ADD, newWit)); + } + + /** + * Metrics disabled → record() short-circuits even though the active set changes. + */ + @Test + public void testNoMetricWhenMetricsDisabled() { + Args.getInstance().setMetricsPrometheusEnable(false); + try { + ByteString stableWit = registerWitness(); + ByteString newWit = registerWitness(); + + chainBaseManager.getWitnessScheduleStore() + .saveActiveWitnesses(Collections.singletonList(stableWit)); + + seedVote(stableWit); + + maintenanceManager.doMaintenance(); + + Assert.assertNull(sample(MetricLabels.Counter.SR_ADD, newWit)); + } finally { + Args.getInstance().setMetricsPrometheusEnable(true); + } + } + + /** + * SR_REMOVE is verified by directly calling record() instead of going through doMaintenance(), + * because driving a removal through the real flow is impractical here: + * + *

Inside doMaintenance(), the block before SRMetrics.recordSrSetChange() iterates currentWits + * and calls setIsJobs(false) on each WitnessCapsule fetched from WitnessStore. If currentWits + * contains any address that is not present in WitnessStore, getWitness() returns null and the + * code NPEs — so SR_REMOVE cannot be triggered by simply pointing the active set at an + * "obsolete" address. + * + *

The only other path to SR_REMOVE is rank-based eviction: with more than + * MAX_ACTIVE_WITNESS_NUM (27) witnesses, sorting drops the lowest-ranked one. Building that + * setup just to exercise this branch is heavy and adds little value, since SR_ADD and + * SR_REMOVE share the exact same emit logic in record() — verifying SR_ADD via doMaintenance + * already proves the wiring is correct, and this direct call covers the symmetric branch. + */ + @Test + public void testSrRemoveDirect() { + ByteString stableWit = uniqueAddress(); + ByteString removedWit = uniqueAddress(); + + SRMetrics.recordSrSetChange( + Arrays.asList(stableWit, removedWit), + Collections.singletonList(stableWit)); + + Assert.assertEquals(1, sample(MetricLabels.Counter.SR_REMOVE, removedWit).intValue()); + Assert.assertNull(sample(MetricLabels.Counter.SR_ADD, removedWit)); + Assert.assertNull(sample(MetricLabels.Counter.SR_REMOVE, stableWit)); + } + + private ByteString registerWitness() { + ByteString address = uniqueAddress(); + chainBaseManager.getWitnessStore().put(address.toByteArray(), new WitnessCapsule(address)); + chainBaseManager.addWitness(address); + chainBaseManager.getAccountStore().put(address.toByteArray(), + new AccountCapsule(Protocol.Account.newBuilder().setAddress(address).build())); + return address; + } + + private void seedVote(ByteString voteFor) { + ByteString voter = uniqueAddress(); + VotesCapsule votes = new VotesCapsule(voter, Collections.emptyList(), + Collections.singletonList(Vote.newBuilder() + .setVoteAddress(voteFor) + .setVoteCount(1L) + .build())); + chainBaseManager.getVotesStore().put(voter.toByteArray(), votes); + } + + private ByteString uniqueAddress() { + int n = UNIQUE.incrementAndGet(); + byte[] bytes = new byte[21]; + bytes[0] = 0x41; + bytes[17] = (byte) ((n >> 16) & 0xFF); + bytes[18] = (byte) ((n >> 8) & 0xFF); + bytes[19] = (byte) (n & 0xFF); + bytes[20] = 0x01; + return ByteString.copyFrom(bytes); + } + + private Double sample(String action, ByteString witness) { + return CollectorRegistry.defaultRegistry.getSampleValue( + MetricKeys.Counter.SR_SET_CHANGE + "_total", + new String[]{"action", "witness"}, + new String[]{action, StringUtil.encode58Check(witness.toByteArray())}); + } +} diff --git a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java index d4d758b7a98..dd260a1b869 100644 --- a/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java +++ b/framework/src/test/java/org/tron/core/metrics/prometheus/PrometheusApiServiceTest.java @@ -7,6 +7,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -25,6 +26,7 @@ import org.tron.common.utils.ByteArray; import org.tron.common.utils.PublicMethod; import org.tron.common.utils.Sha256Hash; +import org.tron.common.utils.StringUtil; import org.tron.common.utils.Utils; import org.tron.consensus.dpos.DposSlot; import org.tron.core.ChainBaseManager; @@ -38,6 +40,8 @@ @Slf4j(topic = "metric") public class PrometheusApiServiceTest extends BaseTest { + + static LocalDateTime localDateTime = LocalDateTime.now(); @Resource private DposSlot dposSlot; @@ -65,7 +69,7 @@ protected static void initParameter(CommonParameter parameter) { parameter.setMetricsPrometheusEnable(true); } - protected void check() throws Exception { + protected void check(byte[] address, Map witnessAndAccount) throws Exception { Double memoryBytes = CollectorRegistry.defaultRegistry.getSampleValue( "system_total_physical_memory_bytes"); Assert.assertNotNull(memoryBytes); @@ -80,6 +84,32 @@ protected void check() throws Exception { new String[] {"sync"}, new String[] {"false"}); Assert.assertNotNull(pushBlock); Assert.assertEquals(pushBlock.intValue(), blocks + 1); + + String minerBase58 = StringUtil.encode58Check(address); + // Query histogram bucket le="0.0" for empty blocks + Double emptyBlock = CollectorRegistry.defaultRegistry.getSampleValue( + "tron:block_transaction_count_bucket", + new String[] {MetricLabels.Histogram.MINER, "le"}, new String[] {minerBase58, "0.0"}); + + Assert.assertNotNull("Empty block bucket should exist for miner: " + minerBase58, emptyBlock); + Assert.assertEquals("Should have 1 empty block", 1, emptyBlock.intValue()); + + // Collect empty blocks for each new witness in witnessAndAccount (excluding initial address) + ByteString addressByteString = ByteString.copyFrom(address); + int totalNewWitnessEmptyBlocks = 0; + for (ByteString witnessAddress : witnessAndAccount.keySet()) { + if (witnessAddress.equals(addressByteString)) { + continue; + } + String witnessBase58 = StringUtil.encode58Check(witnessAddress.toByteArray()); + int witnessEmptyBlock = CollectorRegistry.defaultRegistry.getSampleValue( + "tron:block_transaction_count_bucket", + new String[] {MetricLabels.Histogram.MINER, "le"}, new String[] {witnessBase58, "0.0"}) + .intValue(); + totalNewWitnessEmptyBlocks += witnessEmptyBlock; + } + Assert.assertEquals(blocks, totalNewWitnessEmptyBlocks); + Double errorLogs = CollectorRegistry.defaultRegistry.getSampleValue( "tron:error_info_total", new String[] {"net"}, new String[] {MetricLabels.UNDEFINED}); Assert.assertNull(errorLogs); @@ -130,10 +160,16 @@ public void testMetric() throws Exception { Map witnessAndAccount = addTestWitnessAndAccount(); witnessAndAccount.put(ByteString.copyFrom(address), key); + + // Schedule the new witnesses (excluding initial address) so dposSlot rotates blocks among them + List newActiveWitnesses = new ArrayList<>(witnessAndAccount.keySet()); + newActiveWitnesses.remove(ByteString.copyFrom(address)); + chainBaseManager.getWitnessScheduleStore().saveActiveWitnesses(newActiveWitnesses); + for (int i = 0; i < blocks; i++) { generateBlock(witnessAndAccount); } - check(); + check(address, witnessAndAccount); } private Map addTestWitnessAndAccount() {