diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index a900e1c39e5..4164a32f3a1 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -57,8 +57,9 @@ import org.apache.accumulo.monitor.next.SystemInformation.TimeOrderedRunningCompactionSet; import org.apache.accumulo.monitor.next.deployment.DeploymentOverview; import org.apache.accumulo.monitor.next.ec.CompactorsSummary; -import org.apache.accumulo.monitor.next.views.ServersView; -import org.apache.accumulo.monitor.next.views.ServersView.Status; +import org.apache.accumulo.monitor.next.views.Status; +import org.apache.accumulo.monitor.next.views.TableData; +import org.apache.accumulo.monitor.next.views.TableDataFactory; import io.micrometer.core.instrument.Meter.Id; import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; @@ -267,12 +268,12 @@ public Map getScanServerAllMetricSummary() { @GET @Path("servers/view") @Produces(MediaType.APPLICATION_JSON) - @Description("Returns a UI-ready table model for server process pages. Add ';table=' to URL") - public ServersView getServerProcessView(@MatrixParam("table") ServersView.ServerTable table) { + @Description("Returns a UI-ready table model for server process pages. Add ';table=' to URL") + public TableData getServerProcessView(@MatrixParam("table") TableDataFactory.TableName table) { if (table == null) { throw new BadRequestException("A 'table' parameter is required"); } - ServersView view = + TableData view = monitor.getInformationFetcher().getSummaryForEndpoint().getServerProcessView(table); if (view == null) { throw new NotFoundException("ServersView object for table " + table.name() + " not found"); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index 2009068829b..411dcf9d39f 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -67,10 +67,11 @@ import org.apache.accumulo.core.spi.balancer.TableLoadBalancer; import org.apache.accumulo.core.util.compaction.RunningCompactionInfo; import org.apache.accumulo.monitor.next.deployment.DeploymentOverview; -import org.apache.accumulo.monitor.next.views.ServersView; -import org.apache.accumulo.monitor.next.views.ServersView.Column; -import org.apache.accumulo.monitor.next.views.ServersView.ColumnFactory; -import org.apache.accumulo.monitor.next.views.ServersView.Status; +import org.apache.accumulo.monitor.next.views.ColumnFactory; +import org.apache.accumulo.monitor.next.views.Status; +import org.apache.accumulo.monitor.next.views.TableData; +import org.apache.accumulo.monitor.next.views.TableData.Column; +import org.apache.accumulo.monitor.next.views.TableDataFactory; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.metrics.MetricResponseWrapper; @@ -431,8 +432,8 @@ public record CompactionGroupSummary(String groupId, long running) { private final AtomicLong timestamp = new AtomicLong(0); private final EnumMap componentStatuses = new EnumMap<>(ServerId.Type.class); - private final EnumMap> serverMetricsView = - new EnumMap<>(ServersView.ServerTable.class); + private final EnumMap> serverMetricsView = + new EnumMap<>(TableDataFactory.TableName.class); private DeploymentOverview deploymentOverview = new DeploymentOverview(0L, List.of()); private String managerGoalState; private final int rgLongRunningCompactionSize; @@ -512,18 +513,19 @@ private void updateAggregates(final MetricResponse response, } - private ServersView createCompactionQueueSummary(final Set managers) { + private TableData createCompactionQueueSummary(final Set managers) { final Column COMPACTION_QUEUE_COL = - new Column(ServersView.RG_COL_KEY, "Compaction Queue", "Compaction Queue", ""); + new Column(TableDataFactory.RG_COL_KEY, "Compaction Queue", "Compaction Queue", ""); - List cols = ServersView.columnsFor(ServersView.ServerTable.COORDINATOR_QUEUES); + List cols = + TableDataFactory.columnsFor(TableDataFactory.TableName.COORDINATOR_QUEUES); // Remove the column mapping for the resource group and replace it so that // the column header reads "Compaction Queue" instead of "Resource Group" int rgIdx = -1; for (int idx = 0; idx < cols.size(); idx++) { - if (cols.get(idx).getColumn().key().equals(ServersView.RG_COL_KEY)) { + if (cols.get(idx).getColumn().key().equals(TableDataFactory.RG_COL_KEY)) { rgIdx = idx; break; } @@ -583,9 +585,9 @@ public Object getRowData(ServerId sid, MetricResponse mr, if (!qm.isEmpty()) { // Create a ServersView object from the MetricResponse for each queue - return new ServersView(qm.keySet(), 0, qm, timestamp.get(), cols); + return TableDataFactory.forColumns(qm.keySet(), qm, timestamp.get(), cols); } - return new ServersView(Set.of(), 0, Map.of(), timestamp.get(), cols); + return TableDataFactory.forColumns(Set.of(), Map.of(), timestamp.get(), cols); } public void processResponse(final ServerId server, final MetricResponse response) { @@ -760,30 +762,30 @@ public void finish() { switch (type) { case COMPACTOR: compactors.values().forEach(servers::addAll); - cacheServerProcessView(ServersView.ServerTable.COMPACTORS, servers); + cacheServerProcessView(TableDataFactory.TableName.COMPACTORS, servers); break; case GARBAGE_COLLECTOR: servers.add(gc.get()); - cacheServerProcessView(ServersView.ServerTable.GC_SUMMARY, servers); - cacheServerProcessView(ServersView.ServerTable.GC_FILES, servers); - cacheServerProcessView(ServersView.ServerTable.GC_WALS, servers); + cacheServerProcessView(TableDataFactory.TableName.GC_SUMMARY, servers); + cacheServerProcessView(TableDataFactory.TableName.GC_FILES, servers); + cacheServerProcessView(TableDataFactory.TableName.GC_WALS, servers); break; case MANAGER: servers.addAll(managers); - cacheServerProcessView(ServersView.ServerTable.MANAGERS, servers); - cacheServerProcessView(ServersView.ServerTable.MANAGER_FATE, servers); - cacheServerProcessView(ServersView.ServerTable.MANAGER_COMPACTIONS, servers); - ServersView coordinatorQueues = createCompactionQueueSummary(servers); - serverMetricsView.put(ServersView.ServerTable.COORDINATOR_QUEUES, + cacheServerProcessView(TableDataFactory.TableName.MANAGERS, servers); + cacheServerProcessView(TableDataFactory.TableName.MANAGER_FATE, servers); + cacheServerProcessView(TableDataFactory.TableName.MANAGER_COMPACTIONS, servers); + TableData coordinatorQueues = createCompactionQueueSummary(servers); + serverMetricsView.put(TableDataFactory.TableName.COORDINATOR_QUEUES, memoize(() -> coordinatorQueues)); break; case SCAN_SERVER: sservers.values().forEach(servers::addAll); - cacheServerProcessView(ServersView.ServerTable.SCAN_SERVERS, servers); + cacheServerProcessView(TableDataFactory.TableName.SCAN_SERVERS, servers); break; case TABLET_SERVER: tservers.values().forEach(servers::addAll); - cacheServerProcessView(ServersView.ServerTable.TABLET_SERVERS, servers); + cacheServerProcessView(TableDataFactory.TableName.TABLET_SERVERS, servers); break; case MONITOR: default: @@ -822,14 +824,15 @@ private Status computeServerStatus(ServerId.Type type) { long problemHostCount = problemHosts.stream().filter(serverId -> serverId.getType() == type).count(); int missingMetricCount = (int) servers.stream() - .filter(serverId -> !ServersView.hasMetricData(allMetrics.getIfPresent(serverId))).count(); - return ServersView.buildStatus(servers.size(), problemHostCount, missingMetricCount, + .filter(serverId -> !TableDataFactory.hasMetricData(allMetrics.getIfPresent(serverId))) + .count(); + return Status.buildStatus(servers.size(), problemHostCount, missingMetricCount, type == ServerId.Type.TABLET_SERVER); } private String computeManagerGoalState() { Integer goalState = managers.stream().map(allMetrics::getIfPresent) - .map(response -> ServersView.metricValuesByName(response) + .map(response -> TableDataFactory.metricValuesByName(response) .get(Metric.MANAGER_GOAL_STATE.getName())) .filter(value -> value != null && !value.isEmpty()) .map(value -> value.stream().map(SystemInformation::getMetricValue).filter(Objects::nonNull) @@ -949,15 +952,13 @@ public long getTimestamp() { /** * Cache a ServersView for the given table and set of servers. */ - private void cacheServerProcessView(ServersView.ServerTable table, Set servers) { - long problemHostCount = - problemHosts.stream().filter(serverId -> servers.contains(serverId)).count(); - serverMetricsView.put(table, memoize(() -> new ServersView(servers, problemHostCount, - allMetrics.asMap(), timestamp.get(), ServersView.columnsFor(table)))); + private void cacheServerProcessView(TableDataFactory.TableName table, Set servers) { + serverMetricsView.put(table, memoize( + () -> TableDataFactory.forTable(table, servers, allMetrics.asMap(), timestamp.get()))); } - public ServersView getServerProcessView(ServersView.ServerTable table) { - Supplier view = this.serverMetricsView.get(table); + public TableData getServerProcessView(TableDataFactory.TableName table) { + Supplier view = this.serverMetricsView.get(table); if (view != null) { return view.get(); } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ColumnFactory.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ColumnFactory.java new file mode 100644 index 00000000000..3b13a692568 --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ColumnFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.next.views; + +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.core.metrics.MonitorMeterRegistry; +import org.apache.accumulo.core.metrics.flatbuffers.FMetric; +import org.apache.accumulo.core.process.thrift.MetricResponse; +import org.apache.accumulo.monitor.next.SystemInformation; +import org.apache.accumulo.monitor.next.views.TableData.Column; + +public interface ColumnFactory { + + default Number computeRate(Number sum) { + if (sum == null) { + return null; + } + return sum.doubleValue() / MonitorMeterRegistry.STEP.toSeconds(); + } + + default Number add(Number n1, Number n2) { + if (n1 == null && n2 == null) { + return null; + } else if (n1 == null) { + return n2; + } else if (n2 == null) { + return n1; + } else if (n1 instanceof Double || n2 instanceof Double) { + return n1.doubleValue() + n2.doubleValue(); + } else if (n1 instanceof Long || n2 instanceof Long) { + return n1.longValue() + n2.longValue(); + } else if (n1 instanceof Integer || n2 instanceof Integer) { + return n1.intValue() + n2.intValue(); + } else { + throw new IllegalArgumentException( + "Unexpected value type: " + n1.getClass().getName() + " " + n2.getClass().getName()); + } + } + + default Number sum(List metrics) { + Number sum = null; + for (var metric : metrics) { + sum = add(sum, SystemInformation.getMetricValue(metric)); + } + return sum; + } + + Column getColumn(); + + Object getRowData(ServerId sid, MetricResponse mr, Map> serverMetrics); + +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ExecutorColumnFactory.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ExecutorColumnFactory.java new file mode 100644 index 00000000000..2e611e00668 --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ExecutorColumnFactory.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.next.views; + +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.core.metrics.Metric; +import org.apache.accumulo.core.metrics.flatbuffers.FMetric; +import org.apache.accumulo.core.metrics.flatbuffers.FTag; +import org.apache.accumulo.core.process.thrift.MetricResponse; +import org.apache.accumulo.monitor.next.SystemInformation; +import org.apache.accumulo.monitor.next.views.TableData.Column; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ExecutorColumnFactory implements ColumnFactory { + + private static final Logger log = LoggerFactory.getLogger(ExecutorColumnFactory.class); + + private final Column column; + private final String metricName; + private final Predicate tagPredicate; + private final Type type; + + enum Type { + COMPLETED, QUEUED + } + + String getMetricName(Type t) { + return switch (t) { + case COMPLETED -> Metric.EXECUTOR_COMPLETED.getName(); + case QUEUED -> Metric.EXECUTOR_QUEUED.getName(); + }; + } + + String getCssClass(Type t) { + return switch (t) { + case COMPLETED -> Metric.MonitorCssClass.RATE.getCssClass(); + case QUEUED -> Metric.MonitorCssClass.NUMBER.getCssClass(); + }; + } + + ExecutorColumnFactory(Type type, String theadPoolPrefix, String label, String description) { + this.type = type; + this.metricName = getMetricName(type); + this.tagPredicate = s -> s.startsWith(theadPoolPrefix); + this.column = + new Column(metricName + "-" + theadPoolPrefix, label, description, getCssClass(type)); + } + + ExecutorColumnFactory(Type type, String keySuffix, Predicate tagPredicate, String label, + String description) { + this.type = type; + this.metricName = getMetricName(type); + this.tagPredicate = tagPredicate; + this.column = new Column(metricName + "-" + keySuffix, label, description, getCssClass(type)); + } + + @Override + public Column getColumn() { + return column; + } + + @Override + public Object getRowData(ServerId sid, MetricResponse mr, + Map> serverMetrics) { + + var metrics = serverMetrics.getOrDefault(metricName, List.of()); + + Number sum = null; + + FTag ftag = new FTag(); + for (var metric : metrics) { + boolean foundTag = false; + String tag = null; + for (int i = 0; i < metric.tagsLength(); i++) { + metric.tags(ftag, i); + var key = ftag.key(); + var value = ftag.value(); + if (key != null && value != null && key.equals("name") && tagPredicate.test(value)) { + foundTag = true; + tag = value; + break; + } + } + + var metricStatistic = TableDataFactory.extractStatistic(metric); + if (foundTag && (metricStatistic == null || metricStatistic.equals("value") + || metricStatistic.equals("count"))) { + var val = SystemInformation.getMetricValue(metric); + log.trace("adding {}+{} for {} {} {} {}", sum, val, metric.name(), tag, + sid.toHostPortString(), sid.getResourceGroup()); + sum = add(sum, SystemInformation.getMetricValue(metric)); + + } + } + + if (type == Type.COMPLETED) { + // Convert to a rate + return computeRate(sum); + } + + return sum; + } + +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/MetricColumnFactory.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/MetricColumnFactory.java new file mode 100644 index 00000000000..10c0372f74e --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/MetricColumnFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.next.views; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.core.metrics.Metric; +import org.apache.accumulo.core.metrics.flatbuffers.FMetric; +import org.apache.accumulo.core.process.thrift.MetricResponse; +import org.apache.accumulo.monitor.next.views.TableData.Column; + +public class MetricColumnFactory implements ColumnFactory { + + private final Column column; + private final boolean computeRate; + + MetricColumnFactory(Metric metric) { + String classes; + if (metric.getType() == Metric.MetricType.FUNCTION_COUNTER) { + if (Arrays.asList(metric.getColumnClasses()).contains(Metric.MonitorCssClass.BYTES)) { + classes = Metric.MonitorCssClass.BYTES_RATE.getCssClass(); + } else { + classes = Metric.MonitorCssClass.RATE.getCssClass(); + } + computeRate = true; + } else { + classes = Arrays.stream(metric.getColumnClasses()).map(Metric.MonitorCssClass::getCssClass) + .collect(Collectors.joining(" ")); + computeRate = false; + } + this.column = new Column(metric.getName(), metric.getColumnHeader(), + metric.getColumnDescription(), classes); + } + + @Override + public Column getColumn() { + return column; + } + + @Override + public Object getRowData(ServerId sid, MetricResponse mr, + Map> serverMetrics) { + var sum = sum(serverMetrics.getOrDefault(column.key(), List.of())); + if (computeRate) { + return computeRate(sum); + } else { + return sum; + } + } +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/RatioColumnFactory.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/RatioColumnFactory.java new file mode 100644 index 00000000000..15c4be5f9cd --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/RatioColumnFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.next.views; + +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.core.metrics.Metric; +import org.apache.accumulo.core.metrics.flatbuffers.FMetric; +import org.apache.accumulo.core.process.thrift.MetricResponse; +import org.apache.accumulo.monitor.next.views.TableData.Column; + +public class RatioColumnFactory implements ColumnFactory { + + private final Column column; + private final Metric numerator; + private final Metric denominator; + + RatioColumnFactory(String label, String description, Metric numerator, Metric denominator) { + this.column = new Column(numerator.getName() + "/" + denominator.getName(), label, description, + Metric.MonitorCssClass.PERCENT.getCssClass()); + this.numerator = numerator; + this.denominator = denominator; + } + + @Override + public Column getColumn() { + return column; + } + + @Override + public Object getRowData(ServerId sid, MetricResponse mr, + Map> serverMetrics) { + var n = serverMetrics.get(numerator.getName()); + var d = serverMetrics.get(denominator.getName()); + + if (n == null || d == null) { + return null; + } + + var numeratorSum = sum(n).doubleValue(); + var denominatorSum = sum(n).doubleValue(); + + if (denominatorSum == 0) { + return null; + } + + return numeratorSum / denominatorSum; + } +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/Status.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/Status.java new file mode 100644 index 00000000000..e89a039fcad --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/Status.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.next.views; + +import java.util.ArrayList; +import java.util.List; + +/** + * all the data needed for the Server status indicator(s) + */ +public record Status(boolean hasServers, boolean hasProblemServers, boolean hasMissingMetrics, + long serverCount, long problemServerCount, long missingMetricServerCount, String level, + String message) { + + private static final String LEVEL_OK = "OK"; + private static final String LEVEL_ERROR = "ERROR"; + private static final String LEVEL_WARN = "WARN"; + + public static Status buildStatus(int serverCount, long problemServerCount, + int serversMissingMetrics) { + return buildStatus(serverCount, problemServerCount, serversMissingMetrics, false); + } + + public static Status buildStatus(int serverCount, long problemServerCount, + int serversMissingMetrics, boolean errorWhenAllServersUnavailable) { + final boolean hasServers = serverCount > 0; + final boolean hasProblemServers = problemServerCount > 0; + final boolean hasMissingMetrics = serversMissingMetrics > 0; + final boolean allServersUnavailable = + errorWhenAllServersUnavailable && hasServers && problemServerCount >= serverCount; + + if (allServersUnavailable) { + return new Status(true, true, hasMissingMetrics, serverCount, problemServerCount, + serversMissingMetrics, LEVEL_ERROR, "ERROR: All servers are unavailable."); + } + + List warnings = new ArrayList<>(2); + if (hasProblemServers) { + warnings.add("One or more servers are unavailable"); + } + if (hasMissingMetrics) { + warnings.add("Metrics are not present (are metrics enabled?)"); + } + + if (warnings.isEmpty()) { + // no warnings, set status to OK + return new Status(hasServers, false, false, serverCount, 0, 0, LEVEL_OK, null); + } + + final String message = "WARN: " + String.join("; ", warnings) + "."; + return new Status(hasServers, hasProblemServers, hasMissingMetrics, serverCount, + problemServerCount, serversMissingMetrics, LEVEL_WARN, message); + } + +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/TableData.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/TableData.java new file mode 100644 index 00000000000..7b08cf0b963 --- /dev/null +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/TableData.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.monitor.next.views; + +import java.util.List; +import java.util.Map; + +/** + * Generic Data Transfer Object (DTO) for a set of rows that have the same columns. Each row will + * typically represent a server and the columns would typically include the server's address, + * resource group, and type, and a set of metrics. + * + * The response object contains several fields: + * + * 
+ * columns - contains an array of column definitions that can be used to create the table headers
+ *           and DataTable columns
+ * data    - an array of objects that can be used for the DataTable data definition
+ *
+ * + */ +public record TableData(List columns, List> data, long timestamp) { + + /** + * Definition of a column to be rendered in the UI + */ + public record Column(String key, String label, String description, String uiClass) { + } + +} diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ServersView.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/TableDataFactory.java similarity index 55% rename from server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ServersView.java rename to server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/TableDataFactory.java index cb750437521..e3eb7310f84 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/ServersView.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/views/TableDataFactory.java @@ -18,9 +18,9 @@ */ package org.apache.accumulo.monitor.next.views; -import static org.apache.accumulo.monitor.next.views.ServersView.MetricFilterPrefixes.GC_WAL; -import static org.apache.accumulo.monitor.next.views.ServersView.MetricFilterPrefixes.MINC; -import static org.apache.accumulo.monitor.next.views.ServersView.MetricFilterPrefixes.MINC_COMPACTION; +import static org.apache.accumulo.monitor.next.views.TableDataFactory.MetricFilterPrefixes.GC_WAL; +import static org.apache.accumulo.monitor.next.views.TableDataFactory.MetricFilterPrefixes.MINC; +import static org.apache.accumulo.monitor.next.views.TableDataFactory.MetricFilterPrefixes.MINC_COMPACTION; import java.util.ArrayList; import java.util.Arrays; @@ -35,57 +35,38 @@ import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.metrics.Metric; import org.apache.accumulo.core.metrics.Metric.MetricDocSection; -import org.apache.accumulo.core.metrics.MonitorMeterRegistry; import org.apache.accumulo.core.metrics.flatbuffers.FMetric; import org.apache.accumulo.core.metrics.flatbuffers.FTag; import org.apache.accumulo.core.process.thrift.MetricResponse; import org.apache.accumulo.core.util.threads.ThreadPoolNames; -import org.apache.accumulo.monitor.next.SystemInformation; +import org.apache.accumulo.monitor.next.views.TableData.Column; import org.apache.accumulo.server.metrics.MetricResponseWrapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** - * Generic Data Transfer Object (DTO) for a set of Accumulo server processes of the same type. The - * response object contains several fields: - * - * - * 
- * columns - contains an array of column definitions that can be used to create the table headers
- *           and Data Table columns
- * data    - an array of objects that can be used for the Data Table data definition
- *
- * - * Each server-process table is identified by {@link ServerTable}. The table-specific metric methods - * define the metrics for each table, {@link #columnsFor(ServerTable)} converts those metrics to - * column definitions. The frontend uses the returned column definitions to build the table headers - * and DataTables column configuration. + * A factory for creating {@code TableData} Data Transfer Objects identified by a + * {@link TableDataFactory.TableName}. The {@link TableDataFactory.TableName} defines the metrics + * for each table, {@link #columnsFor(TableName)} converts those metrics to column definitions. The + * frontend uses the returned column definitions to build the table headers and DataTables column + * configuration. */ -public class ServersView { - - /** - * all the data needed for the Server status indicator(s) - */ - public record Status(boolean hasServers, boolean hasProblemServers, boolean hasMissingMetrics, - long serverCount, long problemServerCount, long missingMetricServerCount, String level, - String message) { - } +public class TableDataFactory { - /** - * Definition of a column to be rendered in the UI - */ - public record Column(String key, String label, String description, String uiClass) { - } + public static final String RG_COL_KEY = "resourceGroup"; + public static final String ADDR_COL_KEY = "serverAddress"; + public static final String TIME_COL_KEY = "lastContact"; - private record ServerMetricRow(ServerId server, MetricResponse response, - Map> metrics) { - } + public static final Column LAST_CONTACT_COLUMN = new Column(TIME_COL_KEY, "Last Contact", + "Time since the server last responded to the monitor", "duration"); + public static final Column RG_COLUMN = + new Column(RG_COL_KEY, "Resource Group", "Resource Group", ""); + public static final Column ADDR_COLUMN = + new Column(ADDR_COL_KEY, "Server Address", "Server address", ""); /** * Server-process table identifiers accepted by /rest-v2/servers/view. These enum names are used * directly as the frontend table parameter values. */ - public enum ServerTable { + public enum TableName { COORDINATOR_QUEUES, COMPACTORS, GC_SUMMARY, @@ -98,21 +79,6 @@ public enum ServerTable { TABLET_SERVERS } - private static final String LEVEL_OK = "OK"; - private static final String LEVEL_ERROR = "ERROR"; - private static final String LEVEL_WARN = "WARN"; - - public static final String RG_COL_KEY = "resourceGroup"; - public static final String ADDR_COL_KEY = "serverAddress"; - public static final String TIME_COL_KEY = "lastContact"; - - public static final Column LAST_CONTACT_COLUMN = new Column(TIME_COL_KEY, "Last Contact", - "Time since the server last responded to the monitor", "duration"); - public static final Column RG_COLUMN = - new Column(RG_COL_KEY, "Resource Group", "Resource Group", ""); - public static final Column ADDR_COLUMN = - new Column(ADDR_COL_KEY, "Server Address", "Server address", ""); - /** * Common columns that are included in every ServersView table */ @@ -154,13 +120,19 @@ public Object getRowData(ServerId sid, MetricResponse mr, } }); - public final List> data = new ArrayList<>(); - public final List columns; - public final long timestamp; + private record ServerMetricRow(ServerId server, MetricResponse response, + Map> metrics) { + } - public ServersView(final Set servers, final long problemServerCount, + public static boolean hasMetricData(MetricResponse mr) { + return mr != null && mr.getMetrics() != null && !mr.getMetrics().isEmpty(); + } + + public static TableData forColumns(final Set servers, final Map allMetrics, final long timestamp, - final List requestedColumns) { + List requestedColumns) { + + List columns = requestedColumns.stream().map(ColumnFactory::getColumn).toList(); // Grab the current metrics for each server List serverMetricRows = servers.stream().sorted().map(serverId -> { @@ -172,8 +144,7 @@ public ServersView(final Set servers, final long problemServerCount, return new ServerMetricRow(serverId, metricResponse, serverMetrics); }).toList(); - this.columns = requestedColumns.stream().map(ColumnFactory::getColumn).toList(); - + List> data = new ArrayList<>(); serverMetricRows.forEach(serverMetricRow -> { Map row = new LinkedHashMap<>(); for (ColumnFactory colf : requestedColumns) { @@ -182,238 +153,23 @@ public ServersView(final Set servers, final long problemServerCount, } data.add(row); }); - this.timestamp = timestamp; - } - - public static Status buildStatus(int serverCount, long problemServerCount, - int serversMissingMetrics) { - return buildStatus(serverCount, problemServerCount, serversMissingMetrics, false); - } - - public static Status buildStatus(int serverCount, long problemServerCount, - int serversMissingMetrics, boolean errorWhenAllServersUnavailable) { - final boolean hasServers = serverCount > 0; - final boolean hasProblemServers = problemServerCount > 0; - final boolean hasMissingMetrics = serversMissingMetrics > 0; - final boolean allServersUnavailable = - errorWhenAllServersUnavailable && hasServers && problemServerCount >= serverCount; - - if (allServersUnavailable) { - return new Status(true, true, hasMissingMetrics, serverCount, problemServerCount, - serversMissingMetrics, LEVEL_ERROR, "ERROR: All servers are unavailable."); - } - List warnings = new ArrayList<>(2); - if (hasProblemServers) { - warnings.add("One or more servers are unavailable"); - } - if (hasMissingMetrics) { - warnings.add("Metrics are not present (are metrics enabled?)"); - } + return new TableData(columns, data, timestamp); - if (warnings.isEmpty()) { - // no warnings, set status to OK - return new Status(hasServers, false, false, serverCount, 0, 0, LEVEL_OK, null); - } - - final String message = "WARN: " + String.join("; ", warnings) + "."; - return new Status(hasServers, hasProblemServers, hasMissingMetrics, serverCount, - problemServerCount, serversMissingMetrics, LEVEL_WARN, message); - } - - public static boolean hasMetricData(MetricResponse mr) { - return mr != null && mr.getMetrics() != null && !mr.getMetrics().isEmpty(); - } - - public interface ColumnFactory { - Column getColumn(); - - Object getRowData(ServerId sid, MetricResponse mr, Map> serverMetrics); - } - - private static class RatioColumnFactory implements ColumnFactory { - - private final Column column; - private final Metric numerator; - private final Metric denominator; - - RatioColumnFactory(String label, String description, Metric numerator, Metric denominator) { - this.column = new Column(numerator.getName() + "/" + denominator.getName(), label, - description, Metric.MonitorCssClass.PERCENT.getCssClass()); - this.numerator = numerator; - this.denominator = denominator; - } - - @Override - public Column getColumn() { - return column; - } - - @Override - public Object getRowData(ServerId sid, MetricResponse mr, - Map> serverMetrics) { - var n = serverMetrics.get(numerator.getName()); - var d = serverMetrics.get(denominator.getName()); - - if (n == null || d == null) { - return null; - } - - var numeratorSum = sum(n).doubleValue(); - var denominatorSum = sum(n).doubleValue(); - - if (denominatorSum == 0) { - return null; - } - - return numeratorSum / denominatorSum; - } } - private static class MetricColumnFactory implements ColumnFactory { - - private final Column column; - private final boolean computeRate; - - MetricColumnFactory(Metric metric) { - String classes; - if (metric.getType() == Metric.MetricType.FUNCTION_COUNTER) { - if (Arrays.asList(metric.getColumnClasses()).contains(Metric.MonitorCssClass.BYTES)) { - classes = Metric.MonitorCssClass.BYTES_RATE.getCssClass(); - } else { - classes = Metric.MonitorCssClass.RATE.getCssClass(); - } - computeRate = true; - } else { - classes = Arrays.stream(metric.getColumnClasses()).map(Metric.MonitorCssClass::getCssClass) - .collect(Collectors.joining(" ")); - computeRate = false; - } - this.column = new Column(metric.getName(), metric.getColumnHeader(), - metric.getColumnDescription(), classes); - } + public static TableData forTable(final TableName table, final Set servers, + final Map allMetrics, final long timestamp) { - @Override - public Column getColumn() { - return column; - } - - @Override - public Object getRowData(ServerId sid, MetricResponse mr, - Map> serverMetrics) { - var sum = sum(serverMetrics.getOrDefault(column.key, List.of())); - if (computeRate) { - return computeRate(sum); - } else { - return sum; - } - } - } - - private static class ExecutorColumnFactory implements ColumnFactory { - - private static final Logger log = LoggerFactory.getLogger(ExecutorColumnFactory.class); - - private final Column column; - private final String metricName; - private final Predicate tagPredicate; - private final Type type; - - enum Type { - COMPLETED, QUEUED - } - - String getMetricName(Type t) { - return switch (t) { - case COMPLETED -> Metric.EXECUTOR_COMPLETED.getName(); - case QUEUED -> Metric.EXECUTOR_QUEUED.getName(); - }; - } - - String getCssClass(Type t) { - return switch (t) { - case COMPLETED -> Metric.MonitorCssClass.RATE.getCssClass(); - case QUEUED -> Metric.MonitorCssClass.NUMBER.getCssClass(); - }; - } - - ExecutorColumnFactory(Type type, String theadPoolPrefix, String label, String description) { - this.type = type; - this.metricName = getMetricName(type); - this.tagPredicate = s -> s.startsWith(theadPoolPrefix); - this.column = - new Column(metricName + "-" + theadPoolPrefix, label, description, getCssClass(type)); - } - - ExecutorColumnFactory(Type type, String keySuffix, Predicate tagPredicate, String label, - String description) { - this.type = type; - this.metricName = getMetricName(type); - this.tagPredicate = tagPredicate; - this.column = new Column(metricName + "-" + keySuffix, label, description, getCssClass(type)); - } - - @Override - public Column getColumn() { - return column; - } - - @Override - public Object getRowData(ServerId sid, MetricResponse mr, - Map> serverMetrics) { - - var metrics = serverMetrics.getOrDefault(metricName, List.of()); - - Number sum = null; - - FTag ftag = new FTag(); - for (var metric : metrics) { - boolean foundTag = false; - String tag = null; - for (int i = 0; i < metric.tagsLength(); i++) { - metric.tags(ftag, i); - var key = ftag.key(); - var value = ftag.value(); - if (key != null && value != null && key.equals("name") && tagPredicate.test(value)) { - foundTag = true; - tag = value; - break; - } - } - - var metricStatistic = extractStatistic(metric); - if (foundTag && (metricStatistic == null || metricStatistic.equals("value") - || metricStatistic.equals("count"))) { - var val = SystemInformation.getMetricValue(metric); - log.trace("adding {}+{} for {} {} {} {}", sum, val, metric.name(), tag, - sid.toHostPortString(), sid.getResourceGroup()); - sum = add(sum, SystemInformation.getMetricValue(metric)); - - } - } - - if (type == Type.COMPLETED) { - // Convert to a rate - return computeRate(sum); - } - - return sum; - } - - } - - private static Number computeRate(Number sum) { - if (sum == null) { - return null; - } - return sum.doubleValue() / MonitorMeterRegistry.STEP.toSeconds(); + List requestedColumns = columnsFor(table); + return forColumns(servers, allMetrics, timestamp, requestedColumns); } /** * Builds the final ordered columns for a table. First adds the common columns, then adds the * table specific metrics. */ - public static List columnsFor(ServerTable table) { + public static List columnsFor(TableName table) { List cols = new ArrayList<>(COMMON_COLUMNS); switch (table) { @@ -491,6 +247,41 @@ private static void tabletServerColumns(List cols) { // TODO create scan problems that is a sum of zombie and low memory } + public static Map> metricValuesByName(MetricResponse response) { + var values = new HashMap>(); + if (response == null || response.getMetrics() == null || response.getMetrics().isEmpty()) { + return values; + } + + for (var binary : response.getMetrics()) { + var metric = FMetric.getRootAsFMetric(binary); + var metricStatistic = extractStatistic(metric); + if (metricStatistic == null || metricStatistic.equals("value") + || metricStatistic.equals("count")) { + values.computeIfAbsent(metric.name(), m -> new ArrayList<>()).add(metric); + } + } + return values; + } + + public static String extractStatistic(FMetric metric) { + FTag tag = new FTag(); + for (int i = 0; i < metric.tagsLength(); i++) { + tag = metric.tags(tag, i); + if (MetricResponseWrapper.STATISTIC_TAG.equals(tag.key())) { + return normalizeStatistic(tag.value()); + } + } + return null; + } + + private static String normalizeStatistic(String statistic) { + if (statistic == null) { + return null; + } + return statistic.toLowerCase(); + } + // Filters to use with the metricList method public static enum MetricFilterPrefixes { @@ -553,11 +344,6 @@ private static List scanServerMetrics() { MetricDocSection.SCAN, MetricDocSection.BLOCK_CACHE); } - private static List tabletServerMetrics() { - return metricList(null, MetricDocSection.GENERAL_SERVER, MetricDocSection.TABLET_SERVER, - MetricDocSection.SCAN, MetricDocSection.COMPACTION, MetricDocSection.BLOCK_CACHE); - } - /** * @return all the metrics for the given sections */ @@ -573,66 +359,4 @@ private static List metricList(Predicate filter, MetricDocSectio .filter(filter == null ? m -> true : filter).collect(Collectors.toList()); } - public static Number add(Number n1, Number n2) { - if (n1 == null && n2 == null) { - return null; - } else if (n1 == null) { - return n2; - } else if (n2 == null) { - return n1; - } else if (n1 instanceof Double || n2 instanceof Double) { - return n1.doubleValue() + n2.doubleValue(); - } else if (n1 instanceof Long || n2 instanceof Long) { - return n1.longValue() + n2.longValue(); - } else if (n1 instanceof Integer || n2 instanceof Integer) { - return n1.intValue() + n2.intValue(); - } else { - throw new IllegalArgumentException( - "Unexpected value type: " + n1.getClass().getName() + " " + n2.getClass().getName()); - } - } - - private static Number sum(List metrics) { - Number sum = null; - for (var metric : metrics) { - sum = add(sum, SystemInformation.getMetricValue(metric)); - } - return sum; - } - - public static Map> metricValuesByName(MetricResponse response) { - var values = new HashMap>(); - if (response == null || response.getMetrics() == null || response.getMetrics().isEmpty()) { - return values; - } - - for (var binary : response.getMetrics()) { - var metric = FMetric.getRootAsFMetric(binary); - var metricStatistic = extractStatistic(metric); - if (metricStatistic == null || metricStatistic.equals("value") - || metricStatistic.equals("count")) { - values.computeIfAbsent(metric.name(), m -> new ArrayList<>()).add(metric); - } - } - return values; - } - - private static String extractStatistic(FMetric metric) { - FTag tag = new FTag(); - for (int i = 0; i < metric.tagsLength(); i++) { - tag = metric.tags(tag, i); - if (MetricResponseWrapper.STATISTIC_TAG.equals(tag.key())) { - return normalizeStatistic(tag.value()); - } - } - return null; - } - - private static String normalizeStatistic(String statistic) { - if (statistic == null) { - return null; - } - return statistic.toLowerCase(); - } - }