Skip to content

Commit e368f08

Browse files
mqliangmqliang
committed
Prototyping of OpenTelemetry metric plugin
1 parent 2188ca2 commit e368f08

File tree

35 files changed

+1654
-29
lines changed

35 files changed

+1654
-29
lines changed

‎pinot-common/src/main/java/org/apache/pinot/common/metrics/AbstractMetrics.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -603,7 +603,7 @@ public void addCallbackGaugeIfNeeded(final String metricName, final Callable<Lon
603603
public void addCallbackGauge(final String metricName, final Callable<Long> valueCallback) {
604604
PinotMetricUtils
605605
.makeGauge(_metricsRegistry, PinotMetricUtils.makePinotMetricName(_clazz, _metricPrefix + metricName),
606-
PinotMetricUtils.makePinotGauge(avoid -> {
606+
PinotMetricUtils.makePinotGauge(_metricPrefix + metricName, avoid -> {
607607
try {
608608
return valueCallback.call();
609609
} catch (Exception e) {
@@ -680,7 +680,7 @@ public void setOrUpdateTableGauge(final String tableName, final G gauge,
680680
public void setOrUpdateGauge(final String metricName, long value) {
681681
PinotGauge<Long> pinotGauge = PinotMetricUtils.makeGauge(_metricsRegistry,
682682
PinotMetricUtils.makePinotMetricName(_clazz, _metricPrefix + metricName),
683-
PinotMetricUtils.makePinotGauge(avoid -> value));
683+
PinotMetricUtils.makePinotGauge(_metricPrefix + metricName, avoid -> value));
684684
pinotGauge.setValue(value);
685685
}
686686

@@ -694,7 +694,7 @@ public void setOrUpdateGauge(final String metricName, long value) {
694694
public void setOrUpdateGauge(final String metricName, final Supplier<Long> valueSupplier) {
695695
PinotGauge<Long> pinotGauge = PinotMetricUtils.makeGauge(_metricsRegistry,
696696
PinotMetricUtils.makePinotMetricName(_clazz, _metricPrefix + metricName),
697-
PinotMetricUtils.makePinotGauge(avoid -> valueSupplier.get()));
697+
PinotMetricUtils.makePinotGauge(_metricPrefix + metricName, avoid -> valueSupplier.get()));
698698
pinotGauge.setValueSupplier(valueSupplier);
699699
}
700700

@@ -704,7 +704,7 @@ public void setOrUpdateGauge(final String metricName, final Supplier<Long> value
704704
public void setOrUpdateGauge(final String metricName, final LongSupplier valueSupplier) {
705705
PinotGauge<Long> pinotGauge = PinotMetricUtils.makeGauge(_metricsRegistry,
706706
PinotMetricUtils.makePinotMetricName(_clazz, _metricPrefix + metricName),
707-
PinotMetricUtils.makePinotGauge(avoid -> valueSupplier.getAsLong()));
707+
PinotMetricUtils.makePinotGauge(_metricPrefix + metricName, avoid -> valueSupplier.getAsLong()));
708708
pinotGauge.setValueSupplier((Supplier<Long>) () -> (Long) valueSupplier.getAsLong());
709709
}
710710

‎pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public Long value() {
5757

5858
@Override
5959
public Object getGauge() {
60-
return PinotMetricUtils.makePinotGauge(avoid -> value()).getGauge();
60+
return PinotMetricUtils.makePinotGauge(_key, avoid -> value()).getGauge();
6161
}
6262

6363
@Override
@@ -92,10 +92,9 @@ public Double value() {
9292
public Object getMetric() {
9393
return getGauge();
9494
}
95-
9695
@Override
9796
public Object getGauge() {
98-
return PinotMetricUtils.makePinotGauge(avoid -> value()).getGauge();
97+
return PinotMetricUtils.makePinotGauge(_key, avoid -> value()).getGauge();
9998
}
10099
}
101100

‎pinot-common/src/test/java/org/apache/pinot/common/metrics/prometheus/PinotPrometheusMetricsTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public void setupTest() {
9797
_pinotMetricsFactory.getClass().getCanonicalName());
9898
PinotMetricUtils.init(pinotConfiguration);
9999

100-
_pinotMetricsFactory.makePinotJmxReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start();
100+
_pinotMetricsFactory.makePinotMetricReporter(_pinotMetricsFactory.getPinotMetricsRegistry()).start();
101101
_httpClient = new HttpClient();
102102
_httpServer = startExporter();
103103
}

‎pinot-plugins/pinot-metrics/pinot-compound-metrics/src/main/java/org/apache/pinot/plugin/metrics/compound/CompoundPinotJmxReporter.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@
2222
import org.apache.pinot.spi.metrics.PinotJmxReporter;
2323

2424

25+
/**
26+
* CompoundPinotJmxReporter is a composite reporter that aggregates multiple PinotJmxReporters.
27+
* @deprecated Use {@link org.apache.pinot.plugin.metrics.compound.CompoundPinotMetricReporter} instead.
28+
*/
29+
@Deprecated
2530
public class CompoundPinotJmxReporter implements PinotJmxReporter {
2631
private final List<PinotJmxReporter> _reporters;
2732

‎pinot-plugins/pinot-metrics/pinot-compound-metrics/src/main/java/org/apache/pinot/plugin/metrics/compound/CompoundPinotMetricRegistry.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,8 @@ public <T> PinotGauge<T> newGauge(PinotMetricName name, PinotGauge<T> gauge) {
7272
return (PinotGauge<T>) _allMetrics.computeIfAbsent(name,
7373
key -> new CompoundPinotGauge<>(map(key, (reg, subName) -> reg.newGauge(subName, null))));
7474
} else {
75-
CompoundPinotGauge<T> compoundGauge =
76-
(CompoundPinotGauge<T>) PinotMetricUtils.makePinotGauge(avoid -> gauge.value());
75+
CompoundPinotGauge<T> compoundGauge = (CompoundPinotGauge<T>) PinotMetricUtils
76+
.makePinotGauge(name.getMetricName().toString(), avoid -> gauge.value());
7777

7878
Function<PinotMetricName, CompoundPinotGauge<?>> creator = key -> {
7979
CompoundPinotMetricName compoundName = (CompoundPinotMetricName) key;

‎pinot-plugins/pinot-metrics/pinot-compound-metrics/src/main/java/org/apache/pinot/plugin/metrics/compound/CompoundPinotMetricReporter.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.pinot.plugin.metrics.compound;
20+
21+
import java.util.List;
22+
import org.apache.pinot.spi.metrics.PinotMetricReporter;
23+
24+
25+
/**
26+
* CompoundPinotMetricReporter delegates metric reporting to each of the reporters in the list.
27+
*/
28+
public class CompoundPinotMetricReporter implements PinotMetricReporter {
29+
private final List<PinotMetricReporter> _reporters;
30+
31+
public CompoundPinotMetricReporter(List<PinotMetricReporter> reporters) {
32+
_reporters = reporters;
33+
}
34+
35+
@Override
36+
public void start() {
37+
for (PinotMetricReporter reporter : _reporters) {
38+
reporter.start();
39+
}
40+
}
41+
}

‎pinot-plugins/pinot-metrics/pinot-compound-metrics/src/main/java/org/apache/pinot/plugin/metrics/compound/CompoundPinotMetricsFactory.java

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.pinot.spi.metrics.PinotGauge;
3636
import org.apache.pinot.spi.metrics.PinotJmxReporter;
3737
import org.apache.pinot.spi.metrics.PinotMetricName;
38+
import org.apache.pinot.spi.metrics.PinotMetricReporter;
3839
import org.apache.pinot.spi.metrics.PinotMetricUtils;
3940
import org.apache.pinot.spi.metrics.PinotMetricsRegistry;
4041
import org.apache.pinot.spi.plugin.PluginManager;
@@ -47,11 +48,14 @@
4748
* When it is created, a bunch of factories are used as sub-factories. Whenever a metric is registered in this factory,
4849
* it actually registers the metric in all the sub-factories.
4950
*
50-
* Probably the main reason to use this metrics is to compare the differences between one metric registry and another.
51-
* For example, Yammer and Dropwizard provide their own timer, but each one provides their own metrics on their timers.
52-
* Most metrics are the same (p50, p90, p95, etc) but some other may be different.
51+
* The main use cases for this metrics registry including:
52+
* - to progressively migrate from one metrics plugin to another. In production there must be a lot of dashboards
53+
* and alerts set up based on old metrics, it needs dual reporting for some time to allow progressive migration.
54+
* - to compare the differences between different metrics registries. for example, Yammer and Dropwizard provide their
55+
* own timer, but each one provides their own metrics on their timers. Most metrics are the same (p50, p90, p95, etc)
56+
* but some other may be different.
5357
*
54-
* Alternative it could be used in production, but it is important to make sure that the JMX MBeans produced by each
58+
* If both Yammer and Dropwizard are used, it is important to make sure that the JMX MBeans produced by each
5559
* sub-registry are different. Otherwise the reported value is undetermined.
5660
*
5761
* In order to use this factory, you have to set the following properties in Pinot configuration:
@@ -144,6 +148,7 @@ public PinotMetricName makePinotMetricName(Class<?> klass, String name) {
144148
}
145149

146150
@Override
151+
@Deprecated
147152
public <T> PinotGauge<T> makePinotGauge(Function<Void, T> condition) {
148153
List<PinotGauge<T>> gauges = _factories.stream()
149154
.map(factory -> factory.makePinotGauge(condition))
@@ -152,6 +157,18 @@ public <T> PinotGauge<T> makePinotGauge(Function<Void, T> condition) {
152157
}
153158

154159
@Override
160+
public <T> PinotGauge<T> makePinotGauge(String metricName, Function<Void, T> condition) {
161+
List<PinotGauge<T>> gauges = _factories.stream()
162+
.map(factory -> factory.makePinotGauge(metricName, condition))
163+
.collect(Collectors.toList());
164+
return new CompoundPinotGauge<T>(gauges);
165+
}
166+
167+
/**
168+
* @deprecated Use {@link #makePinotMetricReporter(PinotMetricsRegistry)} instead.
169+
*/
170+
@Override
171+
@Deprecated
155172
public PinotJmxReporter makePinotJmxReporter(PinotMetricsRegistry metricsRegistry) {
156173
CompoundPinotMetricRegistry registry = (CompoundPinotMetricRegistry) metricsRegistry;
157174
List<PinotMetricsRegistry> subRegistries = registry.getRegistries();
@@ -164,12 +181,30 @@ public PinotJmxReporter makePinotJmxReporter(PinotMetricsRegistry metricsRegistr
164181
PinotMetricsFactory subFactory = _factories.get(i);
165182
PinotMetricsRegistry subRegistry = subRegistries.get(i);
166183

167-
subJmx.add(subFactory.makePinotJmxReporter(subRegistry));
184+
subJmx.add(subFactory.makePinotMetricReporter(subRegistry));
168185
}
169186

170187
return new CompoundPinotJmxReporter(subJmx);
171188
}
172189

190+
public PinotMetricReporter makePinotMetricReporter(PinotMetricsRegistry metricsRegistry) {
191+
CompoundPinotMetricRegistry registry = (CompoundPinotMetricRegistry) metricsRegistry;
192+
List<PinotMetricsRegistry> subRegistries = registry.getRegistries();
193+
Preconditions.checkState(subRegistries.size() == _factories.size(),
194+
"Number of registries ({}) should be the same than the number of factories ({})",
195+
subRegistries.size(), _factories.size());
196+
197+
ArrayList<PinotMetricReporter> subReporter = new ArrayList<>(_factories.size());
198+
for (int i = 0; i < _factories.size(); i++) {
199+
PinotMetricsFactory subFactory = _factories.get(i);
200+
PinotMetricsRegistry subRegistry = subRegistries.get(i);
201+
202+
subReporter.add(subFactory.makePinotMetricReporter(subRegistry));
203+
}
204+
205+
return new CompoundPinotMetricReporter(subReporter);
206+
}
207+
173208
@Override
174209
public String getMetricsFactoryName() {
175210
return "Compound";

‎pinot-plugins/pinot-metrics/pinot-dropwizard/src/main/java/org/apache/pinot/plugin/metrics/dropwizard/DropwizardJmxReporter.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020

2121
import com.codahale.metrics.MetricRegistry;
2222
import com.codahale.metrics.jmx.JmxReporter;
23-
import org.apache.pinot.spi.metrics.PinotJmxReporter;
23+
import org.apache.pinot.spi.metrics.PinotMetricReporter;
2424
import org.apache.pinot.spi.metrics.PinotMetricsRegistry;
2525

26-
27-
public class DropwizardJmxReporter implements PinotJmxReporter {
26+
/**
27+
* DropwizardJmxReporter is a metric reporter that exposes metrics to JMX using Dropwizard's JmxReporter.
28+
*/
29+
public class DropwizardJmxReporter implements PinotMetricReporter {
2830
private final JmxReporter _jmxReporter;
2931

3032
public DropwizardJmxReporter(PinotMetricsRegistry metricsRegistry, String domainName) {

‎pinot-plugins/pinot-metrics/pinot-dropwizard/src/main/java/org/apache/pinot/plugin/metrics/dropwizard/DropwizardMetricsFactory.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.pinot.spi.metrics.PinotGauge;
2727
import org.apache.pinot.spi.metrics.PinotJmxReporter;
2828
import org.apache.pinot.spi.metrics.PinotMetricName;
29+
import org.apache.pinot.spi.metrics.PinotMetricReporter;
2930
import org.apache.pinot.spi.metrics.PinotMetricsRegistry;
3031

3132

@@ -57,15 +58,28 @@ public PinotMetricName makePinotMetricName(Class<?> klass, String name) {
5758
}
5859

5960
@Override
61+
@Deprecated
6062
public <T> PinotGauge<T> makePinotGauge(Function<Void, T> condition) {
6163
return new DropwizardGauge<T>(condition);
6264
}
6365

66+
@Override
67+
public <T> PinotGauge<T> makePinotGauge(String metricName, Function<Void, T> condition) {
68+
return new DropwizardGauge<T>(condition);
69+
}
70+
71+
/**
72+
* @deprecated Use {@link #makePinotMetricReporter(PinotMetricsRegistry)} instead.
73+
*/
6474
@Override
6575
public PinotJmxReporter makePinotJmxReporter(PinotMetricsRegistry metricsRegistry) {
6676
return new DropwizardJmxReporter(metricsRegistry, _domainName);
6777
}
6878

79+
public PinotMetricReporter makePinotMetricReporter(PinotMetricsRegistry metricsRegistry) {
80+
return new DropwizardJmxReporter(metricsRegistry, _domainName);
81+
}
82+
6983
@Override
7084
public String getMetricsFactoryName() {
7185
return "Dropwizard";

‎pinot-plugins/pinot-metrics/pinot-open-telemetry/pom.xml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
4+
Licensed to the Apache Software Foundation (ASF) under one
5+
or more contributor license agreements. See the NOTICE file
6+
distributed with this work for additional information
7+
regarding copyright ownership. The ASF licenses this file
8+
to you under the Apache License, Version 2.0 (the
9+
"License"); you may not use this file except in compliance
10+
with the License. You may obtain a copy of the License at
11+
12+
http://www.apache.org/licenses/LICENSE-2.0
13+
14+
Unless required by applicable law or agreed to in writing,
15+
software distributed under the License is distributed on an
16+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
KIND, either express or implied. See the License for the
18+
specific language governing permissions and limitations
19+
under the License.
20+
21+
-->
22+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<artifactId>pinot-metrics</artifactId>
26+
<groupId>org.apache.pinot</groupId>
27+
<version>1.4.0-SNAPSHOT</version>
28+
</parent>
29+
30+
<artifactId>pinot-open-telemetry</artifactId>
31+
<name>Pinot OpenTelemetry Metrics</name>
32+
<url>https://pinot.apache.org/</url>
33+
<properties>
34+
<pinot.root>${basedir}/../../..</pinot.root>
35+
<shade.phase.prop>package</shade.phase.prop>
36+
</properties>
37+
38+
<dependencies>
39+
<dependency>
40+
<groupId>com.google.auto.service</groupId>
41+
<artifactId>auto-service-annotations</artifactId>
42+
</dependency>
43+
<dependency>
44+
<groupId>io.opentelemetry</groupId>
45+
<artifactId>opentelemetry-api</artifactId>
46+
</dependency>
47+
<dependency>
48+
<groupId>io.opentelemetry</groupId>
49+
<artifactId>opentelemetry-sdk</artifactId>
50+
</dependency>
51+
<dependency>
52+
<groupId>io.opentelemetry</groupId>
53+
<artifactId>opentelemetry-exporter-common</artifactId>
54+
</dependency>
55+
<dependency>
56+
<groupId>io.opentelemetry</groupId>
57+
<artifactId>opentelemetry-exporter-logging</artifactId>
58+
</dependency>
59+
<dependency>
60+
<groupId>io.opentelemetry</groupId>
61+
<artifactId>opentelemetry-exporter-otlp</artifactId>
62+
</dependency>
63+
<dependency>
64+
<groupId>org.apache.pinot</groupId>
65+
<artifactId>pinot-common</artifactId>
66+
</dependency>
67+
</dependencies>
68+
69+
<profiles>
70+
<profile>
71+
<id>pinot-fastdev</id>
72+
<properties>
73+
<shade.phase.prop>none</shade.phase.prop>
74+
</properties>
75+
</profile>
76+
</profiles>
77+
78+
</project>

0 commit comments

Comments
 (0)