Source code

001package io.prometheus.jmx;
002
003import io.prometheus.client.Collector;
004import io.prometheus.client.Counter;
005import org.yaml.snakeyaml.Yaml;
006
007import java.io.File;
008import java.io.FileReader;
009import java.io.IOException;
010import java.io.InputStream;
011import java.io.PrintWriter;
012import java.io.StringWriter;
013import java.util.ArrayList;
014import java.util.HashMap;
015import java.util.Iterator;
016import java.util.LinkedHashMap;
017import java.util.LinkedList;
018import java.util.List;
019import java.util.Map;
020import java.util.Set;
021import java.util.TreeMap;
022import java.util.logging.Logger;
023import java.util.regex.Matcher;
024import java.util.regex.Pattern;
025import javax.management.MalformedObjectNameException;
026import javax.management.ObjectName;
027
028import static java.lang.String.format;
029
030public class JmxCollector extends Collector implements Collector.Describable {
031    static final Counter configReloadSuccess = Counter.build()
032      .name("jmx_config_reload_success_total")
033      .help("Number of times configuration have successfully been reloaded.").register();
034
035    static final Counter configReloadFailure = Counter.build()
036      .name("jmx_config_reload_failure_total")
037      .help("Number of times configuration have failed to be reloaded.").register();
038
039    private static final Logger LOGGER = Logger.getLogger(JmxCollector.class.getName());
040
041    static class Rule {
042      Pattern pattern;
043      String name;
044      String value;
045      Double valueFactor = 1.0;
046      String help;
047      boolean attrNameSnakeCase;
048      boolean cache = false;
049      Type type = Type.UNKNOWN;
050      ArrayList<String> labelNames;
051      ArrayList<String> labelValues;
052    }
053
054    private static class Config {
055      Integer startDelaySeconds = 0;
056      String jmxUrl = "";
057      String username = "";
058      String password = "";
059      boolean ssl = false;
060      boolean lowercaseOutputName;
061      boolean lowercaseOutputLabelNames;
062      List<ObjectName> whitelistObjectNames = new ArrayList<ObjectName>();
063      List<ObjectName> blacklistObjectNames = new ArrayList<ObjectName>();
064      List<Rule> rules = new ArrayList<Rule>();
065      long lastUpdate = 0L;
066
067      MatchedRulesCache rulesCache;
068    }
069
070    private Config config;
071    private File configFile;
072    private long createTimeNanoSecs = System.nanoTime();
073
074    private final JmxMBeanPropertyCache jmxMBeanPropertyCache = new JmxMBeanPropertyCache();
075
076    public JmxCollector(File in) throws IOException, MalformedObjectNameException {
077        configFile = in;
078        config = loadConfig((Map<String, Object>)new Yaml().load(new FileReader(in)));
079        config.lastUpdate = configFile.lastModified();
080    }
081
082    public JmxCollector(String yamlConfig) throws MalformedObjectNameException {
083      config = loadConfig((Map<String, Object>)new Yaml().load(yamlConfig));
084    }
085
086    public JmxCollector(InputStream inputStream) throws MalformedObjectNameException {
087      config = loadConfig((Map<String, Object>)new Yaml().load(inputStream));
088    }
089
090    private void reloadConfig() {
091      try {
092        FileReader fr = new FileReader(configFile);
093
094        try {
095          Map<String, Object> newYamlConfig = (Map<String, Object>)new Yaml().load(fr);
096          config = loadConfig(newYamlConfig);
097          config.lastUpdate = configFile.lastModified();
098          configReloadSuccess.inc();
099        } catch (Exception e) {
100          LOGGER.severe("Configuration reload failed: " + e.toString());
101          configReloadFailure.inc();
102        } finally {
103          fr.close();
104        }
105
106      } catch (IOException e) {
107        LOGGER.severe("Configuration reload failed: " + e.toString());
108        configReloadFailure.inc();
109      }
110    }
111
112    private synchronized Config getLatestConfig() {
113      if (configFile != null) {
114          long mtime = configFile.lastModified();
115          if (mtime > config.lastUpdate) {
116            LOGGER.fine("Configuration file changed, reloading...");
117            reloadConfig();
118          }
119      }
120
121      return config;
122    }
123
124  private Config loadConfig(Map<String, Object> yamlConfig) throws MalformedObjectNameException {
125        Config cfg = new Config();
126
127        if (yamlConfig == null) {  // Yaml config empty, set config to empty map.
128          yamlConfig = new HashMap<String, Object>();
129        }
130
131        if (yamlConfig.containsKey("startDelaySeconds")) {
132          try {
133            cfg.startDelaySeconds = (Integer) yamlConfig.get("startDelaySeconds");
134          } catch (NumberFormatException e) {
135            throw new IllegalArgumentException("Invalid number provided for startDelaySeconds", e);
136          }
137        }
138        if (yamlConfig.containsKey("hostPort")) {
139          if (yamlConfig.containsKey("jmxUrl")) {
140            throw new IllegalArgumentException("At most one of hostPort and jmxUrl must be provided");
141          }
142          cfg.jmxUrl ="service:jmx:rmi:///jndi/rmi://" + (String)yamlConfig.get("hostPort") + "/jmxrmi";
143        } else if (yamlConfig.containsKey("jmxUrl")) {
144          cfg.jmxUrl = (String)yamlConfig.get("jmxUrl");
145        }
146
147        if (yamlConfig.containsKey("username")) {
148          cfg.username = (String)yamlConfig.get("username");
149        }
150
151        if (yamlConfig.containsKey("password")) {
152          cfg.password = (String)yamlConfig.get("password");
153        }
154
155        if (yamlConfig.containsKey("ssl")) {
156          cfg.ssl = (Boolean)yamlConfig.get("ssl");
157        }
158
159        if (yamlConfig.containsKey("lowercaseOutputName")) {
160          cfg.lowercaseOutputName = (Boolean)yamlConfig.get("lowercaseOutputName");
161        }
162
163        if (yamlConfig.containsKey("lowercaseOutputLabelNames")) {
164          cfg.lowercaseOutputLabelNames = (Boolean)yamlConfig.get("lowercaseOutputLabelNames");
165        }
166
167        if (yamlConfig.containsKey("whitelistObjectNames")) {
168          List<Object> names = (List<Object>) yamlConfig.get("whitelistObjectNames");
169          for(Object name : names) {
170            cfg.whitelistObjectNames.add(new ObjectName((String)name));
171          }
172        } else {
173          cfg.whitelistObjectNames.add(null);
174        }
175
176        if (yamlConfig.containsKey("blacklistObjectNames")) {
177          List<Object> names = (List<Object>) yamlConfig.get("blacklistObjectNames");
178          for (Object name : names) {
179            cfg.blacklistObjectNames.add(new ObjectName((String)name));
180          }
181        }
182
183      if (yamlConfig.containsKey("rules")) {
184          List<Map<String,Object>> configRules = (List<Map<String,Object>>) yamlConfig.get("rules");
185          for (Map<String, Object> ruleObject : configRules) {
186            Map<String, Object> yamlRule = ruleObject;
187            Rule rule = new Rule();
188            cfg.rules.add(rule);
189            if (yamlRule.containsKey("pattern")) {
190              rule.pattern = Pattern.compile("^.*(?:" + (String)yamlRule.get("pattern") + ").*$");
191            }
192            if (yamlRule.containsKey("name")) {
193              rule.name = (String)yamlRule.get("name");
194            }
195            if (yamlRule.containsKey("value")) {
196              rule.value = String.valueOf(yamlRule.get("value"));
197            }
198            if (yamlRule.containsKey("valueFactor")) {
199              String valueFactor = String.valueOf(yamlRule.get("valueFactor"));
200              try {
201                rule.valueFactor = Double.valueOf(valueFactor);
202              } catch (NumberFormatException e) {
203                // use default value
204              }
205            }
206            if (yamlRule.containsKey("attrNameSnakeCase")) {
207              rule.attrNameSnakeCase = (Boolean)yamlRule.get("attrNameSnakeCase");
208            }
209            if (yamlRule.containsKey("cache")) {
210              rule.cache = (Boolean)yamlRule.get("cache");
211            }
212            if (yamlRule.containsKey("type")) {
213              String t = (String)yamlRule.get("type");
214              // Gracefully handle switch to OM data model.
215              if ("UNTYPED".equals(t)) {
216                t = "UNKNOWN";
217              }
218              rule.type = Type.valueOf(t);
219            }
220            if (yamlRule.containsKey("help")) {
221              rule.help = (String)yamlRule.get("help");
222            }
223            if (yamlRule.containsKey("labels")) {
224              TreeMap labels = new TreeMap((Map<String, Object>)yamlRule.get("labels"));
225              rule.labelNames = new ArrayList<String>();
226              rule.labelValues = new ArrayList<String>();
227              for (Map.Entry<String, Object> entry : (Set<Map.Entry<String, Object>>)labels.entrySet()) {
228                rule.labelNames.add(entry.getKey());
229                rule.labelValues.add((String)entry.getValue());
230              }
231            }
232
233            // Validation.
234            if ((rule.labelNames != null || rule.help != null) && rule.name == null) {
235              throw new IllegalArgumentException("Must provide name, if help or labels are given: " + yamlRule);
236            }
237            if (rule.name != null && rule.pattern == null) {
238              throw new IllegalArgumentException("Must provide pattern, if name is given: " + yamlRule);
239            }
240          }
241        } else {
242          // Default to a single default rule.
243          cfg.rules.add(new Rule());
244        }
245
246        cfg.rulesCache = new MatchedRulesCache(cfg.rules);
247
248        return cfg;
249
250    }
251
252    static String toSnakeAndLowerCase(String attrName) {
253      if (attrName == null || attrName.isEmpty()) {
254        return attrName;
255      }
256      char firstChar = attrName.subSequence(0, 1).charAt(0);
257      boolean prevCharIsUpperCaseOrUnderscore = Character.isUpperCase(firstChar) || firstChar == '_';
258      StringBuilder resultBuilder = new StringBuilder(attrName.length()).append(Character.toLowerCase(firstChar));
259      for (char attrChar : attrName.substring(1).toCharArray()) {
260        boolean charIsUpperCase = Character.isUpperCase(attrChar);
261        if (!prevCharIsUpperCaseOrUnderscore && charIsUpperCase) {
262          resultBuilder.append("_");
263        }
264        resultBuilder.append(Character.toLowerCase(attrChar));
265        prevCharIsUpperCaseOrUnderscore = charIsUpperCase || attrChar == '_';
266      }
267      return resultBuilder.toString();
268    }
269
270  /**
271   * Change invalid chars to underscore, and merge underscores.
272   * @param name Input string
273   * @return
274   */
275  static String safeName(String name) {
276      if (name == null) {
277        return null;
278      }
279      boolean prevCharIsUnderscore = false;
280      StringBuilder safeNameBuilder = new StringBuilder(name.length());
281      if (!name.isEmpty() && Character.isDigit(name.charAt(0))) {
282        // prevent a numeric prefix.
283        safeNameBuilder.append("_");
284      }
285      for (char nameChar : name.toCharArray()) {
286        boolean isUnsafeChar = !JmxCollector.isLegalCharacter(nameChar);
287        if ((isUnsafeChar || nameChar == '_')) {
288          if (prevCharIsUnderscore) {
289            continue;
290          } else {
291            safeNameBuilder.append("_");
292            prevCharIsUnderscore = true;
293          }
294        } else {
295          safeNameBuilder.append(nameChar);
296          prevCharIsUnderscore = false;
297        }
298      }
299
300      return safeNameBuilder.toString();
301    }
302
303  private static boolean isLegalCharacter(char input) {
304    return ((input == ':') ||
305            (input == '_') ||
306            (input >= 'a' && input <= 'z') ||
307            (input >= 'A' && input <= 'Z') ||
308            (input >= '0' && input <= '9'));
309  }
310
311    class Receiver implements JmxScraper.MBeanReceiver {
312      Map<String, MetricFamilySamples> metricFamilySamplesMap =
313        new HashMap<String, MetricFamilySamples>();
314
315      Config config;
316      MatchedRulesCache.StalenessTracker stalenessTracker;
317
318      private static final char SEP = '_';
319
320      Receiver(Config config, MatchedRulesCache.StalenessTracker stalenessTracker) {
321        this.config = config;
322        this.stalenessTracker = stalenessTracker;
323      }
324
325      // [] and () are special in regexes, so swtich to <>.
326      private String angleBrackets(String s) {
327        return "<" + s.substring(1, s.length() - 1) + ">";
328      }
329
330      void addSample(MetricFamilySamples.Sample sample, Type type, String help) {
331        MetricFamilySamples mfs = metricFamilySamplesMap.get(sample.name);
332        if (mfs == null) {
333          // JmxScraper.MBeanReceiver is only called from one thread,
334          // so there's no race here.
335          mfs = new MetricFamilySamples(sample.name, type, help, new ArrayList<MetricFamilySamples.Sample>());
336          metricFamilySamplesMap.put(sample.name, mfs);
337        }
338        mfs.samples.add(sample);
339      }
340
341      // Add the matched rule to the cached rules and tag it as not stale
342      // if the rule is configured to be cached
343      private void addToCache(final Rule rule, final String cacheKey, final MatchedRule matchedRule) {
344        if (rule.cache) {
345          config.rulesCache.put(rule, cacheKey, matchedRule);
346          stalenessTracker.add(rule, cacheKey);
347        }
348      }
349
350      private MatchedRule defaultExport(
351          String matchName,
352          String domain,
353          LinkedHashMap<String, String> beanProperties,
354          LinkedList<String> attrKeys,
355          String attrName,
356          String help,
357          Double value,
358          double valueFactor,
359          Type type) {
360        StringBuilder name = new StringBuilder();
361        name.append(domain);
362        if (beanProperties.size() > 0) {
363            name.append(SEP);
364            name.append(beanProperties.values().iterator().next());
365        }
366        for (String k : attrKeys) {
367            name.append(SEP);
368            name.append(k);
369        }
370        name.append(SEP);
371        name.append(attrName);
372        String fullname = safeName(name.toString());
373
374        if (config.lowercaseOutputName) {
375          fullname = fullname.toLowerCase();
376        }
377
378        List<String> labelNames = new ArrayList<String>();
379        List<String> labelValues = new ArrayList<String>();
380        if (beanProperties.size() > 1) {
381            Iterator<Map.Entry<String, String>> iter = beanProperties.entrySet().iterator();
382            // Skip the first one, it's been used in the name.
383            iter.next();
384            while (iter.hasNext()) {
385              Map.Entry<String, String> entry = iter.next();
386              String labelName = safeName(entry.getKey());
387              if (config.lowercaseOutputLabelNames) {
388                labelName = labelName.toLowerCase();
389              }
390              labelNames.add(labelName);
391              labelValues.add(entry.getValue());
392            }
393        }
394
395        return new MatchedRule(fullname, matchName, type, help, labelNames, labelValues, value, valueFactor);
396      }
397
398      public void recordBean(
399          String domain,
400          LinkedHashMap<String, String> beanProperties,
401          LinkedList<String> attrKeys,
402          String attrName,
403          String attrType,
404          String attrDescription,
405          Object beanValue) {
406
407        String beanName = domain + angleBrackets(beanProperties.toString()) + angleBrackets(attrKeys.toString());
408        // attrDescription tends not to be useful, so give the fully qualified name too.
409        String help = attrDescription + " (" + beanName + attrName + ")";
410        String attrNameSnakeCase = toSnakeAndLowerCase(attrName);
411
412        MatchedRule matchedRule = MatchedRule.unmatched();
413
414        for (Rule rule : config.rules) {
415          // Rules with bean values cannot be properly cached (only the value from the first scrape will be cached).
416          // If caching for the rule is enabled, replace the value with a dummy <cache> to avoid caching different values at different times.
417          Object matchBeanValue = rule.cache ? "<cache>" : beanValue;
418
419          String matchName = beanName + (rule.attrNameSnakeCase ? attrNameSnakeCase : attrName) + ": " + matchBeanValue;
420
421          if (rule.cache) {
422            MatchedRule cachedRule = config.rulesCache.get(rule, matchName);
423            if (cachedRule != null) {
424              stalenessTracker.add(rule, matchName);
425              if (cachedRule.isMatched()) {
426                matchedRule = cachedRule;
427                break;
428              }
429
430              // The bean was cached earlier, but did not match the current rule.
431              // Skip it to avoid matching against the same pattern again
432              continue;
433            }
434          }
435
436          Matcher matcher = null;
437          if (rule.pattern != null) {
438            matcher = rule.pattern.matcher(matchName);
439            if (!matcher.matches()) {
440              addToCache(rule, matchName, MatchedRule.unmatched());
441              continue;
442            }
443          }
444
445          Double value = null;
446          if (rule.value != null && !rule.value.isEmpty()) {
447            String val = matcher.replaceAll(rule.value);
448            try {
449              value = Double.valueOf(val);
450            } catch (NumberFormatException e) {
451              LOGGER.fine("Unable to parse configured value '" + val + "' to number for bean: " + beanName + attrName + ": " + beanValue);
452              return;
453            }
454          }
455
456          // If there's no name provided, use default export format.
457          if (rule.name == null) {
458            matchedRule = defaultExport(matchName, domain, beanProperties, attrKeys, rule.attrNameSnakeCase ? attrNameSnakeCase : attrName, help, value, rule.valueFactor, rule.type);
459            addToCache(rule, matchName, matchedRule);
460            break;
461          }
462
463          // Matcher is set below here due to validation in the constructor.
464          String name = safeName(matcher.replaceAll(rule.name));
465          if (name.isEmpty()) {
466            return;
467          }
468          if (config.lowercaseOutputName) {
469            name = name.toLowerCase();
470          }
471
472          // Set the help.
473          if (rule.help != null) {
474            help = matcher.replaceAll(rule.help);
475          }
476
477          // Set the labels.
478          ArrayList<String> labelNames = new ArrayList<String>();
479          ArrayList<String> labelValues = new ArrayList<String>();
480          if (rule.labelNames != null) {
481            for (int i = 0; i < rule.labelNames.size(); i++) {
482              final String unsafeLabelName = rule.labelNames.get(i);
483              final String labelValReplacement = rule.labelValues.get(i);
484              try {
485                String labelName = safeName(matcher.replaceAll(unsafeLabelName));
486                String labelValue = matcher.replaceAll(labelValReplacement);
487                if (config.lowercaseOutputLabelNames) {
488                  labelName = labelName.toLowerCase();
489                }
490                if (!labelName.isEmpty() && !labelValue.isEmpty()) {
491                  labelNames.add(labelName);
492                  labelValues.add(labelValue);
493                }
494              } catch (Exception e) {
495                throw new RuntimeException(
496                        format("Matcher '%s' unable to use: '%s' value: '%s'", matcher, unsafeLabelName, labelValReplacement), e);
497              }
498            }
499          }
500
501          matchedRule = new MatchedRule(name, matchName, rule.type, help, labelNames, labelValues, value, rule.valueFactor);
502          addToCache(rule, matchName, matchedRule);
503          break;
504        }
505
506        if (matchedRule.isUnmatched()) {
507          return;
508        }
509
510        Number value;
511        if (matchedRule.value != null) {
512          beanValue = matchedRule.value;
513        }
514
515        if (beanValue instanceof Number) {
516          value = ((Number) beanValue).doubleValue() * matchedRule.valueFactor;
517        } else if (beanValue instanceof Boolean) {
518          value = (Boolean) beanValue ? 1 : 0;
519        } else {
520          LOGGER.fine("Ignoring unsupported bean: " + beanName + attrName + ": " + beanValue);
521          return;
522        }
523
524        // Add to samples.
525        LOGGER.fine("add metric sample: " + matchedRule.name + " " + matchedRule.labelNames + " " + matchedRule.labelValues + " " + value.doubleValue());
526        addSample(new MetricFamilySamples.Sample(matchedRule.name, matchedRule.labelNames, matchedRule.labelValues, value.doubleValue()), matchedRule.type, matchedRule.help);
527      }
528
529    }
530
531  public List<MetricFamilySamples> collect() {
532      // Take a reference to the current config and collect with this one
533      // (to avoid race conditions in case another thread reloads the config in the meantime)
534      Config config = getLatestConfig();
535
536      MatchedRulesCache.StalenessTracker stalenessTracker = new MatchedRulesCache.StalenessTracker();
537      Receiver receiver = new Receiver(config, stalenessTracker);
538      JmxScraper scraper = new JmxScraper(config.jmxUrl, config.username, config.password, config.ssl,
539              config.whitelistObjectNames, config.blacklistObjectNames, receiver, jmxMBeanPropertyCache);
540      long start = System.nanoTime();
541      double error = 0;
542      if ((config.startDelaySeconds > 0) &&
543        ((start - createTimeNanoSecs) / 1000000000L < config.startDelaySeconds)) {
544        throw new IllegalStateException("JMXCollector waiting for startDelaySeconds");
545      }
546      try {
547        scraper.doScrape();
548      } catch (Exception e) {
549        error = 1;
550        StringWriter sw = new StringWriter();
551        e.printStackTrace(new PrintWriter(sw));
552        LOGGER.severe("JMX scrape failed: " + sw.toString());
553      }
554      config.rulesCache.evictStaleEntries(stalenessTracker);
555
556      List<MetricFamilySamples> mfsList = new ArrayList<MetricFamilySamples>();
557      mfsList.addAll(receiver.metricFamilySamplesMap.values());
558      List<MetricFamilySamples.Sample> samples = new ArrayList<MetricFamilySamples.Sample>();
559      samples.add(new MetricFamilySamples.Sample(
560          "jmx_scrape_duration_seconds", new ArrayList<String>(), new ArrayList<String>(), (System.nanoTime() - start) / 1.0E9));
561      mfsList.add(new MetricFamilySamples("jmx_scrape_duration_seconds", Type.GAUGE, "Time this JMX scrape took, in seconds.", samples));
562
563      samples = new ArrayList<MetricFamilySamples.Sample>();
564      samples.add(new MetricFamilySamples.Sample(
565          "jmx_scrape_error", new ArrayList<String>(), new ArrayList<String>(), error));
566      mfsList.add(new MetricFamilySamples("jmx_scrape_error", Type.GAUGE, "Non-zero if this scrape failed.", samples));
567      samples = new ArrayList<MetricFamilySamples.Sample>();
568      samples.add(new MetricFamilySamples.Sample(
569              "jmx_scrape_cached_beans", new ArrayList<String>(), new ArrayList<String>(), stalenessTracker.cachedCount()));
570      mfsList.add(new MetricFamilySamples("jmx_scrape_cached_beans", Type.GAUGE, "Number of beans with their matching rule cached", samples));
571      return mfsList;
572    }
573
574    public List<MetricFamilySamples> describe() {
575      List<MetricFamilySamples> sampleFamilies = new ArrayList<MetricFamilySamples>();
576      sampleFamilies.add(new MetricFamilySamples("jmx_scrape_duration_seconds", Type.GAUGE, "Time this JMX scrape took, in seconds.", new ArrayList<MetricFamilySamples.Sample>()));
577      sampleFamilies.add(new MetricFamilySamples("jmx_scrape_error", Type.GAUGE, "Non-zero if this scrape failed.", new ArrayList<MetricFamilySamples.Sample>()));
578      sampleFamilies.add(new MetricFamilySamples("jmx_scrape_cached_beans", Type.GAUGE, "Number of beans with their matching rule cached", new ArrayList<MetricFamilySamples.Sample>()));
579      return sampleFamilies;
580    }
581
582    /**
583     * Convenience function to run standalone.
584     */
585    public static void main(String[] args) throws Exception {
586      String hostPort = "";
587      if (args.length > 0) {
588        hostPort = args[0];
589      }
590      JmxCollector jc = new JmxCollector(("{"
591      + "`hostPort`: `" + hostPort + "`,"
592      + "}").replace('`', '"'));
593      for(MetricFamilySamples mfs : jc.collect()) {
594        System.out.println(mfs);
595      }
596    }
597}