001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred; 020 021 import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString; 022 import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString; 023 024 import java.io.DataInput; 025 import java.io.DataOutput; 026 import java.io.IOException; 027 import java.text.ParseException; 028 import java.util.Collection; 029 import java.util.Iterator; 030 031 import org.apache.commons.collections.IteratorUtils; 032 import org.apache.commons.logging.Log; 033 import org.apache.hadoop.classification.InterfaceAudience; 034 import org.apache.hadoop.classification.InterfaceStability; 035 import org.apache.hadoop.mapreduce.FileSystemCounter; 036 import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup; 037 import org.apache.hadoop.mapreduce.counters.AbstractCounters; 038 import org.apache.hadoop.mapreduce.counters.CounterGroupBase; 039 import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; 040 import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; 041 import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; 042 import org.apache.hadoop.mapreduce.counters.GenericCounter; 043 import org.apache.hadoop.mapreduce.counters.Limits; 044 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; 045 import org.apache.hadoop.mapreduce.util.CountersStrings; 046 047 import com.google.common.collect.Iterators; 048 049 /** 050 * A set of named counters. 051 * 052 * <p><code>Counters</code> represent global counters, defined either by the 053 * Map-Reduce framework or applications. Each <code>Counter</code> can be of 054 * any {@link Enum} type.</p> 055 * 056 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of 057 * counters from a particular <code>Enum</code> class. 058 */ 059 @InterfaceAudience.Public 060 @InterfaceStability.Stable 061 public class Counters 062 extends AbstractCounters<Counters.Counter, Counters.Group> { 063 064 public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX; 065 066 public Counters() { 067 super(groupFactory); 068 } 069 070 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) { 071 super(newCounters, groupFactory); 072 } 073 074 /** 075 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters 076 * @param newCounters new Counters 077 * @return old Counters instance corresponding to newCounters 078 */ 079 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) { 080 return new Counters(newCounters); 081 } 082 083 public synchronized Group getGroup(String groupName) { 084 return super.getGroup(groupName); 085 } 086 087 @SuppressWarnings("unchecked") 088 public synchronized Collection<String> getGroupNames() { 089 return IteratorUtils.toList(super.getGroupNames().iterator()); 090 } 091 092 public synchronized String makeCompactString() { 093 return CountersStrings.toEscapedCompactString(this); 094 } 095 096 /** 097 * A counter record, comprising its name and value. 098 */ 099 public static class Counter implements org.apache.hadoop.mapreduce.Counter { 100 org.apache.hadoop.mapreduce.Counter realCounter; 101 102 Counter(org.apache.hadoop.mapreduce.Counter counter) { 103 this.realCounter = counter; 104 } 105 106 public Counter() { 107 this(new GenericCounter()); 108 } 109 110 @SuppressWarnings("deprecation") 111 @Override 112 public void setDisplayName(String displayName) { 113 realCounter.setDisplayName(displayName); 114 } 115 116 @Override 117 public String getName() { 118 return realCounter.getName(); 119 } 120 121 @Override 122 public String getDisplayName() { 123 return realCounter.getDisplayName(); 124 } 125 126 @Override 127 public long getValue() { 128 return realCounter.getValue(); 129 } 130 131 @Override 132 public void setValue(long value) { 133 realCounter.setValue(value); 134 } 135 136 @Override 137 public void increment(long incr) { 138 realCounter.increment(incr); 139 } 140 141 @Override 142 public void write(DataOutput out) throws IOException { 143 realCounter.write(out); 144 } 145 146 @Override 147 public void readFields(DataInput in) throws IOException { 148 realCounter.readFields(in); 149 } 150 151 /** 152 * Returns the compact stringified version of the counter in the format 153 * [(actual-name)(display-name)(value)] 154 * @return the stringified result 155 */ 156 public String makeEscapedCompactString() { 157 return toEscapedCompactString(realCounter); 158 } 159 160 /** 161 * Checks for (content) equality of two (basic) counters 162 * @param counter to compare 163 * @return true if content equals 164 * @deprecated 165 */ 166 @Deprecated 167 public boolean contentEquals(Counter counter) { 168 return realCounter.equals(counter.getUnderlyingCounter()); 169 } 170 171 /** 172 * @return the value of the counter 173 */ 174 public long getCounter() { 175 return realCounter.getValue(); 176 } 177 178 @Override 179 public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() { 180 return realCounter; 181 } 182 183 @Override 184 public synchronized boolean equals(Object genericRight) { 185 if (genericRight instanceof Counter) { 186 synchronized (genericRight) { 187 Counter right = (Counter) genericRight; 188 return getName().equals(right.getName()) && 189 getDisplayName().equals(right.getDisplayName()) && 190 getValue() == right.getValue(); 191 } 192 } 193 return false; 194 } 195 196 @Override 197 public int hashCode() { 198 return realCounter.hashCode(); 199 } 200 } 201 202 203 /** 204 * <code>Group</code> of counters, comprising of counters from a particular 205 * counter {@link Enum} class. 206 * 207 * <p><code>Group</code>handles localization of the class name and the 208 * counter names.</p> 209 */ 210 public static class Group implements CounterGroupBase<Counter> { 211 private CounterGroupBase<Counter> realGroup; 212 213 Group(GenericGroup group) { 214 this.realGroup = group; 215 } 216 Group(FSGroupImpl group) { 217 this.realGroup = group; 218 } 219 220 @SuppressWarnings({ "unchecked", "rawtypes" }) 221 Group(FrameworkGroupImpl group) { 222 this.realGroup = group; 223 } 224 225 /** 226 * @param counterName the name of the counter 227 * @return the value of the specified counter, or 0 if the counter does 228 * not exist. 229 */ 230 public long getCounter(String counterName) { 231 return getCounterValue(realGroup, counterName); 232 } 233 234 /** 235 * @return the compact stringified version of the group in the format 236 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings 237 * for the counters within. 238 */ 239 public String makeEscapedCompactString() { 240 return toEscapedCompactString(realGroup); 241 } 242 243 /** 244 * Get the counter for the given id and create it if it doesn't exist. 245 * @param id the numeric id of the counter within the group 246 * @param name the internal counter name 247 * @return the counter 248 * @deprecated use {@link #findCounter(String)} instead 249 */ 250 @Deprecated 251 public Counter getCounter(int id, String name) { 252 return findCounter(name); 253 } 254 255 /** 256 * Get the counter for the given name and create it if it doesn't exist. 257 * @param name the internal counter name 258 * @return the counter 259 */ 260 public Counter getCounterForName(String name) { 261 return findCounter(name); 262 } 263 264 @Override 265 public void write(DataOutput out) throws IOException { 266 realGroup.write(out); 267 } 268 269 @Override 270 public void readFields(DataInput in) throws IOException { 271 realGroup.readFields(in); 272 } 273 274 @Override 275 public Iterator<Counter> iterator() { 276 return realGroup.iterator(); 277 } 278 279 @Override 280 public String getName() { 281 return realGroup.getName(); 282 } 283 284 @Override 285 public String getDisplayName() { 286 return realGroup.getDisplayName(); 287 } 288 289 @Override 290 public void setDisplayName(String displayName) { 291 realGroup.setDisplayName(displayName); 292 } 293 294 @Override 295 public void addCounter(Counter counter) { 296 realGroup.addCounter(counter); 297 } 298 299 @Override 300 public Counter addCounter(String name, String displayName, long value) { 301 return realGroup.addCounter(name, displayName, value); 302 } 303 304 @Override 305 public Counter findCounter(String counterName, String displayName) { 306 return realGroup.findCounter(counterName, displayName); 307 } 308 309 @Override 310 public Counter findCounter(String counterName, boolean create) { 311 return realGroup.findCounter(counterName, create); 312 } 313 314 @Override 315 public Counter findCounter(String counterName) { 316 return realGroup.findCounter(counterName); 317 } 318 319 @Override 320 public int size() { 321 return realGroup.size(); 322 } 323 324 @Override 325 public void incrAllCounters(CounterGroupBase<Counter> rightGroup) { 326 realGroup.incrAllCounters(rightGroup); 327 } 328 329 @Override 330 public CounterGroupBase<Counter> getUnderlyingGroup() { 331 return realGroup; 332 } 333 334 @Override 335 public synchronized boolean equals(Object genericRight) { 336 if (genericRight instanceof CounterGroupBase<?>) { 337 @SuppressWarnings("unchecked") 338 CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 339 genericRight).getUnderlyingGroup(); 340 return Iterators.elementsEqual(iterator(), right.iterator()); 341 } 342 return false; 343 } 344 345 @Override 346 public int hashCode() { 347 return realGroup.hashCode(); 348 } 349 } 350 351 // All the group impls need this for legacy group interface 352 static long getCounterValue(CounterGroupBase<Counter> group, String counterName) { 353 Counter counter = group.findCounter(counterName, false); 354 if (counter != null) return counter.getValue(); 355 return 0L; 356 } 357 358 // Mix the generic group implementation into the Group interface 359 private static class GenericGroup extends AbstractCounterGroup<Counter> { 360 361 GenericGroup(String name, String displayName, Limits limits) { 362 super(name, displayName, limits); 363 } 364 365 @Override 366 protected Counter newCounter(String counterName, String displayName, 367 long value) { 368 return new Counter(new GenericCounter(counterName, displayName, value)); 369 } 370 371 @Override 372 protected Counter newCounter() { 373 return new Counter(); 374 } 375 376 @Override 377 public CounterGroupBase<Counter> getUnderlyingGroup() { 378 return this; 379 } 380 } 381 382 // Mix the framework group implementation into the Group interface 383 private static class FrameworkGroupImpl<T extends Enum<T>> 384 extends FrameworkCounterGroup<T, Counter> { 385 386 // Mix the framework counter implementation into the Counter interface 387 class FrameworkCounterImpl extends FrameworkCounter { 388 FrameworkCounterImpl(T key) { 389 super(key); 390 } 391 392 } 393 394 FrameworkGroupImpl(Class<T> cls) { 395 super(cls); 396 } 397 398 @Override 399 protected Counter newCounter(T key) { 400 return new Counter(new FrameworkCounterImpl(key)); 401 } 402 403 @Override 404 public CounterGroupBase<Counter> getUnderlyingGroup() { 405 return this; 406 } 407 } 408 409 // Mix the file system counter group implementation into the Group interface 410 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> { 411 412 private class FSCounterImpl extends FSCounter { 413 414 FSCounterImpl(String scheme, FileSystemCounter key) { 415 super(scheme, key); 416 } 417 418 } 419 420 @Override 421 protected Counter newCounter(String scheme, FileSystemCounter key) { 422 return new Counter(new FSCounterImpl(scheme, key)); 423 } 424 425 @Override 426 public CounterGroupBase<Counter> getUnderlyingGroup() { 427 return this; 428 } 429 } 430 431 public synchronized Counter findCounter(String group, String name) { 432 if (name.equals("MAP_INPUT_BYTES")) { 433 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + 434 "Use FileInputFormatCounters as group name and " + 435 " BYTES_READ as counter name instead"); 436 return findCounter(FileInputFormatCounter.BYTES_READ); 437 } 438 return getGroup(group).getCounterForName(name); 439 } 440 441 /** 442 * Provide factory methods for counter group factory implementation. 443 * See also the GroupFactory in 444 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters} 445 */ 446 static class GroupFactory extends CounterGroupFactory<Counter, Group> { 447 448 @Override 449 protected <T extends Enum<T>> 450 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) { 451 return new FrameworkGroupFactory<Group>() { 452 @Override public Group newGroup(String name) { 453 return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package 454 } 455 }; 456 } 457 458 @Override 459 protected Group newGenericGroup(String name, String displayName, 460 Limits limits) { 461 return new Group(new GenericGroup(name, displayName, limits)); 462 } 463 464 @Override 465 protected Group newFileSystemGroup() { 466 return new Group(new FSGroupImpl()); 467 } 468 } 469 470 private static final GroupFactory groupFactory = new GroupFactory(); 471 472 /** 473 * Find a counter by using strings 474 * @param group the name of the group 475 * @param id the id of the counter within the group (0 to N-1) 476 * @param name the internal name of the counter 477 * @return the counter for that name 478 * @deprecated use {@link #findCounter(String, String)} instead 479 */ 480 @Deprecated 481 public Counter findCounter(String group, int id, String name) { 482 return findCounter(group, name); 483 } 484 485 /** 486 * Increments the specified counter by the specified amount, creating it if 487 * it didn't already exist. 488 * @param key identifies a counter 489 * @param amount amount by which counter is to be incremented 490 */ 491 public void incrCounter(Enum<?> key, long amount) { 492 findCounter(key).increment(amount); 493 } 494 495 /** 496 * Increments the specified counter by the specified amount, creating it if 497 * it didn't already exist. 498 * @param group the name of the group 499 * @param counter the internal name of the counter 500 * @param amount amount by which counter is to be incremented 501 */ 502 public void incrCounter(String group, String counter, long amount) { 503 findCounter(group, counter).increment(amount); 504 } 505 506 /** 507 * Returns current value of the specified counter, or 0 if the counter 508 * does not exist. 509 * @param key the counter enum to lookup 510 * @return the counter value or 0 if counter not found 511 */ 512 public synchronized long getCounter(Enum<?> key) { 513 return findCounter(key).getValue(); 514 } 515 516 /** 517 * Increments multiple counters by their amounts in another Counters 518 * instance. 519 * @param other the other Counters instance 520 */ 521 public synchronized void incrAllCounters(Counters other) { 522 for (Group otherGroup: other) { 523 Group group = getGroup(otherGroup.getName()); 524 group.setDisplayName(otherGroup.getDisplayName()); 525 for (Counter otherCounter : otherGroup) { 526 Counter counter = group.getCounterForName(otherCounter.getName()); 527 counter.setDisplayName(otherCounter.getDisplayName()); 528 counter.increment(otherCounter.getValue()); 529 } 530 } 531 } 532 533 /** 534 * @return the total number of counters 535 * @deprecated use {@link #countCounters()} instead 536 */ 537 public int size() { 538 return countCounters(); 539 } 540 541 /** 542 * Convenience method for computing the sum of two sets of counters. 543 * @param a the first counters 544 * @param b the second counters 545 * @return a new summed counters object 546 */ 547 public static Counters sum(Counters a, Counters b) { 548 Counters counters = new Counters(); 549 counters.incrAllCounters(a); 550 counters.incrAllCounters(b); 551 return counters; 552 } 553 554 /** 555 * Logs the current counter values. 556 * @param log The log to use. 557 */ 558 public void log(Log log) { 559 log.info("Counters: " + size()); 560 for(Group group: this) { 561 log.info(" " + group.getDisplayName()); 562 for (Counter counter: group) { 563 log.info(" " + counter.getDisplayName() + "=" + 564 counter.getCounter()); 565 } 566 } 567 } 568 569 /** 570 * Represent the counter in a textual format that can be converted back to 571 * its object form 572 * @return the string in the following format 573 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}* 574 */ 575 public String makeEscapedCompactString() { 576 return toEscapedCompactString(this); 577 } 578 579 /** 580 * Convert a stringified (by {@link #makeEscapedCompactString()} counter 581 * representation into a counter object. 582 * @param compactString to parse 583 * @return a new counters object 584 * @throws ParseException 585 */ 586 public static Counters fromEscapedCompactString(String compactString) 587 throws ParseException { 588 return parseEscapedCompactString(compactString, new Counters()); 589 } 590 }