001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred; 020 021 import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString; 022 import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString; 023 024 import java.io.DataInput; 025 import java.io.DataOutput; 026 import java.io.IOException; 027 import java.text.ParseException; 028 import java.util.Collection; 029 import java.util.Iterator; 030 031 import org.apache.commons.collections.IteratorUtils; 032 import org.apache.commons.logging.Log; 033 import org.apache.hadoop.classification.InterfaceAudience; 034 import org.apache.hadoop.classification.InterfaceStability; 035 import org.apache.hadoop.mapreduce.FileSystemCounter; 036 import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup; 037 import org.apache.hadoop.mapreduce.counters.AbstractCounters; 038 import org.apache.hadoop.mapreduce.counters.CounterGroupBase; 039 import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; 040 import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; 041 import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; 042 import org.apache.hadoop.mapreduce.counters.GenericCounter; 043 import org.apache.hadoop.mapreduce.counters.Limits; 044 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; 045 import org.apache.hadoop.mapreduce.util.CountersStrings; 046 047 import com.google.common.collect.Iterators; 048 049 /** 050 * A set of named counters. 051 * 052 * <p><code>Counters</code> represent global counters, defined either by the 053 * Map-Reduce framework or applications. Each <code>Counter</code> can be of 054 * any {@link Enum} type.</p> 055 * 056 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of 057 * counters from a particular <code>Enum</code> class. 058 */ 059 @InterfaceAudience.Public 060 @InterfaceStability.Stable 061 public class Counters 062 extends AbstractCounters<Counters.Counter, Counters.Group> { 063 064 public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX; 065 066 public Counters() { 067 super(groupFactory); 068 } 069 070 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) { 071 super(newCounters, groupFactory); 072 } 073 074 /** 075 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters 076 * @param newCounters new Counters 077 * @return old Counters instance corresponding to newCounters 078 */ 079 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) { 080 return new Counters(newCounters); 081 } 082 083 public synchronized Group getGroup(String groupName) { 084 return super.getGroup(groupName); 085 } 086 087 @SuppressWarnings("unchecked") 088 public synchronized Collection<String> getGroupNames() { 089 return IteratorUtils.toList(super.getGroupNames().iterator()); 090 } 091 092 public synchronized String makeCompactString() { 093 return CountersStrings.toEscapedCompactString(this); 094 } 095 096 /** 097 * A counter record, comprising its name and value. 098 */ 099 @InterfaceAudience.Public 100 @InterfaceStability.Stable 101 public static class Counter implements org.apache.hadoop.mapreduce.Counter { 102 org.apache.hadoop.mapreduce.Counter realCounter; 103 104 Counter(org.apache.hadoop.mapreduce.Counter counter) { 105 this.realCounter = counter; 106 } 107 108 public Counter() { 109 this(new GenericCounter()); 110 } 111 112 @SuppressWarnings("deprecation") 113 @Override 114 public void setDisplayName(String displayName) { 115 realCounter.setDisplayName(displayName); 116 } 117 118 @Override 119 public String getName() { 120 return realCounter.getName(); 121 } 122 123 @Override 124 public String getDisplayName() { 125 return realCounter.getDisplayName(); 126 } 127 128 @Override 129 public long getValue() { 130 return realCounter.getValue(); 131 } 132 133 @Override 134 public void setValue(long value) { 135 realCounter.setValue(value); 136 } 137 138 @Override 139 public void increment(long incr) { 140 realCounter.increment(incr); 141 } 142 143 @Override 144 public void write(DataOutput out) throws IOException { 145 realCounter.write(out); 146 } 147 148 @Override 149 public void readFields(DataInput in) throws IOException { 150 realCounter.readFields(in); 151 } 152 153 /** 154 * Returns the compact stringified version of the counter in the format 155 * [(actual-name)(display-name)(value)] 156 * @return the stringified result 157 */ 158 public String makeEscapedCompactString() { 159 return toEscapedCompactString(realCounter); 160 } 161 162 /** 163 * Checks for (content) equality of two (basic) counters 164 * @param counter to compare 165 * @return true if content equals 166 * @deprecated 167 */ 168 @Deprecated 169 public boolean contentEquals(Counter counter) { 170 return realCounter.equals(counter.getUnderlyingCounter()); 171 } 172 173 /** 174 * @return the value of the counter 175 */ 176 public long getCounter() { 177 return realCounter.getValue(); 178 } 179 180 @Override 181 public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() { 182 return realCounter; 183 } 184 185 @Override 186 public synchronized boolean equals(Object genericRight) { 187 if (genericRight instanceof Counter) { 188 synchronized (genericRight) { 189 Counter right = (Counter) genericRight; 190 return getName().equals(right.getName()) && 191 getDisplayName().equals(right.getDisplayName()) && 192 getValue() == right.getValue(); 193 } 194 } 195 return false; 196 } 197 198 @Override 199 public int hashCode() { 200 return realCounter.hashCode(); 201 } 202 } 203 204 205 /** 206 * <code>Group</code> of counters, comprising of counters from a particular 207 * counter {@link Enum} class. 208 * 209 * <p><code>Group</code>handles localization of the class name and the 210 * counter names.</p> 211 */ 212 @InterfaceAudience.Public 213 @InterfaceStability.Stable 214 public static class Group implements CounterGroupBase<Counter> { 215 private CounterGroupBase<Counter> realGroup; 216 217 Group(GenericGroup group) { 218 this.realGroup = group; 219 } 220 Group(FSGroupImpl group) { 221 this.realGroup = group; 222 } 223 224 @SuppressWarnings({ "unchecked", "rawtypes" }) 225 Group(FrameworkGroupImpl group) { 226 this.realGroup = group; 227 } 228 229 /** 230 * @param counterName the name of the counter 231 * @return the value of the specified counter, or 0 if the counter does 232 * not exist. 233 */ 234 public long getCounter(String counterName) { 235 return getCounterValue(realGroup, counterName); 236 } 237 238 /** 239 * @return the compact stringified version of the group in the format 240 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings 241 * for the counters within. 242 */ 243 public String makeEscapedCompactString() { 244 return toEscapedCompactString(realGroup); 245 } 246 247 /** 248 * Get the counter for the given id and create it if it doesn't exist. 249 * @param id the numeric id of the counter within the group 250 * @param name the internal counter name 251 * @return the counter 252 * @deprecated use {@link #findCounter(String)} instead 253 */ 254 @Deprecated 255 public Counter getCounter(int id, String name) { 256 return findCounter(name); 257 } 258 259 /** 260 * Get the counter for the given name and create it if it doesn't exist. 261 * @param name the internal counter name 262 * @return the counter 263 */ 264 public Counter getCounterForName(String name) { 265 return findCounter(name); 266 } 267 268 @Override 269 public void write(DataOutput out) throws IOException { 270 realGroup.write(out); 271 } 272 273 @Override 274 public void readFields(DataInput in) throws IOException { 275 realGroup.readFields(in); 276 } 277 278 @Override 279 public Iterator<Counter> iterator() { 280 return realGroup.iterator(); 281 } 282 283 @Override 284 public String getName() { 285 return realGroup.getName(); 286 } 287 288 @Override 289 public String getDisplayName() { 290 return realGroup.getDisplayName(); 291 } 292 293 @Override 294 public void setDisplayName(String displayName) { 295 realGroup.setDisplayName(displayName); 296 } 297 298 @Override 299 public void addCounter(Counter counter) { 300 realGroup.addCounter(counter); 301 } 302 303 @Override 304 public Counter addCounter(String name, String displayName, long value) { 305 return realGroup.addCounter(name, displayName, value); 306 } 307 308 @Override 309 public Counter findCounter(String counterName, String displayName) { 310 return realGroup.findCounter(counterName, displayName); 311 } 312 313 @Override 314 public Counter findCounter(String counterName, boolean create) { 315 return realGroup.findCounter(counterName, create); 316 } 317 318 @Override 319 public Counter findCounter(String counterName) { 320 return realGroup.findCounter(counterName); 321 } 322 323 @Override 324 public int size() { 325 return realGroup.size(); 326 } 327 328 @Override 329 public void incrAllCounters(CounterGroupBase<Counter> rightGroup) { 330 realGroup.incrAllCounters(rightGroup); 331 } 332 333 @Override 334 public CounterGroupBase<Counter> getUnderlyingGroup() { 335 return realGroup; 336 } 337 338 @Override 339 public synchronized boolean equals(Object genericRight) { 340 if (genericRight instanceof CounterGroupBase<?>) { 341 @SuppressWarnings("unchecked") 342 CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 343 genericRight).getUnderlyingGroup(); 344 return Iterators.elementsEqual(iterator(), right.iterator()); 345 } 346 return false; 347 } 348 349 @Override 350 public int hashCode() { 351 return realGroup.hashCode(); 352 } 353 } 354 355 // All the group impls need this for legacy group interface 356 static long getCounterValue(CounterGroupBase<Counter> group, String counterName) { 357 Counter counter = group.findCounter(counterName, false); 358 if (counter != null) return counter.getValue(); 359 return 0L; 360 } 361 362 // Mix the generic group implementation into the Group interface 363 private static class GenericGroup extends AbstractCounterGroup<Counter> { 364 365 GenericGroup(String name, String displayName, Limits limits) { 366 super(name, displayName, limits); 367 } 368 369 @Override 370 protected Counter newCounter(String counterName, String displayName, 371 long value) { 372 return new Counter(new GenericCounter(counterName, displayName, value)); 373 } 374 375 @Override 376 protected Counter newCounter() { 377 return new Counter(); 378 } 379 380 @Override 381 public CounterGroupBase<Counter> getUnderlyingGroup() { 382 return this; 383 } 384 } 385 386 // Mix the framework group implementation into the Group interface 387 private static class FrameworkGroupImpl<T extends Enum<T>> 388 extends FrameworkCounterGroup<T, Counter> { 389 390 FrameworkGroupImpl(Class<T> cls) { 391 super(cls); 392 } 393 394 @Override 395 protected Counter newCounter(T key) { 396 return new Counter(new FrameworkCounter<T>(key, getName())); 397 } 398 399 @Override 400 public CounterGroupBase<Counter> getUnderlyingGroup() { 401 return this; 402 } 403 } 404 405 // Mix the file system counter group implementation into the Group interface 406 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> { 407 408 @Override 409 protected Counter newCounter(String scheme, FileSystemCounter key) { 410 return new Counter(new FSCounter(scheme, key)); 411 } 412 413 @Override 414 public CounterGroupBase<Counter> getUnderlyingGroup() { 415 return this; 416 } 417 } 418 419 public synchronized Counter findCounter(String group, String name) { 420 if (name.equals("MAP_INPUT_BYTES")) { 421 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + 422 "Use FileInputFormatCounters as group name and " + 423 " BYTES_READ as counter name instead"); 424 return findCounter(FileInputFormatCounter.BYTES_READ); 425 } 426 return getGroup(group).getCounterForName(name); 427 } 428 429 /** 430 * Provide factory methods for counter group factory implementation. 431 * See also the GroupFactory in 432 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters} 433 */ 434 static class GroupFactory extends CounterGroupFactory<Counter, Group> { 435 436 @Override 437 protected <T extends Enum<T>> 438 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) { 439 return new FrameworkGroupFactory<Group>() { 440 @Override public Group newGroup(String name) { 441 return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package 442 } 443 }; 444 } 445 446 @Override 447 protected Group newGenericGroup(String name, String displayName, 448 Limits limits) { 449 return new Group(new GenericGroup(name, displayName, limits)); 450 } 451 452 @Override 453 protected Group newFileSystemGroup() { 454 return new Group(new FSGroupImpl()); 455 } 456 } 457 458 private static final GroupFactory groupFactory = new GroupFactory(); 459 460 /** 461 * Find a counter by using strings 462 * @param group the name of the group 463 * @param id the id of the counter within the group (0 to N-1) 464 * @param name the internal name of the counter 465 * @return the counter for that name 466 * @deprecated use {@link #findCounter(String, String)} instead 467 */ 468 @Deprecated 469 public Counter findCounter(String group, int id, String name) { 470 return findCounter(group, name); 471 } 472 473 /** 474 * Increments the specified counter by the specified amount, creating it if 475 * it didn't already exist. 476 * @param key identifies a counter 477 * @param amount amount by which counter is to be incremented 478 */ 479 public void incrCounter(Enum<?> key, long amount) { 480 findCounter(key).increment(amount); 481 } 482 483 /** 484 * Increments the specified counter by the specified amount, creating it if 485 * it didn't already exist. 486 * @param group the name of the group 487 * @param counter the internal name of the counter 488 * @param amount amount by which counter is to be incremented 489 */ 490 public void incrCounter(String group, String counter, long amount) { 491 findCounter(group, counter).increment(amount); 492 } 493 494 /** 495 * Returns current value of the specified counter, or 0 if the counter 496 * does not exist. 497 * @param key the counter enum to lookup 498 * @return the counter value or 0 if counter not found 499 */ 500 public synchronized long getCounter(Enum<?> key) { 501 return findCounter(key).getValue(); 502 } 503 504 /** 505 * Increments multiple counters by their amounts in another Counters 506 * instance. 507 * @param other the other Counters instance 508 */ 509 public synchronized void incrAllCounters(Counters other) { 510 for (Group otherGroup: other) { 511 Group group = getGroup(otherGroup.getName()); 512 group.setDisplayName(otherGroup.getDisplayName()); 513 for (Counter otherCounter : otherGroup) { 514 Counter counter = group.getCounterForName(otherCounter.getName()); 515 counter.setDisplayName(otherCounter.getDisplayName()); 516 counter.increment(otherCounter.getValue()); 517 } 518 } 519 } 520 521 /** 522 * @return the total number of counters 523 * @deprecated use {@link #countCounters()} instead 524 */ 525 public int size() { 526 return countCounters(); 527 } 528 529 /** 530 * Convenience method for computing the sum of two sets of counters. 531 * @param a the first counters 532 * @param b the second counters 533 * @return a new summed counters object 534 */ 535 public static Counters sum(Counters a, Counters b) { 536 Counters counters = new Counters(); 537 counters.incrAllCounters(a); 538 counters.incrAllCounters(b); 539 return counters; 540 } 541 542 /** 543 * Logs the current counter values. 544 * @param log The log to use. 545 */ 546 public void log(Log log) { 547 log.info("Counters: " + size()); 548 for(Group group: this) { 549 log.info(" " + group.getDisplayName()); 550 for (Counter counter: group) { 551 log.info(" " + counter.getDisplayName() + "=" + 552 counter.getCounter()); 553 } 554 } 555 } 556 557 /** 558 * Represent the counter in a textual format that can be converted back to 559 * its object form 560 * @return the string in the following format 561 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}* 562 */ 563 public String makeEscapedCompactString() { 564 return toEscapedCompactString(this); 565 } 566 567 /** 568 * Convert a stringified (by {@link #makeEscapedCompactString()} counter 569 * representation into a counter object. 570 * @param compactString to parse 571 * @return a new counters object 572 * @throws ParseException 573 */ 574 public static Counters fromEscapedCompactString(String compactString) 575 throws ParseException { 576 return parseEscapedCompactString(compactString, new Counters()); 577 } 578 }