001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapred;
020    
021    import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString;
022    import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString;
023    
024    import java.io.DataInput;
025    import java.io.DataOutput;
026    import java.io.IOException;
027    import java.text.ParseException;
028    import java.util.Collection;
029    import java.util.Iterator;
030    
031    import org.apache.commons.collections.IteratorUtils;
032    import org.apache.commons.logging.Log;
033    import org.apache.hadoop.classification.InterfaceAudience;
034    import org.apache.hadoop.classification.InterfaceStability;
035    import org.apache.hadoop.mapreduce.FileSystemCounter;
036    import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
037    import org.apache.hadoop.mapreduce.counters.AbstractCounters;
038    import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
039    import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
040    import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
041    import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
042    import org.apache.hadoop.mapreduce.counters.GenericCounter;
043    import org.apache.hadoop.mapreduce.counters.Limits;
044    import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
045    import org.apache.hadoop.mapreduce.util.CountersStrings;
046    
047    import com.google.common.collect.Iterators;
048    
049    /**
050     * A set of named counters.
051     *
052     * <p><code>Counters</code> represent global counters, defined either by the
053     * Map-Reduce framework or applications. Each <code>Counter</code> can be of
054     * any {@link Enum} type.</p>
055     *
056     * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
057     * counters from a particular <code>Enum</code> class.
058     */
059    @InterfaceAudience.Public
060    @InterfaceStability.Stable
061    public class Counters
062        extends AbstractCounters<Counters.Counter, Counters.Group> {
063      
064      public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
065      
066      public Counters() {
067        super(groupFactory);
068      }
069    
070      public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
071        super(newCounters, groupFactory);
072      }
073    
074      /**
075       * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
076       * @param newCounters new Counters
077       * @return old Counters instance corresponding to newCounters
078       */
079      static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
080        return new Counters(newCounters);
081      }
082    
083      public synchronized Group getGroup(String groupName) {
084        return super.getGroup(groupName);
085      }
086    
087      @SuppressWarnings("unchecked")
088      public synchronized Collection<String> getGroupNames() {
089        return IteratorUtils.toList(super.getGroupNames().iterator());
090      }
091    
092      public synchronized String makeCompactString() {
093        return CountersStrings.toEscapedCompactString(this);
094      }
095      
096      /**
097       * A counter record, comprising its name and value.
098       */
099      @InterfaceAudience.Public
100      @InterfaceStability.Stable
101      public static class Counter implements org.apache.hadoop.mapreduce.Counter {
102        org.apache.hadoop.mapreduce.Counter realCounter;
103    
104        Counter(org.apache.hadoop.mapreduce.Counter counter) {
105          this.realCounter = counter;
106        }
107    
108        public Counter() {
109          this(new GenericCounter());
110        }
111    
112        @SuppressWarnings("deprecation")
113        @Override
114        public void setDisplayName(String displayName) {
115          realCounter.setDisplayName(displayName);
116        }
117    
118        @Override
119        public String getName() {
120          return realCounter.getName();
121        }
122    
123        @Override
124        public String getDisplayName() {
125          return realCounter.getDisplayName();
126        }
127    
128        @Override
129        public long getValue() {
130          return realCounter.getValue();
131        }
132    
133        @Override
134        public void setValue(long value) {
135          realCounter.setValue(value);
136        }
137    
138        @Override
139        public void increment(long incr) {
140          realCounter.increment(incr);
141        }
142    
143        @Override
144        public void write(DataOutput out) throws IOException {
145          realCounter.write(out);
146        }
147    
148        @Override
149        public void readFields(DataInput in) throws IOException {
150          realCounter.readFields(in);
151        }
152    
153        /**
154         * Returns the compact stringified version of the counter in the format
155         * [(actual-name)(display-name)(value)]
156         * @return the stringified result
157         */
158        public String makeEscapedCompactString() {
159          return toEscapedCompactString(realCounter);
160        }
161    
162        /**
163         * Checks for (content) equality of two (basic) counters
164         * @param counter to compare
165         * @return true if content equals
166         * @deprecated
167         */
168        @Deprecated
169        public boolean contentEquals(Counter counter) {
170          return realCounter.equals(counter.getUnderlyingCounter());
171        }
172    
173        /**
174         * @return the value of the counter
175         */
176        public long getCounter() {
177          return realCounter.getValue();
178        }
179    
180        @Override
181        public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() {
182          return realCounter;
183        }
184        
185        @Override
186        public synchronized boolean equals(Object genericRight) {
187          if (genericRight instanceof Counter) {
188            synchronized (genericRight) {
189              Counter right = (Counter) genericRight;
190              return getName().equals(right.getName()) &&
191                     getDisplayName().equals(right.getDisplayName()) &&
192                     getValue() == right.getValue();
193            }
194          }
195          return false;
196        }
197        
198        @Override
199        public int hashCode() {
200          return realCounter.hashCode();
201        }
202      }
203    
204    
205      /**
206       *  <code>Group</code> of counters, comprising of counters from a particular
207       *  counter {@link Enum} class.
208       *
209       *  <p><code>Group</code>handles localization of the class name and the
210       *  counter names.</p>
211       */
212      @InterfaceAudience.Public
213      @InterfaceStability.Stable
214      public static class Group implements CounterGroupBase<Counter> {
215        private CounterGroupBase<Counter> realGroup;
216        
217        Group(GenericGroup group) {
218          this.realGroup = group;
219        }
220        Group(FSGroupImpl group) {
221          this.realGroup = group;
222        }
223        
224        @SuppressWarnings({ "unchecked", "rawtypes" })
225        Group(FrameworkGroupImpl group) {
226          this.realGroup = group;
227        }
228        
229        /**
230         * @param counterName the name of the counter
231         * @return the value of the specified counter, or 0 if the counter does
232         * not exist.
233         */
234        public long getCounter(String counterName)  {
235          return getCounterValue(realGroup, counterName);
236        }
237    
238        /**
239         * @return the compact stringified version of the group in the format
240         * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
241         * for the counters within.
242         */
243        public String makeEscapedCompactString() {
244          return toEscapedCompactString(realGroup);
245        }
246    
247        /**
248         * Get the counter for the given id and create it if it doesn't exist.
249         * @param id the numeric id of the counter within the group
250         * @param name the internal counter name
251         * @return the counter
252         * @deprecated use {@link #findCounter(String)} instead
253         */
254        @Deprecated
255        public Counter getCounter(int id, String name) {
256          return findCounter(name);
257        }
258    
259        /**
260         * Get the counter for the given name and create it if it doesn't exist.
261         * @param name the internal counter name
262         * @return the counter
263         */
264        public Counter getCounterForName(String name) {
265          return findCounter(name);
266        }
267    
268        @Override
269        public void write(DataOutput out) throws IOException {
270         realGroup.write(out); 
271        }
272    
273        @Override
274        public void readFields(DataInput in) throws IOException {
275          realGroup.readFields(in);
276        }
277    
278        @Override
279        public Iterator<Counter> iterator() {
280          return realGroup.iterator();
281        }
282    
283        @Override
284        public String getName() {
285          return realGroup.getName();
286        }
287    
288        @Override
289        public String getDisplayName() {
290          return realGroup.getDisplayName();
291        }
292    
293        @Override
294        public void setDisplayName(String displayName) {
295          realGroup.setDisplayName(displayName);
296        }
297    
298        @Override
299        public void addCounter(Counter counter) {
300          realGroup.addCounter(counter);
301        }
302    
303        @Override
304        public Counter addCounter(String name, String displayName, long value) {
305          return realGroup.addCounter(name, displayName, value);
306        }
307    
308        @Override
309        public Counter findCounter(String counterName, String displayName) {
310          return realGroup.findCounter(counterName, displayName);
311        }
312    
313        @Override
314        public Counter findCounter(String counterName, boolean create) {
315          return realGroup.findCounter(counterName, create);
316        }
317    
318        @Override
319        public Counter findCounter(String counterName) {
320          return realGroup.findCounter(counterName);
321        }
322    
323        @Override
324        public int size() {
325          return realGroup.size();
326        }
327    
328        @Override
329        public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
330          realGroup.incrAllCounters(rightGroup);
331        }
332        
333        @Override
334        public CounterGroupBase<Counter> getUnderlyingGroup() {
335          return realGroup;
336        }
337    
338        @Override
339        public synchronized boolean equals(Object genericRight) {
340          if (genericRight instanceof CounterGroupBase<?>) {
341            @SuppressWarnings("unchecked")
342            CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 
343            genericRight).getUnderlyingGroup();
344            return Iterators.elementsEqual(iterator(), right.iterator());
345          }
346          return false;
347        }
348    
349        @Override
350        public int hashCode() {
351          return realGroup.hashCode();
352        }
353      }
354    
355      // All the group impls need this for legacy group interface
356      static long getCounterValue(CounterGroupBase<Counter> group, String counterName) {
357        Counter counter = group.findCounter(counterName, false);
358        if (counter != null) return counter.getValue();
359        return 0L;
360      }
361    
362      // Mix the generic group implementation into the Group interface
363      private static class GenericGroup extends AbstractCounterGroup<Counter> {
364    
365        GenericGroup(String name, String displayName, Limits limits) {
366          super(name, displayName, limits);
367        }
368    
369        @Override
370        protected Counter newCounter(String counterName, String displayName,
371                                     long value) {
372          return new Counter(new GenericCounter(counterName, displayName, value));
373        }
374    
375        @Override
376        protected Counter newCounter() {
377          return new Counter();
378        }
379        
380        @Override
381        public CounterGroupBase<Counter> getUnderlyingGroup() {
382         return this;
383        }
384      }
385    
386      // Mix the framework group implementation into the Group interface
387      private static class FrameworkGroupImpl<T extends Enum<T>>
388          extends FrameworkCounterGroup<T, Counter> {
389    
390        FrameworkGroupImpl(Class<T> cls) {
391          super(cls);
392        }
393    
394        @Override
395        protected Counter newCounter(T key) {
396          return new Counter(new FrameworkCounter<T>(key, getName()));
397        }
398    
399        @Override
400        public CounterGroupBase<Counter> getUnderlyingGroup() {
401          return this;
402        }
403      }
404    
405      // Mix the file system counter group implementation into the Group interface
406      private static class FSGroupImpl extends FileSystemCounterGroup<Counter> {
407    
408        @Override
409        protected Counter newCounter(String scheme, FileSystemCounter key) {
410          return new Counter(new FSCounter(scheme, key));
411        }
412    
413        @Override
414        public CounterGroupBase<Counter> getUnderlyingGroup() {
415          return this;
416        }
417      }
418    
419      public synchronized Counter findCounter(String group, String name) {
420        if (name.equals("MAP_INPUT_BYTES")) {
421          LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
422                   "Use FileInputFormatCounters as group name and " +
423                   " BYTES_READ as counter name instead");
424          return findCounter(FileInputFormatCounter.BYTES_READ);
425        }
426        return getGroup(group).getCounterForName(name);
427      }
428    
429      /**
430       * Provide factory methods for counter group factory implementation.
431       * See also the GroupFactory in
432       *  {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
433       */
434      static class GroupFactory extends CounterGroupFactory<Counter, Group> {
435    
436        @Override
437        protected <T extends Enum<T>>
438        FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
439          return new FrameworkGroupFactory<Group>() {
440            @Override public Group newGroup(String name) {
441              return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package
442            }
443          };
444        }
445    
446        @Override
447        protected Group newGenericGroup(String name, String displayName,
448                                        Limits limits) {
449          return new Group(new GenericGroup(name, displayName, limits));
450        }
451    
452        @Override
453        protected Group newFileSystemGroup() {
454          return new Group(new FSGroupImpl());
455        }
456      }
457    
458      private static final GroupFactory groupFactory = new GroupFactory();
459    
460      /**
461       * Find a counter by using strings
462       * @param group the name of the group
463       * @param id the id of the counter within the group (0 to N-1)
464       * @param name the internal name of the counter
465       * @return the counter for that name
466       * @deprecated use {@link #findCounter(String, String)} instead
467       */
468      @Deprecated
469      public Counter findCounter(String group, int id, String name) {
470        return findCounter(group, name);
471      }
472    
473      /**
474       * Increments the specified counter by the specified amount, creating it if
475       * it didn't already exist.
476       * @param key identifies a counter
477       * @param amount amount by which counter is to be incremented
478       */
479      public void incrCounter(Enum<?> key, long amount) {
480        findCounter(key).increment(amount);
481      }
482    
483      /**
484       * Increments the specified counter by the specified amount, creating it if
485       * it didn't already exist.
486       * @param group the name of the group
487       * @param counter the internal name of the counter
488       * @param amount amount by which counter is to be incremented
489       */
490      public void incrCounter(String group, String counter, long amount) {
491        findCounter(group, counter).increment(amount);
492      }
493    
494      /**
495       * Returns current value of the specified counter, or 0 if the counter
496       * does not exist.
497       * @param key the counter enum to lookup
498       * @return the counter value or 0 if counter not found
499       */
500      public synchronized long getCounter(Enum<?> key) {
501        return findCounter(key).getValue();
502      }
503    
504      /**
505       * Increments multiple counters by their amounts in another Counters
506       * instance.
507       * @param other the other Counters instance
508       */
509      public synchronized void incrAllCounters(Counters other) {
510        for (Group otherGroup: other) {
511          Group group = getGroup(otherGroup.getName());
512          group.setDisplayName(otherGroup.getDisplayName());
513          for (Counter otherCounter : otherGroup) {
514            Counter counter = group.getCounterForName(otherCounter.getName());
515            counter.setDisplayName(otherCounter.getDisplayName());
516            counter.increment(otherCounter.getValue());
517          }
518        }
519      }
520    
521      /**
522       * @return the total number of counters
523       * @deprecated use {@link #countCounters()} instead
524       */
525      public int size() {
526        return countCounters();
527      }
528    
529      /**
530       * Convenience method for computing the sum of two sets of counters.
531       * @param a the first counters
532       * @param b the second counters
533       * @return a new summed counters object
534       */
535      public static Counters sum(Counters a, Counters b) {
536        Counters counters = new Counters();
537        counters.incrAllCounters(a);
538        counters.incrAllCounters(b);
539        return counters;
540      }
541    
542      /**
543       * Logs the current counter values.
544       * @param log The log to use.
545       */
546      public void log(Log log) {
547        log.info("Counters: " + size());
548        for(Group group: this) {
549          log.info("  " + group.getDisplayName());
550          for (Counter counter: group) {
551            log.info("    " + counter.getDisplayName() + "=" +
552                     counter.getCounter());
553          }
554        }
555      }
556    
557      /**
558       * Represent the counter in a textual format that can be converted back to
559       * its object form
560       * @return the string in the following format
561       * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
562       */
563      public String makeEscapedCompactString() {
564        return toEscapedCompactString(this);
565      }
566    
567      /**
568       * Convert a stringified (by {@link #makeEscapedCompactString()} counter
569       * representation into a counter object.
570       * @param compactString to parse
571       * @return a new counters object
572       * @throws ParseException
573       */
574      public static Counters fromEscapedCompactString(String compactString)
575          throws ParseException {
576        return parseEscapedCompactString(compactString, new Counters());
577      }
578    }