001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapred;
020    
021    import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString;
022    import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString;
023    
024    import java.io.DataInput;
025    import java.io.DataOutput;
026    import java.io.IOException;
027    import java.text.ParseException;
028    import java.util.Collection;
029    import java.util.Iterator;
030    
031    import org.apache.commons.collections.IteratorUtils;
032    import org.apache.commons.logging.Log;
033    import org.apache.hadoop.classification.InterfaceAudience;
034    import org.apache.hadoop.classification.InterfaceStability;
035    import org.apache.hadoop.mapreduce.FileSystemCounter;
036    import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
037    import org.apache.hadoop.mapreduce.counters.AbstractCounters;
038    import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
039    import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
040    import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
041    import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
042    import org.apache.hadoop.mapreduce.counters.GenericCounter;
043    import org.apache.hadoop.mapreduce.counters.Limits;
044    import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
045    import org.apache.hadoop.mapreduce.util.CountersStrings;
046    
047    import com.google.common.collect.Iterators;
048    
049    /**
050     * A set of named counters.
051     *
052     * <p><code>Counters</code> represent global counters, defined either by the
053     * Map-Reduce framework or applications. Each <code>Counter</code> can be of
054     * any {@link Enum} type.</p>
055     *
056     * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
057     * counters from a particular <code>Enum</code> class.
058     */
059    @InterfaceAudience.Public
060    @InterfaceStability.Stable
061    public class Counters
062        extends AbstractCounters<Counters.Counter, Counters.Group> {
063      
064      public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
065      
066      public Counters() {
067        super(groupFactory);
068      }
069    
070      public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
071        super(newCounters, groupFactory);
072      }
073    
074      /**
075       * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
076       * @param newCounters new Counters
077       * @return old Counters instance corresponding to newCounters
078       */
079      static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
080        return new Counters(newCounters);
081      }
082    
083      public synchronized Group getGroup(String groupName) {
084        return super.getGroup(groupName);
085      }
086    
087      @SuppressWarnings("unchecked")
088      public synchronized Collection<String> getGroupNames() {
089        return IteratorUtils.toList(super.getGroupNames().iterator());
090      }
091    
092      public synchronized String makeCompactString() {
093        return CountersStrings.toEscapedCompactString(this);
094      }
095      
096      /**
097       * A counter record, comprising its name and value.
098       */
099      public static class Counter implements org.apache.hadoop.mapreduce.Counter {
100        org.apache.hadoop.mapreduce.Counter realCounter;
101    
102        Counter(org.apache.hadoop.mapreduce.Counter counter) {
103          this.realCounter = counter;
104        }
105    
106        public Counter() {
107          this(new GenericCounter());
108        }
109    
110        @SuppressWarnings("deprecation")
111        @Override
112        public void setDisplayName(String displayName) {
113          realCounter.setDisplayName(displayName);
114        }
115    
116        @Override
117        public String getName() {
118          return realCounter.getName();
119        }
120    
121        @Override
122        public String getDisplayName() {
123          return realCounter.getDisplayName();
124        }
125    
126        @Override
127        public long getValue() {
128          return realCounter.getValue();
129        }
130    
131        @Override
132        public void setValue(long value) {
133          realCounter.setValue(value);
134        }
135    
136        @Override
137        public void increment(long incr) {
138          realCounter.increment(incr);
139        }
140    
141        @Override
142        public void write(DataOutput out) throws IOException {
143          realCounter.write(out);
144        }
145    
146        @Override
147        public void readFields(DataInput in) throws IOException {
148          realCounter.readFields(in);
149        }
150    
151        /**
152         * Returns the compact stringified version of the counter in the format
153         * [(actual-name)(display-name)(value)]
154         * @return the stringified result
155         */
156        public String makeEscapedCompactString() {
157          return toEscapedCompactString(realCounter);
158        }
159    
160        /**
161         * Checks for (content) equality of two (basic) counters
162         * @param counter to compare
163         * @return true if content equals
164         * @deprecated
165         */
166        @Deprecated
167        public boolean contentEquals(Counter counter) {
168          return realCounter.equals(counter.getUnderlyingCounter());
169        }
170    
171        /**
172         * @return the value of the counter
173         */
174        public long getCounter() {
175          return realCounter.getValue();
176        }
177    
178        @Override
179        public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() {
180          return realCounter;
181        }
182        
183        @Override
184        public synchronized boolean equals(Object genericRight) {
185          if (genericRight instanceof Counter) {
186            synchronized (genericRight) {
187              Counter right = (Counter) genericRight;
188              return getName().equals(right.getName()) &&
189                     getDisplayName().equals(right.getDisplayName()) &&
190                     getValue() == right.getValue();
191            }
192          }
193          return false;
194        }
195        
196        @Override
197        public int hashCode() {
198          return realCounter.hashCode();
199        }
200      }
201    
202    
203      /**
204       *  <code>Group</code> of counters, comprising of counters from a particular
205       *  counter {@link Enum} class.
206       *
207       *  <p><code>Group</code>handles localization of the class name and the
208       *  counter names.</p>
209       */
210      public static class Group implements CounterGroupBase<Counter> {
211        private CounterGroupBase<Counter> realGroup;
212        
213        Group(GenericGroup group) {
214          this.realGroup = group;
215        }
216        Group(FSGroupImpl group) {
217          this.realGroup = group;
218        }
219        
220        @SuppressWarnings({ "unchecked", "rawtypes" })
221        Group(FrameworkGroupImpl group) {
222          this.realGroup = group;
223        }
224        
225        /**
226         * @param counterName the name of the counter
227         * @return the value of the specified counter, or 0 if the counter does
228         * not exist.
229         */
230        public long getCounter(String counterName)  {
231          return getCounterValue(realGroup, counterName);
232        }
233    
234        /**
235         * @return the compact stringified version of the group in the format
236         * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
237         * for the counters within.
238         */
239        public String makeEscapedCompactString() {
240          return toEscapedCompactString(realGroup);
241        }
242    
243        /**
244         * Get the counter for the given id and create it if it doesn't exist.
245         * @param id the numeric id of the counter within the group
246         * @param name the internal counter name
247         * @return the counter
248         * @deprecated use {@link #findCounter(String)} instead
249         */
250        @Deprecated
251        public Counter getCounter(int id, String name) {
252          return findCounter(name);
253        }
254    
255        /**
256         * Get the counter for the given name and create it if it doesn't exist.
257         * @param name the internal counter name
258         * @return the counter
259         */
260        public Counter getCounterForName(String name) {
261          return findCounter(name);
262        }
263    
264        @Override
265        public void write(DataOutput out) throws IOException {
266         realGroup.write(out); 
267        }
268    
269        @Override
270        public void readFields(DataInput in) throws IOException {
271          realGroup.readFields(in);
272        }
273    
274        @Override
275        public Iterator<Counter> iterator() {
276          return realGroup.iterator();
277        }
278    
279        @Override
280        public String getName() {
281          return realGroup.getName();
282        }
283    
284        @Override
285        public String getDisplayName() {
286          return realGroup.getDisplayName();
287        }
288    
289        @Override
290        public void setDisplayName(String displayName) {
291          realGroup.setDisplayName(displayName);
292        }
293    
294        @Override
295        public void addCounter(Counter counter) {
296          realGroup.addCounter(counter);
297        }
298    
299        @Override
300        public Counter addCounter(String name, String displayName, long value) {
301          return realGroup.addCounter(name, displayName, value);
302        }
303    
304        @Override
305        public Counter findCounter(String counterName, String displayName) {
306          return realGroup.findCounter(counterName, displayName);
307        }
308    
309        @Override
310        public Counter findCounter(String counterName, boolean create) {
311          return realGroup.findCounter(counterName, create);
312        }
313    
314        @Override
315        public Counter findCounter(String counterName) {
316          return realGroup.findCounter(counterName);
317        }
318    
319        @Override
320        public int size() {
321          return realGroup.size();
322        }
323    
324        @Override
325        public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
326          realGroup.incrAllCounters(rightGroup);
327        }
328        
329        @Override
330        public CounterGroupBase<Counter> getUnderlyingGroup() {
331          return realGroup;
332        }
333    
334        @Override
335        public synchronized boolean equals(Object genericRight) {
336          if (genericRight instanceof CounterGroupBase<?>) {
337            @SuppressWarnings("unchecked")
338            CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 
339            genericRight).getUnderlyingGroup();
340            return Iterators.elementsEqual(iterator(), right.iterator());
341          }
342          return false;
343        }
344    
345        @Override
346        public int hashCode() {
347          return realGroup.hashCode();
348        }
349      }
350    
351      // All the group impls need this for legacy group interface
352      static long getCounterValue(CounterGroupBase<Counter> group, String counterName) {
353        Counter counter = group.findCounter(counterName, false);
354        if (counter != null) return counter.getValue();
355        return 0L;
356      }
357    
358      // Mix the generic group implementation into the Group interface
359      private static class GenericGroup extends AbstractCounterGroup<Counter> {
360    
361        GenericGroup(String name, String displayName, Limits limits) {
362          super(name, displayName, limits);
363        }
364    
365        @Override
366        protected Counter newCounter(String counterName, String displayName,
367                                     long value) {
368          return new Counter(new GenericCounter(counterName, displayName, value));
369        }
370    
371        @Override
372        protected Counter newCounter() {
373          return new Counter();
374        }
375        
376        @Override
377        public CounterGroupBase<Counter> getUnderlyingGroup() {
378         return this;
379        }
380      }
381    
382      // Mix the framework group implementation into the Group interface
383      private static class FrameworkGroupImpl<T extends Enum<T>>
384          extends FrameworkCounterGroup<T, Counter> {
385    
386        // Mix the framework counter implementation into the Counter interface
387        class FrameworkCounterImpl extends FrameworkCounter {
388          FrameworkCounterImpl(T key) {
389            super(key);
390          }
391    
392        }
393    
394        FrameworkGroupImpl(Class<T> cls) {
395          super(cls);
396        }
397    
398        @Override
399        protected Counter newCounter(T key) {
400          return new Counter(new FrameworkCounterImpl(key));
401        }
402    
403        @Override
404        public CounterGroupBase<Counter> getUnderlyingGroup() {
405          return this;
406        }
407      }
408    
409      // Mix the file system counter group implementation into the Group interface
410      private static class FSGroupImpl extends FileSystemCounterGroup<Counter> {
411    
412        private class FSCounterImpl extends FSCounter {
413    
414          FSCounterImpl(String scheme, FileSystemCounter key) {
415            super(scheme, key);
416          }
417    
418        }
419    
420        @Override
421        protected Counter newCounter(String scheme, FileSystemCounter key) {
422          return new Counter(new FSCounterImpl(scheme, key));
423        }
424    
425        @Override
426        public CounterGroupBase<Counter> getUnderlyingGroup() {
427          return this;
428        }
429      }
430    
431      public synchronized Counter findCounter(String group, String name) {
432        if (name.equals("MAP_INPUT_BYTES")) {
433          LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
434                   "Use FileInputFormatCounters as group name and " +
435                   " BYTES_READ as counter name instead");
436          return findCounter(FileInputFormatCounter.BYTES_READ);
437        }
438        return getGroup(group).getCounterForName(name);
439      }
440    
441      /**
442       * Provide factory methods for counter group factory implementation.
443       * See also the GroupFactory in
444       *  {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
445       */
446      static class GroupFactory extends CounterGroupFactory<Counter, Group> {
447    
448        @Override
449        protected <T extends Enum<T>>
450        FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
451          return new FrameworkGroupFactory<Group>() {
452            @Override public Group newGroup(String name) {
453              return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package
454            }
455          };
456        }
457    
458        @Override
459        protected Group newGenericGroup(String name, String displayName,
460                                        Limits limits) {
461          return new Group(new GenericGroup(name, displayName, limits));
462        }
463    
464        @Override
465        protected Group newFileSystemGroup() {
466          return new Group(new FSGroupImpl());
467        }
468      }
469    
470      private static final GroupFactory groupFactory = new GroupFactory();
471    
472      /**
473       * Find a counter by using strings
474       * @param group the name of the group
475       * @param id the id of the counter within the group (0 to N-1)
476       * @param name the internal name of the counter
477       * @return the counter for that name
478       * @deprecated use {@link #findCounter(String, String)} instead
479       */
480      @Deprecated
481      public Counter findCounter(String group, int id, String name) {
482        return findCounter(group, name);
483      }
484    
485      /**
486       * Increments the specified counter by the specified amount, creating it if
487       * it didn't already exist.
488       * @param key identifies a counter
489       * @param amount amount by which counter is to be incremented
490       */
491      public void incrCounter(Enum<?> key, long amount) {
492        findCounter(key).increment(amount);
493      }
494    
495      /**
496       * Increments the specified counter by the specified amount, creating it if
497       * it didn't already exist.
498       * @param group the name of the group
499       * @param counter the internal name of the counter
500       * @param amount amount by which counter is to be incremented
501       */
502      public void incrCounter(String group, String counter, long amount) {
503        findCounter(group, counter).increment(amount);
504      }
505    
506      /**
507       * Returns current value of the specified counter, or 0 if the counter
508       * does not exist.
509       * @param key the counter enum to lookup
510       * @return the counter value or 0 if counter not found
511       */
512      public synchronized long getCounter(Enum<?> key) {
513        return findCounter(key).getValue();
514      }
515    
516      /**
517       * Increments multiple counters by their amounts in another Counters
518       * instance.
519       * @param other the other Counters instance
520       */
521      public synchronized void incrAllCounters(Counters other) {
522        for (Group otherGroup: other) {
523          Group group = getGroup(otherGroup.getName());
524          group.setDisplayName(otherGroup.getDisplayName());
525          for (Counter otherCounter : otherGroup) {
526            Counter counter = group.getCounterForName(otherCounter.getName());
527            counter.setDisplayName(otherCounter.getDisplayName());
528            counter.increment(otherCounter.getValue());
529          }
530        }
531      }
532    
533      /**
534       * @return the total number of counters
535       * @deprecated use {@link #countCounters()} instead
536       */
537      public int size() {
538        return countCounters();
539      }
540    
541      /**
542       * Convenience method for computing the sum of two sets of counters.
543       * @param a the first counters
544       * @param b the second counters
545       * @return a new summed counters object
546       */
547      public static Counters sum(Counters a, Counters b) {
548        Counters counters = new Counters();
549        counters.incrAllCounters(a);
550        counters.incrAllCounters(b);
551        return counters;
552      }
553    
554      /**
555       * Logs the current counter values.
556       * @param log The log to use.
557       */
558      public void log(Log log) {
559        log.info("Counters: " + size());
560        for(Group group: this) {
561          log.info("  " + group.getDisplayName());
562          for (Counter counter: group) {
563            log.info("    " + counter.getDisplayName() + "=" +
564                     counter.getCounter());
565          }
566        }
567      }
568    
569      /**
570       * Represent the counter in a textual format that can be converted back to
571       * its object form
572       * @return the string in the following format
573       * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
574       */
575      public String makeEscapedCompactString() {
576        return toEscapedCompactString(this);
577      }
578    
579      /**
580       * Convert a stringified (by {@link #makeEscapedCompactString()} counter
581       * representation into a counter object.
582       * @param compactString to parse
583       * @return a new counters object
584       * @throws ParseException
585       */
586      public static Counters fromEscapedCompactString(String compactString)
587          throws ParseException {
588        return parseEscapedCompactString(compactString, new Counters());
589      }
590    }