001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapred;
020    
021    import java.io.DataInput;
022    import java.io.DataOutput;
023    import java.io.IOException;
024    import java.util.ArrayList;
025    import java.util.Collection;
026    
027    import org.apache.hadoop.classification.InterfaceAudience;
028    import org.apache.hadoop.classification.InterfaceStability;
029    import org.apache.hadoop.io.Text;
030    import org.apache.hadoop.io.Writable;
031    import org.apache.hadoop.io.WritableUtils;
032    import org.apache.hadoop.mapreduce.ClusterMetrics;
033    import org.apache.hadoop.mapreduce.TaskTrackerInfo;
034    import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
035    import org.apache.hadoop.util.StringInterner;
036    
037    /**
038     * Status information on the current state of the Map-Reduce cluster.
039     * 
040     * <p><code>ClusterStatus</code> provides clients with information such as:
041     * <ol>
042     *   <li>
043     *   Size of the cluster. 
044     *   </li>
045     *   <li>
046     *   Name of the trackers. 
047     *   </li>
048     *   <li>
049     *   Task capacity of the cluster. 
050     *   </li>
051     *   <li>
052     *   The number of currently running map & reduce tasks.
053     *   </li>
054     *   <li>
055     *   State of the <code>JobTracker</code>.
056     *   </li>
057     *   <li>
058     *   Details regarding black listed trackers.
059     *   </li>
060     * </ol></p>
061     * 
062     * <p>Clients can query for the latest <code>ClusterStatus</code>, via 
063     * {@link JobClient#getClusterStatus()}.</p>
064     * 
065     * @see JobClient
066     */
067    @InterfaceAudience.Public
068    @InterfaceStability.Stable
069    public class ClusterStatus implements Writable {
070      /**
071       * Class which encapsulates information about a blacklisted tasktracker.
072       *  
073       * The information includes the tasktracker's name and reasons for
074       * getting blacklisted. The toString method of the class will print
075       * the information in a whitespace separated fashion to enable parsing.
076       */
077      public static class BlackListInfo implements Writable {
078    
079        private String trackerName;
080    
081        private String reasonForBlackListing;
082        
083        private String blackListReport;
084        
085        BlackListInfo() {
086        }
087        
088    
089        /**
090         * Gets the blacklisted tasktracker's name.
091         * 
092         * @return tracker's name.
093         */
094        public String getTrackerName() {
095          return trackerName;
096        }
097    
098        /**
099         * Gets the reason for which the tasktracker was blacklisted.
100         * 
101         * @return reason which tracker was blacklisted
102         */
103        public String getReasonForBlackListing() {
104          return reasonForBlackListing;
105        }
106    
107        /**
108         * Sets the blacklisted tasktracker's name.
109         * 
110         * @param trackerName of the tracker.
111         */
112        void setTrackerName(String trackerName) {
113          this.trackerName = trackerName;
114        }
115    
116        /**
117         * Sets the reason for which the tasktracker was blacklisted.
118         * 
119         * @param reasonForBlackListing
120         */
121        void setReasonForBlackListing(String reasonForBlackListing) {
122          this.reasonForBlackListing = reasonForBlackListing;
123        }
124    
125        /**
126         * Gets a descriptive report about why the tasktracker was blacklisted.
127         * 
128         * @return report describing why the tasktracker was blacklisted.
129         */
130        public String getBlackListReport() {
131          return blackListReport;
132        }
133    
134        /**
135         * Sets a descriptive report about why the tasktracker was blacklisted.
136         * @param blackListReport report describing why the tasktracker 
137         *                        was blacklisted.
138         */
139        void setBlackListReport(String blackListReport) {
140          this.blackListReport = blackListReport;
141        }
142    
143        @Override
144        public void readFields(DataInput in) throws IOException {
145          trackerName = StringInterner.weakIntern(Text.readString(in));
146          reasonForBlackListing = StringInterner.weakIntern(Text.readString(in));
147          blackListReport = StringInterner.weakIntern(Text.readString(in));
148        }
149    
150        @Override
151        public void write(DataOutput out) throws IOException {
152          Text.writeString(out, trackerName);
153          Text.writeString(out, reasonForBlackListing);
154          Text.writeString(out, blackListReport);
155        }
156    
157        @Override
158        /**
159         * Print information related to the blacklisted tasktracker in a
160         * whitespace separated fashion.
161         * 
162         * The method changes any newlines in the report describing why
163         * the tasktracker was blacklisted to a ':' for enabling better
164         * parsing.
165         */
166        public String toString() {
167          StringBuilder sb = new StringBuilder();
168          sb.append(trackerName);
169          sb.append("\t");
170          sb.append(reasonForBlackListing);
171          sb.append("\t");
172          sb.append(blackListReport.replace("\n", ":"));
173          return sb.toString();
174        }
175        
176      }
177      
178      private int numActiveTrackers;
179      private Collection<String> activeTrackers = new ArrayList<String>();
180      private int numBlacklistedTrackers;
181      private int numExcludedNodes;
182      private long ttExpiryInterval;
183      private int map_tasks;
184      private int reduce_tasks;
185      private int max_map_tasks;
186      private int max_reduce_tasks;
187      private JobTrackerStatus status;
188      private Collection<BlackListInfo> blacklistedTrackersInfo =
189        new ArrayList<BlackListInfo>();
190    
191      ClusterStatus() {}
192      
193      /**
194       * Construct a new cluster status.
195       * 
196       * @param trackers no. of tasktrackers in the cluster
197       * @param blacklists no of blacklisted task trackers in the cluster
198       * @param ttExpiryInterval the tasktracker expiry interval
199       * @param maps no. of currently running map-tasks in the cluster
200       * @param reduces no. of currently running reduce-tasks in the cluster
201       * @param maxMaps the maximum no. of map tasks in the cluster
202       * @param maxReduces the maximum no. of reduce tasks in the cluster
203       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
204       */
205      ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 
206                    int maps, int reduces,
207                    int maxMaps, int maxReduces, JobTrackerStatus status) {
208        this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps, 
209             maxReduces, status, 0);
210      }
211    
212      /**
213       * Construct a new cluster status.
214       * 
215       * @param trackers no. of tasktrackers in the cluster
216       * @param blacklists no of blacklisted task trackers in the cluster
217       * @param ttExpiryInterval the tasktracker expiry interval
218       * @param maps no. of currently running map-tasks in the cluster
219       * @param reduces no. of currently running reduce-tasks in the cluster
220       * @param maxMaps the maximum no. of map tasks in the cluster
221       * @param maxReduces the maximum no. of reduce tasks in the cluster
222       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
223       * @param numDecommissionedNodes number of decommission trackers
224       */
225      ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 
226                    int maps, int reduces, int maxMaps, int maxReduces, 
227                    JobTrackerStatus status, int numDecommissionedNodes) {
228        numActiveTrackers = trackers;
229        numBlacklistedTrackers = blacklists;
230        this.numExcludedNodes = numDecommissionedNodes;
231        this.ttExpiryInterval = ttExpiryInterval;
232        map_tasks = maps;
233        reduce_tasks = reduces;
234        max_map_tasks = maxMaps;
235        max_reduce_tasks = maxReduces;
236        this.status = status;
237      }
238    
239      /**
240       * Construct a new cluster status.
241       * 
242       * @param activeTrackers active tasktrackers in the cluster
243       * @param blacklistedTrackers blacklisted tasktrackers in the cluster
244       * @param ttExpiryInterval the tasktracker expiry interval
245       * @param maps no. of currently running map-tasks in the cluster
246       * @param reduces no. of currently running reduce-tasks in the cluster
247       * @param maxMaps the maximum no. of map tasks in the cluster
248       * @param maxReduces the maximum no. of reduce tasks in the cluster
249       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
250       */
251      ClusterStatus(Collection<String> activeTrackers, 
252          Collection<BlackListInfo> blacklistedTrackers,
253          long ttExpiryInterval,
254          int maps, int reduces, int maxMaps, int maxReduces, 
255          JobTrackerStatus status) {
256        this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces, 
257             maxMaps, maxReduces, status, 0);
258      }
259    
260    
261      /**
262       * Construct a new cluster status.
263       * 
264       * @param activeTrackers active tasktrackers in the cluster
265       * @param blackListedTrackerInfo blacklisted tasktrackers information 
266       * in the cluster
267       * @param ttExpiryInterval the tasktracker expiry interval
268       * @param maps no. of currently running map-tasks in the cluster
269       * @param reduces no. of currently running reduce-tasks in the cluster
270       * @param maxMaps the maximum no. of map tasks in the cluster
271       * @param maxReduces the maximum no. of reduce tasks in the cluster
272       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
273       * @param numDecommissionNodes number of decommission trackers
274       */
275      
276      ClusterStatus(Collection<String> activeTrackers,
277          Collection<BlackListInfo> blackListedTrackerInfo, long ttExpiryInterval,
278          int maps, int reduces, int maxMaps, int maxReduces,
279          JobTrackerStatus status, int numDecommissionNodes) {
280        this(activeTrackers.size(), blackListedTrackerInfo.size(),
281            ttExpiryInterval, maps, reduces, maxMaps, maxReduces, status,
282            numDecommissionNodes);
283        this.activeTrackers = activeTrackers;
284        this.blacklistedTrackersInfo = blackListedTrackerInfo;
285      }
286    
287      /**
288       * Get the number of task trackers in the cluster.
289       * 
290       * @return the number of task trackers in the cluster.
291       */
292      public int getTaskTrackers() {
293        return numActiveTrackers;
294      }
295      
296      /**
297       * Get the names of task trackers in the cluster.
298       * 
299       * @return the active task trackers in the cluster.
300       */
301      public Collection<String> getActiveTrackerNames() {
302        return activeTrackers;
303      }
304    
305      /**
306       * Get the names of task trackers in the cluster.
307       * 
308       * @return the blacklisted task trackers in the cluster.
309       */
310      public Collection<String> getBlacklistedTrackerNames() {
311        ArrayList<String> blacklistedTrackers = new ArrayList<String>();
312        for(BlackListInfo bi : blacklistedTrackersInfo) {
313          blacklistedTrackers.add(bi.getTrackerName());
314        }
315        return blacklistedTrackers;
316      }
317      
318      /**
319       * Get the number of blacklisted task trackers in the cluster.
320       * 
321       * @return the number of blacklisted task trackers in the cluster.
322       */
323      public int getBlacklistedTrackers() {
324        return numBlacklistedTrackers;
325      }
326      
327      /**
328       * Get the number of excluded hosts in the cluster.
329       * @return the number of excluded hosts in the cluster.
330       */
331      public int getNumExcludedNodes() {
332        return numExcludedNodes;
333      }
334      
335      /**
336       * Get the tasktracker expiry interval for the cluster
337       * @return the expiry interval in msec
338       */
339      public long getTTExpiryInterval() {
340        return ttExpiryInterval;
341      }
342      
343      /**
344       * Get the number of currently running map tasks in the cluster.
345       * 
346       * @return the number of currently running map tasks in the cluster.
347       */
348      public int getMapTasks() {
349        return map_tasks;
350      }
351      
352      /**
353       * Get the number of currently running reduce tasks in the cluster.
354       * 
355       * @return the number of currently running reduce tasks in the cluster.
356       */
357      public int getReduceTasks() {
358        return reduce_tasks;
359      }
360      
361      /**
362       * Get the maximum capacity for running map tasks in the cluster.
363       * 
364       * @return the maximum capacity for running map tasks in the cluster.
365       */
366      public int getMaxMapTasks() {
367        return max_map_tasks;
368      }
369    
370      /**
371       * Get the maximum capacity for running reduce tasks in the cluster.
372       * 
373       * @return the maximum capacity for running reduce tasks in the cluster.
374       */
375      public int getMaxReduceTasks() {
376        return max_reduce_tasks;
377      }
378      
379      /**
380       * Get the JobTracker's status.
381       * 
382       * @return {@link JobTrackerStatus} of the JobTracker
383       */
384      public JobTrackerStatus getJobTrackerStatus() {
385        return status;
386      }
387    
388      /**
389       * Gets the list of blacklisted trackers along with reasons for blacklisting.
390       * 
391       * @return the collection of {@link BlackListInfo} objects. 
392       * 
393       */
394      public Collection<BlackListInfo> getBlackListedTrackersInfo() {
395        return blacklistedTrackersInfo;
396      }
397    
398      public void write(DataOutput out) throws IOException {
399        if (activeTrackers.size() == 0) {
400          out.writeInt(numActiveTrackers);
401          out.writeInt(0);
402        } else {
403          out.writeInt(activeTrackers.size());
404          out.writeInt(activeTrackers.size());
405          for (String tracker : activeTrackers) {
406            Text.writeString(out, tracker);
407          }
408        }
409        if (blacklistedTrackersInfo.size() == 0) {
410          out.writeInt(numBlacklistedTrackers);
411          out.writeInt(blacklistedTrackersInfo.size());
412        } else {
413          out.writeInt(blacklistedTrackersInfo.size());
414          out.writeInt(blacklistedTrackersInfo.size());
415          for (BlackListInfo tracker : blacklistedTrackersInfo) {
416            tracker.write(out);
417          }
418        }
419        out.writeInt(numExcludedNodes);
420        out.writeLong(ttExpiryInterval);
421        out.writeInt(map_tasks);
422        out.writeInt(reduce_tasks);
423        out.writeInt(max_map_tasks);
424        out.writeInt(max_reduce_tasks);
425        WritableUtils.writeEnum(out, status);
426      }
427    
428      public void readFields(DataInput in) throws IOException {
429        numActiveTrackers = in.readInt();
430        int numTrackerNames = in.readInt();
431        if (numTrackerNames > 0) {
432          for (int i = 0; i < numTrackerNames; i++) {
433            String name = StringInterner.weakIntern(Text.readString(in));
434            activeTrackers.add(name);
435          }
436        }
437        numBlacklistedTrackers = in.readInt();
438        int blackListTrackerInfoSize = in.readInt();
439        if(blackListTrackerInfoSize > 0) {
440          for (int i = 0; i < blackListTrackerInfoSize; i++) {
441            BlackListInfo info = new BlackListInfo();
442            info.readFields(in);
443            blacklistedTrackersInfo.add(info);
444          }
445        }
446        numExcludedNodes = in.readInt();
447        ttExpiryInterval = in.readLong();
448        map_tasks = in.readInt();
449        reduce_tasks = in.readInt();
450        max_map_tasks = in.readInt();
451        max_reduce_tasks = in.readInt();
452        status = WritableUtils.readEnum(in, JobTrackerStatus.class);
453      }
454    }