001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred; 020 021 import java.io.DataInput; 022 import java.io.DataOutput; 023 import java.io.IOException; 024 import java.util.ArrayList; 025 import java.util.Collection; 026 027 import org.apache.hadoop.classification.InterfaceAudience; 028 import org.apache.hadoop.classification.InterfaceStability; 029 import org.apache.hadoop.io.Text; 030 import org.apache.hadoop.io.Writable; 031 import org.apache.hadoop.io.WritableUtils; 032 import org.apache.hadoop.mapreduce.ClusterMetrics; 033 import org.apache.hadoop.mapreduce.TaskTrackerInfo; 034 import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; 035 import org.apache.hadoop.util.StringInterner; 036 037 /** 038 * Status information on the current state of the Map-Reduce cluster. 039 * 040 * <p><code>ClusterStatus</code> provides clients with information such as: 041 * <ol> 042 * <li> 043 * Size of the cluster. 044 * </li> 045 * <li> 046 * Name of the trackers. 047 * </li> 048 * <li> 049 * Task capacity of the cluster. 050 * </li> 051 * <li> 052 * The number of currently running map & reduce tasks. 053 * </li> 054 * <li> 055 * State of the <code>JobTracker</code>. 056 * </li> 057 * <li> 058 * Details regarding black listed trackers. 059 * </li> 060 * </ol></p> 061 * 062 * <p>Clients can query for the latest <code>ClusterStatus</code>, via 063 * {@link JobClient#getClusterStatus()}.</p> 064 * 065 * @see JobClient 066 */ 067 @InterfaceAudience.Public 068 @InterfaceStability.Stable 069 public class ClusterStatus implements Writable { 070 /** 071 * Class which encapsulates information about a blacklisted tasktracker. 072 * 073 * The information includes the tasktracker's name and reasons for 074 * getting blacklisted. The toString method of the class will print 075 * the information in a whitespace separated fashion to enable parsing. 076 */ 077 public static class BlackListInfo implements Writable { 078 079 private String trackerName; 080 081 private String reasonForBlackListing; 082 083 private String blackListReport; 084 085 BlackListInfo() { 086 } 087 088 089 /** 090 * Gets the blacklisted tasktracker's name. 091 * 092 * @return tracker's name. 093 */ 094 public String getTrackerName() { 095 return trackerName; 096 } 097 098 /** 099 * Gets the reason for which the tasktracker was blacklisted. 100 * 101 * @return reason which tracker was blacklisted 102 */ 103 public String getReasonForBlackListing() { 104 return reasonForBlackListing; 105 } 106 107 /** 108 * Sets the blacklisted tasktracker's name. 109 * 110 * @param trackerName of the tracker. 111 */ 112 void setTrackerName(String trackerName) { 113 this.trackerName = trackerName; 114 } 115 116 /** 117 * Sets the reason for which the tasktracker was blacklisted. 118 * 119 * @param reasonForBlackListing 120 */ 121 void setReasonForBlackListing(String reasonForBlackListing) { 122 this.reasonForBlackListing = reasonForBlackListing; 123 } 124 125 /** 126 * Gets a descriptive report about why the tasktracker was blacklisted. 127 * 128 * @return report describing why the tasktracker was blacklisted. 129 */ 130 public String getBlackListReport() { 131 return blackListReport; 132 } 133 134 /** 135 * Sets a descriptive report about why the tasktracker was blacklisted. 136 * @param blackListReport report describing why the tasktracker 137 * was blacklisted. 138 */ 139 void setBlackListReport(String blackListReport) { 140 this.blackListReport = blackListReport; 141 } 142 143 @Override 144 public void readFields(DataInput in) throws IOException { 145 trackerName = StringInterner.weakIntern(Text.readString(in)); 146 reasonForBlackListing = StringInterner.weakIntern(Text.readString(in)); 147 blackListReport = StringInterner.weakIntern(Text.readString(in)); 148 } 149 150 @Override 151 public void write(DataOutput out) throws IOException { 152 Text.writeString(out, trackerName); 153 Text.writeString(out, reasonForBlackListing); 154 Text.writeString(out, blackListReport); 155 } 156 157 @Override 158 /** 159 * Print information related to the blacklisted tasktracker in a 160 * whitespace separated fashion. 161 * 162 * The method changes any newlines in the report describing why 163 * the tasktracker was blacklisted to a ':' for enabling better 164 * parsing. 165 */ 166 public String toString() { 167 StringBuilder sb = new StringBuilder(); 168 sb.append(trackerName); 169 sb.append("\t"); 170 sb.append(reasonForBlackListing); 171 sb.append("\t"); 172 sb.append(blackListReport.replace("\n", ":")); 173 return sb.toString(); 174 } 175 176 } 177 178 private int numActiveTrackers; 179 private Collection<String> activeTrackers = new ArrayList<String>(); 180 private int numBlacklistedTrackers; 181 private int numExcludedNodes; 182 private long ttExpiryInterval; 183 private int map_tasks; 184 private int reduce_tasks; 185 private int max_map_tasks; 186 private int max_reduce_tasks; 187 private JobTrackerStatus status; 188 private Collection<BlackListInfo> blacklistedTrackersInfo = 189 new ArrayList<BlackListInfo>(); 190 191 ClusterStatus() {} 192 193 /** 194 * Construct a new cluster status. 195 * 196 * @param trackers no. of tasktrackers in the cluster 197 * @param blacklists no of blacklisted task trackers in the cluster 198 * @param ttExpiryInterval the tasktracker expiry interval 199 * @param maps no. of currently running map-tasks in the cluster 200 * @param reduces no. of currently running reduce-tasks in the cluster 201 * @param maxMaps the maximum no. of map tasks in the cluster 202 * @param maxReduces the maximum no. of reduce tasks in the cluster 203 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 204 */ 205 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 206 int maps, int reduces, 207 int maxMaps, int maxReduces, JobTrackerStatus status) { 208 this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps, 209 maxReduces, status, 0); 210 } 211 212 /** 213 * Construct a new cluster status. 214 * 215 * @param trackers no. of tasktrackers in the cluster 216 * @param blacklists no of blacklisted task trackers in the cluster 217 * @param ttExpiryInterval the tasktracker expiry interval 218 * @param maps no. of currently running map-tasks in the cluster 219 * @param reduces no. of currently running reduce-tasks in the cluster 220 * @param maxMaps the maximum no. of map tasks in the cluster 221 * @param maxReduces the maximum no. of reduce tasks in the cluster 222 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 223 * @param numDecommissionedNodes number of decommission trackers 224 */ 225 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 226 int maps, int reduces, int maxMaps, int maxReduces, 227 JobTrackerStatus status, int numDecommissionedNodes) { 228 numActiveTrackers = trackers; 229 numBlacklistedTrackers = blacklists; 230 this.numExcludedNodes = numDecommissionedNodes; 231 this.ttExpiryInterval = ttExpiryInterval; 232 map_tasks = maps; 233 reduce_tasks = reduces; 234 max_map_tasks = maxMaps; 235 max_reduce_tasks = maxReduces; 236 this.status = status; 237 } 238 239 /** 240 * Construct a new cluster status. 241 * 242 * @param activeTrackers active tasktrackers in the cluster 243 * @param blacklistedTrackers blacklisted tasktrackers in the cluster 244 * @param ttExpiryInterval the tasktracker expiry interval 245 * @param maps no. of currently running map-tasks in the cluster 246 * @param reduces no. of currently running reduce-tasks in the cluster 247 * @param maxMaps the maximum no. of map tasks in the cluster 248 * @param maxReduces the maximum no. of reduce tasks in the cluster 249 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 250 */ 251 ClusterStatus(Collection<String> activeTrackers, 252 Collection<BlackListInfo> blacklistedTrackers, 253 long ttExpiryInterval, 254 int maps, int reduces, int maxMaps, int maxReduces, 255 JobTrackerStatus status) { 256 this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces, 257 maxMaps, maxReduces, status, 0); 258 } 259 260 261 /** 262 * Construct a new cluster status. 263 * 264 * @param activeTrackers active tasktrackers in the cluster 265 * @param blackListedTrackerInfo blacklisted tasktrackers information 266 * in the cluster 267 * @param ttExpiryInterval the tasktracker expiry interval 268 * @param maps no. of currently running map-tasks in the cluster 269 * @param reduces no. of currently running reduce-tasks in the cluster 270 * @param maxMaps the maximum no. of map tasks in the cluster 271 * @param maxReduces the maximum no. of reduce tasks in the cluster 272 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code> 273 * @param numDecommissionNodes number of decommission trackers 274 */ 275 276 ClusterStatus(Collection<String> activeTrackers, 277 Collection<BlackListInfo> blackListedTrackerInfo, long ttExpiryInterval, 278 int maps, int reduces, int maxMaps, int maxReduces, 279 JobTrackerStatus status, int numDecommissionNodes) { 280 this(activeTrackers.size(), blackListedTrackerInfo.size(), 281 ttExpiryInterval, maps, reduces, maxMaps, maxReduces, status, 282 numDecommissionNodes); 283 this.activeTrackers = activeTrackers; 284 this.blacklistedTrackersInfo = blackListedTrackerInfo; 285 } 286 287 /** 288 * Get the number of task trackers in the cluster. 289 * 290 * @return the number of task trackers in the cluster. 291 */ 292 public int getTaskTrackers() { 293 return numActiveTrackers; 294 } 295 296 /** 297 * Get the names of task trackers in the cluster. 298 * 299 * @return the active task trackers in the cluster. 300 */ 301 public Collection<String> getActiveTrackerNames() { 302 return activeTrackers; 303 } 304 305 /** 306 * Get the names of task trackers in the cluster. 307 * 308 * @return the blacklisted task trackers in the cluster. 309 */ 310 public Collection<String> getBlacklistedTrackerNames() { 311 ArrayList<String> blacklistedTrackers = new ArrayList<String>(); 312 for(BlackListInfo bi : blacklistedTrackersInfo) { 313 blacklistedTrackers.add(bi.getTrackerName()); 314 } 315 return blacklistedTrackers; 316 } 317 318 /** 319 * Get the number of blacklisted task trackers in the cluster. 320 * 321 * @return the number of blacklisted task trackers in the cluster. 322 */ 323 public int getBlacklistedTrackers() { 324 return numBlacklistedTrackers; 325 } 326 327 /** 328 * Get the number of excluded hosts in the cluster. 329 * @return the number of excluded hosts in the cluster. 330 */ 331 public int getNumExcludedNodes() { 332 return numExcludedNodes; 333 } 334 335 /** 336 * Get the tasktracker expiry interval for the cluster 337 * @return the expiry interval in msec 338 */ 339 public long getTTExpiryInterval() { 340 return ttExpiryInterval; 341 } 342 343 /** 344 * Get the number of currently running map tasks in the cluster. 345 * 346 * @return the number of currently running map tasks in the cluster. 347 */ 348 public int getMapTasks() { 349 return map_tasks; 350 } 351 352 /** 353 * Get the number of currently running reduce tasks in the cluster. 354 * 355 * @return the number of currently running reduce tasks in the cluster. 356 */ 357 public int getReduceTasks() { 358 return reduce_tasks; 359 } 360 361 /** 362 * Get the maximum capacity for running map tasks in the cluster. 363 * 364 * @return the maximum capacity for running map tasks in the cluster. 365 */ 366 public int getMaxMapTasks() { 367 return max_map_tasks; 368 } 369 370 /** 371 * Get the maximum capacity for running reduce tasks in the cluster. 372 * 373 * @return the maximum capacity for running reduce tasks in the cluster. 374 */ 375 public int getMaxReduceTasks() { 376 return max_reduce_tasks; 377 } 378 379 /** 380 * Get the JobTracker's status. 381 * 382 * @return {@link JobTrackerStatus} of the JobTracker 383 */ 384 public JobTrackerStatus getJobTrackerStatus() { 385 return status; 386 } 387 388 /** 389 * Gets the list of blacklisted trackers along with reasons for blacklisting. 390 * 391 * @return the collection of {@link BlackListInfo} objects. 392 * 393 */ 394 public Collection<BlackListInfo> getBlackListedTrackersInfo() { 395 return blacklistedTrackersInfo; 396 } 397 398 public void write(DataOutput out) throws IOException { 399 if (activeTrackers.size() == 0) { 400 out.writeInt(numActiveTrackers); 401 out.writeInt(0); 402 } else { 403 out.writeInt(activeTrackers.size()); 404 out.writeInt(activeTrackers.size()); 405 for (String tracker : activeTrackers) { 406 Text.writeString(out, tracker); 407 } 408 } 409 if (blacklistedTrackersInfo.size() == 0) { 410 out.writeInt(numBlacklistedTrackers); 411 out.writeInt(blacklistedTrackersInfo.size()); 412 } else { 413 out.writeInt(blacklistedTrackersInfo.size()); 414 out.writeInt(blacklistedTrackersInfo.size()); 415 for (BlackListInfo tracker : blacklistedTrackersInfo) { 416 tracker.write(out); 417 } 418 } 419 out.writeInt(numExcludedNodes); 420 out.writeLong(ttExpiryInterval); 421 out.writeInt(map_tasks); 422 out.writeInt(reduce_tasks); 423 out.writeInt(max_map_tasks); 424 out.writeInt(max_reduce_tasks); 425 WritableUtils.writeEnum(out, status); 426 } 427 428 public void readFields(DataInput in) throws IOException { 429 numActiveTrackers = in.readInt(); 430 int numTrackerNames = in.readInt(); 431 if (numTrackerNames > 0) { 432 for (int i = 0; i < numTrackerNames; i++) { 433 String name = StringInterner.weakIntern(Text.readString(in)); 434 activeTrackers.add(name); 435 } 436 } 437 numBlacklistedTrackers = in.readInt(); 438 int blackListTrackerInfoSize = in.readInt(); 439 if(blackListTrackerInfoSize > 0) { 440 for (int i = 0; i < blackListTrackerInfoSize; i++) { 441 BlackListInfo info = new BlackListInfo(); 442 info.readFields(in); 443 blacklistedTrackersInfo.add(info); 444 } 445 } 446 numExcludedNodes = in.readInt(); 447 ttExpiryInterval = in.readLong(); 448 map_tasks = in.readInt(); 449 reduce_tasks = in.readInt(); 450 max_map_tasks = in.readInt(); 451 max_reduce_tasks = in.readInt(); 452 status = WritableUtils.readEnum(in, JobTrackerStatus.class); 453 } 454 }