001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapred;
020    
021    
022    import java.io.IOException;
023    import java.net.URL;
024    import java.net.URLDecoder;
025    import java.util.Enumeration;
026    import java.util.regex.Pattern;
027    
028    import org.apache.commons.logging.Log;
029    import org.apache.commons.logging.LogFactory;
030    import org.apache.hadoop.classification.InterfaceAudience;
031    import org.apache.hadoop.classification.InterfaceStability;
032    import org.apache.hadoop.conf.Configuration;
033    import org.apache.hadoop.fs.FileStatus;
034    import org.apache.hadoop.fs.FileSystem;
035    import org.apache.hadoop.fs.Path;
036    import org.apache.hadoop.io.LongWritable;
037    import org.apache.hadoop.io.RawComparator;
038    import org.apache.hadoop.io.Text;
039    import org.apache.hadoop.io.WritableComparable;
040    import org.apache.hadoop.io.WritableComparator;
041    import org.apache.hadoop.io.compress.CompressionCodec;
042    import org.apache.hadoop.mapred.lib.HashPartitioner;
043    import org.apache.hadoop.mapred.lib.IdentityMapper;
044    import org.apache.hadoop.mapred.lib.IdentityReducer;
045    import org.apache.hadoop.mapred.lib.KeyFieldBasedComparator;
046    import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner;
047    import org.apache.hadoop.mapreduce.MRConfig;
048    import org.apache.hadoop.mapreduce.MRJobConfig;
049    import org.apache.hadoop.mapreduce.filecache.DistributedCache;
050    import org.apache.hadoop.mapreduce.util.ConfigUtil;
051    import org.apache.hadoop.security.Credentials;
052    import org.apache.hadoop.util.ReflectionUtils;
053    import org.apache.hadoop.util.Tool;
054    import org.apache.log4j.Level;
055    
056    /** 
057     * A map/reduce job configuration.
058     * 
059     * <p><code>JobConf</code> is the primary interface for a user to describe a 
060     * map-reduce job to the Hadoop framework for execution. The framework tries to
061     * faithfully execute the job as-is described by <code>JobConf</code>, however:
062     * <ol>
063     *   <li>
064     *   Some configuration parameters might have been marked as 
065     *   <a href="{@docRoot}/org/apache/hadoop/conf/Configuration.html#FinalParams">
066     *   final</a> by administrators and hence cannot be altered.
067     *   </li>
068     *   <li>
069     *   While some job parameters are straight-forward to set 
070     *   (e.g. {@link #setNumReduceTasks(int)}), some parameters interact subtly 
071     *   rest of the framework and/or job-configuration and is relatively more 
072     *   complex for the user to control finely (e.g. {@link #setNumMapTasks(int)}).
073     *   </li>
074     * </ol></p>
075     * 
076     * <p><code>JobConf</code> typically specifies the {@link Mapper}, combiner 
077     * (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and 
078     * {@link OutputFormat} implementations to be used etc.
079     *
080     * <p>Optionally <code>JobConf</code> is used to specify other advanced facets 
081     * of the job such as <code>Comparator</code>s to be used, files to be put in  
082     * the {@link DistributedCache}, whether or not intermediate and/or job outputs 
083     * are to be compressed (and how), debugability via user-provided scripts 
084     * ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}),
085     * for doing post-processing on task logs, task's stdout, stderr, syslog. 
086     * and etc.</p>
087     * 
088     * <p>Here is an example on how to configure a job via <code>JobConf</code>:</p>
089     * <p><blockquote><pre>
090     *     // Create a new JobConf
091     *     JobConf job = new JobConf(new Configuration(), MyJob.class);
092     *     
093     *     // Specify various job-specific parameters     
094     *     job.setJobName("myjob");
095     *     
096     *     FileInputFormat.setInputPaths(job, new Path("in"));
097     *     FileOutputFormat.setOutputPath(job, new Path("out"));
098     *     
099     *     job.setMapperClass(MyJob.MyMapper.class);
100     *     job.setCombinerClass(MyJob.MyReducer.class);
101     *     job.setReducerClass(MyJob.MyReducer.class);
102     *     
103     *     job.setInputFormat(SequenceFileInputFormat.class);
104     *     job.setOutputFormat(SequenceFileOutputFormat.class);
105     * </pre></blockquote></p>
106     * 
107     * @see JobClient
108     * @see ClusterStatus
109     * @see Tool
110     * @see DistributedCache
111     */
112    @InterfaceAudience.Public
113    @InterfaceStability.Stable
114    public class JobConf extends Configuration {
115      
116      private static final Log LOG = LogFactory.getLog(JobConf.class);
117    
118      static{
119        ConfigUtil.loadResources();
120      }
121    
122      /**
123       * @deprecated Use {@link #MAPRED_JOB_MAP_MEMORY_MB_PROPERTY} and
124       * {@link #MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY}
125       */
126      @Deprecated
127      public static final String MAPRED_TASK_MAXVMEM_PROPERTY =
128        "mapred.task.maxvmem";
129    
130      /**
131       * @deprecated 
132       */
133      @Deprecated
134      public static final String UPPER_LIMIT_ON_TASK_VMEM_PROPERTY =
135        "mapred.task.limit.maxvmem";
136    
137      /**
138       * @deprecated
139       */
140      @Deprecated
141      public static final String MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY =
142        "mapred.task.default.maxvmem";
143    
144      /**
145       * @deprecated
146       */
147      @Deprecated
148      public static final String MAPRED_TASK_MAXPMEM_PROPERTY =
149        "mapred.task.maxpmem";
150    
151      /**
152       * A value which if set for memory related configuration options,
153       * indicates that the options are turned off.
154       */
155      public static final long DISABLED_MEMORY_LIMIT = -1L;
156    
157      /**
158       * Property name for the configuration property mapreduce.cluster.local.dir
159       */
160      public static final String MAPRED_LOCAL_DIR_PROPERTY = MRConfig.LOCAL_DIR;
161    
162      /**
163       * Name of the queue to which jobs will be submitted, if no queue
164       * name is mentioned.
165       */
166      public static final String DEFAULT_QUEUE_NAME = "default";
167    
168      static final String MAPRED_JOB_MAP_MEMORY_MB_PROPERTY = 
169          JobContext.MAP_MEMORY_MB;
170    
171      static final String MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY =
172        JobContext.REDUCE_MEMORY_MB;
173    
174      /** Pattern for the default unpacking behavior for job jars */
175      public static final Pattern UNPACK_JAR_PATTERN_DEFAULT =
176        Pattern.compile("(?:classes/|lib/).*");
177    
178      /**
179       * Configuration key to set the java command line options for the child
180       * map and reduce tasks.
181       * 
182       * Java opts for the task tracker child processes.
183       * The following symbol, if present, will be interpolated: @taskid@. 
184       * It is replaced by current TaskID. Any other occurrences of '@' will go 
185       * unchanged.
186       * For example, to enable verbose gc logging to a file named for the taskid in
187       * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
188       *          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
189       * 
190       * The configuration variable {@link #MAPRED_TASK_ULIMIT} can be used to 
191       * control the maximum virtual memory of the child processes.
192       * 
193       * The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass 
194       * other environment variables to the child processes.
195       * 
196       * @deprecated Use {@link #MAPRED_MAP_TASK_JAVA_OPTS} or 
197       *                 {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}
198       */
199      @Deprecated
200      public static final String MAPRED_TASK_JAVA_OPTS = "mapred.child.java.opts";
201      
202      /**
203       * Configuration key to set the java command line options for the map tasks.
204       * 
205       * Java opts for the task tracker child map processes.
206       * The following symbol, if present, will be interpolated: @taskid@. 
207       * It is replaced by current TaskID. Any other occurrences of '@' will go 
208       * unchanged.
209       * For example, to enable verbose gc logging to a file named for the taskid in
210       * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
211       *          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
212       * 
213       * The configuration variable {@link #MAPRED_MAP_TASK_ULIMIT} can be used to 
214       * control the maximum virtual memory of the map processes.
215       * 
216       * The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass 
217       * other environment variables to the map processes.
218       */
219      public static final String MAPRED_MAP_TASK_JAVA_OPTS = 
220        JobContext.MAP_JAVA_OPTS;
221      
222      /**
223       * Configuration key to set the java command line options for the reduce tasks.
224       * 
225       * Java opts for the task tracker child reduce processes.
226       * The following symbol, if present, will be interpolated: @taskid@. 
227       * It is replaced by current TaskID. Any other occurrences of '@' will go 
228       * unchanged.
229       * For example, to enable verbose gc logging to a file named for the taskid in
230       * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
231       *          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
232       * 
233       * The configuration variable {@link #MAPRED_REDUCE_TASK_ULIMIT} can be used  
234       * to control the maximum virtual memory of the reduce processes.
235       * 
236       * The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to 
237       * pass process environment variables to the reduce processes.
238       */
239      public static final String MAPRED_REDUCE_TASK_JAVA_OPTS = 
240        JobContext.REDUCE_JAVA_OPTS;
241      
242      public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "-Xmx200m";
243      
244      /**
245       * Configuration key to set the maximum virutal memory available to the child
246       * map and reduce tasks (in kilo-bytes).
247       * 
248       * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
249       *       via {@link #MAPRED_TASK_JAVA_OPTS}, else the VM might not start.
250       * 
251       * @deprecated Use {@link #MAPRED_MAP_TASK_ULIMIT} or 
252       *                 {@link #MAPRED_REDUCE_TASK_ULIMIT}
253       */
254      @Deprecated
255      public static final String MAPRED_TASK_ULIMIT = "mapred.child.ulimit";
256    
257      /**
258       * Configuration key to set the maximum virutal memory available to the
259       * map tasks (in kilo-bytes).
260       * 
261       * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
262       *       via {@link #MAPRED_MAP_TASK_JAVA_OPTS}, else the VM might not start.
263       */
264      public static final String MAPRED_MAP_TASK_ULIMIT = JobContext.MAP_ULIMIT;
265      
266      /**
267       * Configuration key to set the maximum virutal memory available to the
268       * reduce tasks (in kilo-bytes).
269       * 
270       * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
271       *       via {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}, else the VM might not start.
272       */
273      public static final String MAPRED_REDUCE_TASK_ULIMIT = 
274        JobContext.REDUCE_ULIMIT;
275    
276    
277      /**
278       * Configuration key to set the environment of the child map/reduce tasks.
279       * 
280       * The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
281       * reference existing environment variables via <code>$key</code>.
282       * 
283       * Example:
284       * <ul>
285       *   <li> A=foo - This will set the env variable A to foo. </li>
286       *   <li> B=$X:c This is inherit tasktracker's X env variable. </li>
287       * </ul>
288       * 
289       * @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or 
290       *                 {@link #MAPRED_REDUCE_TASK_ENV}
291       */
292      @Deprecated
293      public static final String MAPRED_TASK_ENV = "mapred.child.env";
294    
295      /**
296       * Configuration key to set the maximum virutal memory available to the
297       * map tasks.
298       * 
299       * The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
300       * reference existing environment variables via <code>$key</code>.
301       * 
302       * Example:
303       * <ul>
304       *   <li> A=foo - This will set the env variable A to foo. </li>
305       *   <li> B=$X:c This is inherit tasktracker's X env variable. </li>
306       * </ul>
307       */
308      public static final String MAPRED_MAP_TASK_ENV = JobContext.MAP_ENV;
309      
310      /**
311       * Configuration key to set the maximum virutal memory available to the
312       * reduce tasks.
313       * 
314       * The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
315       * reference existing environment variables via <code>$key</code>.
316       * 
317       * Example:
318       * <ul>
319       *   <li> A=foo - This will set the env variable A to foo. </li>
320       *   <li> B=$X:c This is inherit tasktracker's X env variable. </li>
321       * </ul>
322       */
323      public static final String MAPRED_REDUCE_TASK_ENV = JobContext.REDUCE_ENV;
324    
325      private Credentials credentials = new Credentials();
326      
327      /**
328       * Configuration key to set the logging {@link Level} for the map task.
329       *
330       * The allowed logging levels are:
331       * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
332       */
333      public static final String MAPRED_MAP_TASK_LOG_LEVEL = 
334        JobContext.MAP_LOG_LEVEL;
335      
336      /**
337       * Configuration key to set the logging {@link Level} for the reduce task.
338       *
339       * The allowed logging levels are:
340       * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
341       */
342      public static final String MAPRED_REDUCE_TASK_LOG_LEVEL = 
343        JobContext.REDUCE_LOG_LEVEL;
344      
345      /**
346       * Default logging level for map/reduce tasks.
347       */
348      public static final Level DEFAULT_LOG_LEVEL = Level.INFO;
349      
350      
351      /**
352       * Construct a map/reduce job configuration.
353       */
354      public JobConf() {
355        checkAndWarnDeprecation();
356      }
357    
358      /** 
359       * Construct a map/reduce job configuration.
360       * 
361       * @param exampleClass a class whose containing jar is used as the job's jar.
362       */
363      public JobConf(Class exampleClass) {
364        setJarByClass(exampleClass);
365        checkAndWarnDeprecation();
366      }
367      
368      /**
369       * Construct a map/reduce job configuration.
370       * 
371       * @param conf a Configuration whose settings will be inherited.
372       */
373      public JobConf(Configuration conf) {
374        super(conf);
375        
376        if (conf instanceof JobConf) {
377          JobConf that = (JobConf)conf;
378          credentials = that.credentials;
379        }
380        
381        checkAndWarnDeprecation();
382      }
383    
384    
385      /** Construct a map/reduce job configuration.
386       * 
387       * @param conf a Configuration whose settings will be inherited.
388       * @param exampleClass a class whose containing jar is used as the job's jar.
389       */
390      public JobConf(Configuration conf, Class exampleClass) {
391        this(conf);
392        setJarByClass(exampleClass);
393      }
394    
395    
396      /** Construct a map/reduce configuration.
397       *
398       * @param config a Configuration-format XML job description file.
399       */
400      public JobConf(String config) {
401        this(new Path(config));
402      }
403    
404      /** Construct a map/reduce configuration.
405       *
406       * @param config a Configuration-format XML job description file.
407       */
408      public JobConf(Path config) {
409        super();
410        addResource(config);
411        checkAndWarnDeprecation();
412      }
413    
414      /** A new map/reduce configuration where the behavior of reading from the
415       * default resources can be turned off.
416       * <p/>
417       * If the parameter {@code loadDefaults} is false, the new instance
418       * will not load resources from the default files.
419       *
420       * @param loadDefaults specifies whether to load from the default files
421       */
422      public JobConf(boolean loadDefaults) {
423        super(loadDefaults);
424        checkAndWarnDeprecation();
425      }
426    
427      /**
428       * Get credentials for the job.
429       * @return credentials for the job
430       */
431      public Credentials getCredentials() {
432        return credentials;
433      }
434      
435      void setCredentials(Credentials credentials) {
436        this.credentials = credentials;
437      }
438      
439      /**
440       * Get the user jar for the map-reduce job.
441       * 
442       * @return the user jar for the map-reduce job.
443       */
444      public String getJar() { return get(JobContext.JAR); }
445      
446      /**
447       * Set the user jar for the map-reduce job.
448       * 
449       * @param jar the user jar for the map-reduce job.
450       */
451      public void setJar(String jar) { set(JobContext.JAR, jar); }
452    
453      /**
454       * Get the pattern for jar contents to unpack on the tasktracker
455       */
456      public Pattern getJarUnpackPattern() {
457        return getPattern(JobContext.JAR_UNPACK_PATTERN, UNPACK_JAR_PATTERN_DEFAULT);
458      }
459    
460      
461      /**
462       * Set the job's jar file by finding an example class location.
463       * 
464       * @param cls the example class.
465       */
466      public void setJarByClass(Class cls) {
467        String jar = findContainingJar(cls);
468        if (jar != null) {
469          setJar(jar);
470        }   
471      }
472    
473      public String[] getLocalDirs() throws IOException {
474        return getTrimmedStrings(MRConfig.LOCAL_DIR);
475      }
476    
477      /**
478       * Use MRAsyncDiskService.moveAndDeleteAllVolumes instead.
479       */
480      @Deprecated
481      public void deleteLocalFiles() throws IOException {
482        String[] localDirs = getLocalDirs();
483        for (int i = 0; i < localDirs.length; i++) {
484          FileSystem.getLocal(this).delete(new Path(localDirs[i]), true);
485        }
486      }
487    
488      public void deleteLocalFiles(String subdir) throws IOException {
489        String[] localDirs = getLocalDirs();
490        for (int i = 0; i < localDirs.length; i++) {
491          FileSystem.getLocal(this).delete(new Path(localDirs[i], subdir), true);
492        }
493      }
494    
495      /** 
496       * Constructs a local file name. Files are distributed among configured
497       * local directories.
498       */
499      public Path getLocalPath(String pathString) throws IOException {
500        return getLocalPath(MRConfig.LOCAL_DIR, pathString);
501      }
502    
503      /**
504       * Get the reported username for this job.
505       * 
506       * @return the username
507       */
508      public String getUser() {
509        return get(JobContext.USER_NAME);
510      }
511      
512      /**
513       * Set the reported username for this job.
514       * 
515       * @param user the username for this job.
516       */
517      public void setUser(String user) {
518        set(JobContext.USER_NAME, user);
519      }
520    
521    
522      
523      /**
524       * Set whether the framework should keep the intermediate files for 
525       * failed tasks.
526       * 
527       * @param keep <code>true</code> if framework should keep the intermediate files 
528       *             for failed tasks, <code>false</code> otherwise.
529       * 
530       */
531      public void setKeepFailedTaskFiles(boolean keep) {
532        setBoolean(JobContext.PRESERVE_FAILED_TASK_FILES, keep);
533      }
534      
535      /**
536       * Should the temporary files for failed tasks be kept?
537       * 
538       * @return should the files be kept?
539       */
540      public boolean getKeepFailedTaskFiles() {
541        return getBoolean(JobContext.PRESERVE_FAILED_TASK_FILES, false);
542      }
543      
544      /**
545       * Set a regular expression for task names that should be kept. 
546       * The regular expression ".*_m_000123_0" would keep the files
547       * for the first instance of map 123 that ran.
548       * 
549       * @param pattern the java.util.regex.Pattern to match against the 
550       *        task names.
551       */
552      public void setKeepTaskFilesPattern(String pattern) {
553        set(JobContext.PRESERVE_FILES_PATTERN, pattern);
554      }
555      
556      /**
557       * Get the regular expression that is matched against the task names
558       * to see if we need to keep the files.
559       * 
560       * @return the pattern as a string, if it was set, othewise null.
561       */
562      public String getKeepTaskFilesPattern() {
563        return get(JobContext.PRESERVE_FILES_PATTERN);
564      }
565      
566      /**
567       * Set the current working directory for the default file system.
568       * 
569       * @param dir the new current working directory.
570       */
571      public void setWorkingDirectory(Path dir) {
572        dir = new Path(getWorkingDirectory(), dir);
573        set(JobContext.WORKING_DIR, dir.toString());
574      }
575      
576      /**
577       * Get the current working directory for the default file system.
578       * 
579       * @return the directory name.
580       */
581      public Path getWorkingDirectory() {
582        String name = get(JobContext.WORKING_DIR);
583        if (name != null) {
584          return new Path(name);
585        } else {
586          try {
587            Path dir = FileSystem.get(this).getWorkingDirectory();
588            set(JobContext.WORKING_DIR, dir.toString());
589            return dir;
590          } catch (IOException e) {
591            throw new RuntimeException(e);
592          }
593        }
594      }
595      
596      /**
597       * Sets the number of tasks that a spawned task JVM should run
598       * before it exits
599       * @param numTasks the number of tasks to execute; defaults to 1;
600       * -1 signifies no limit
601       */
602      public void setNumTasksToExecutePerJvm(int numTasks) {
603        setInt(JobContext.JVM_NUMTASKS_TORUN, numTasks);
604      }
605      
606      /**
607       * Get the number of tasks that a spawned JVM should execute
608       */
609      public int getNumTasksToExecutePerJvm() {
610        return getInt(JobContext.JVM_NUMTASKS_TORUN, 1);
611      }
612      
613      /**
614       * Get the {@link InputFormat} implementation for the map-reduce job,
615       * defaults to {@link TextInputFormat} if not specified explicity.
616       * 
617       * @return the {@link InputFormat} implementation for the map-reduce job.
618       */
619      public InputFormat getInputFormat() {
620        return ReflectionUtils.newInstance(getClass("mapred.input.format.class",
621                                                                 TextInputFormat.class,
622                                                                 InputFormat.class),
623                                                        this);
624      }
625      
626      /**
627       * Set the {@link InputFormat} implementation for the map-reduce job.
628       * 
629       * @param theClass the {@link InputFormat} implementation for the map-reduce 
630       *                 job.
631       */
632      public void setInputFormat(Class<? extends InputFormat> theClass) {
633        setClass("mapred.input.format.class", theClass, InputFormat.class);
634      }
635      
636      /**
637       * Get the {@link OutputFormat} implementation for the map-reduce job,
638       * defaults to {@link TextOutputFormat} if not specified explicity.
639       * 
640       * @return the {@link OutputFormat} implementation for the map-reduce job.
641       */
642      public OutputFormat getOutputFormat() {
643        return ReflectionUtils.newInstance(getClass("mapred.output.format.class",
644                                                                  TextOutputFormat.class,
645                                                                  OutputFormat.class),
646                                                         this);
647      }
648    
649      /**
650       * Get the {@link OutputCommitter} implementation for the map-reduce job,
651       * defaults to {@link FileOutputCommitter} if not specified explicitly.
652       * 
653       * @return the {@link OutputCommitter} implementation for the map-reduce job.
654       */
655      public OutputCommitter getOutputCommitter() {
656        return (OutputCommitter)ReflectionUtils.newInstance(
657          getClass("mapred.output.committer.class", FileOutputCommitter.class,
658                   OutputCommitter.class), this);
659      }
660    
661      /**
662       * Set the {@link OutputCommitter} implementation for the map-reduce job.
663       * 
664       * @param theClass the {@link OutputCommitter} implementation for the map-reduce 
665       *                 job.
666       */
667      public void setOutputCommitter(Class<? extends OutputCommitter> theClass) {
668        setClass("mapred.output.committer.class", theClass, OutputCommitter.class);
669      }
670      
671      /**
672       * Set the {@link OutputFormat} implementation for the map-reduce job.
673       * 
674       * @param theClass the {@link OutputFormat} implementation for the map-reduce 
675       *                 job.
676       */
677      public void setOutputFormat(Class<? extends OutputFormat> theClass) {
678        setClass("mapred.output.format.class", theClass, OutputFormat.class);
679      }
680    
681      /**
682       * Should the map outputs be compressed before transfer?
683       * Uses the SequenceFile compression.
684       * 
685       * @param compress should the map outputs be compressed?
686       */
687      public void setCompressMapOutput(boolean compress) {
688        setBoolean(JobContext.MAP_OUTPUT_COMPRESS, compress);
689      }
690      
691      /**
692       * Are the outputs of the maps be compressed?
693       * 
694       * @return <code>true</code> if the outputs of the maps are to be compressed,
695       *         <code>false</code> otherwise.
696       */
697      public boolean getCompressMapOutput() {
698        return getBoolean(JobContext.MAP_OUTPUT_COMPRESS, false);
699      }
700    
701      /**
702       * Set the given class as the  {@link CompressionCodec} for the map outputs.
703       * 
704       * @param codecClass the {@link CompressionCodec} class that will compress  
705       *                   the map outputs.
706       */
707      public void 
708      setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) {
709        setCompressMapOutput(true);
710        setClass(JobContext.MAP_OUTPUT_COMPRESS_CODEC, codecClass, 
711                 CompressionCodec.class);
712      }
713      
714      /**
715       * Get the {@link CompressionCodec} for compressing the map outputs.
716       * 
717       * @param defaultValue the {@link CompressionCodec} to return if not set
718       * @return the {@link CompressionCodec} class that should be used to compress the 
719       *         map outputs.
720       * @throws IllegalArgumentException if the class was specified, but not found
721       */
722      public Class<? extends CompressionCodec> 
723      getMapOutputCompressorClass(Class<? extends CompressionCodec> defaultValue) {
724        Class<? extends CompressionCodec> codecClass = defaultValue;
725        String name = get(JobContext.MAP_OUTPUT_COMPRESS_CODEC);
726        if (name != null) {
727          try {
728            codecClass = getClassByName(name).asSubclass(CompressionCodec.class);
729          } catch (ClassNotFoundException e) {
730            throw new IllegalArgumentException("Compression codec " + name + 
731                                               " was not found.", e);
732          }
733        }
734        return codecClass;
735      }
736      
737      /**
738       * Get the key class for the map output data. If it is not set, use the
739       * (final) output key class. This allows the map output key class to be
740       * different than the final output key class.
741       *  
742       * @return the map output key class.
743       */
744      public Class<?> getMapOutputKeyClass() {
745        Class<?> retv = getClass(JobContext.MAP_OUTPUT_KEY_CLASS, null, Object.class);
746        if (retv == null) {
747          retv = getOutputKeyClass();
748        }
749        return retv;
750      }
751      
752      /**
753       * Set the key class for the map output data. This allows the user to
754       * specify the map output key class to be different than the final output
755       * value class.
756       * 
757       * @param theClass the map output key class.
758       */
759      public void setMapOutputKeyClass(Class<?> theClass) {
760        setClass(JobContext.MAP_OUTPUT_KEY_CLASS, theClass, Object.class);
761      }
762      
763      /**
764       * Get the value class for the map output data. If it is not set, use the
765       * (final) output value class This allows the map output value class to be
766       * different than the final output value class.
767       *  
768       * @return the map output value class.
769       */
770      public Class<?> getMapOutputValueClass() {
771        Class<?> retv = getClass(JobContext.MAP_OUTPUT_VALUE_CLASS, null,
772            Object.class);
773        if (retv == null) {
774          retv = getOutputValueClass();
775        }
776        return retv;
777      }
778      
779      /**
780       * Set the value class for the map output data. This allows the user to
781       * specify the map output value class to be different than the final output
782       * value class.
783       * 
784       * @param theClass the map output value class.
785       */
786      public void setMapOutputValueClass(Class<?> theClass) {
787        setClass(JobContext.MAP_OUTPUT_VALUE_CLASS, theClass, Object.class);
788      }
789      
790      /**
791       * Get the key class for the job output data.
792       * 
793       * @return the key class for the job output data.
794       */
795      public Class<?> getOutputKeyClass() {
796        return getClass(JobContext.OUTPUT_KEY_CLASS,
797                        LongWritable.class, Object.class);
798      }
799      
800      /**
801       * Set the key class for the job output data.
802       * 
803       * @param theClass the key class for the job output data.
804       */
805      public void setOutputKeyClass(Class<?> theClass) {
806        setClass(JobContext.OUTPUT_KEY_CLASS, theClass, Object.class);
807      }
808    
809      /**
810       * Get the {@link RawComparator} comparator used to compare keys.
811       * 
812       * @return the {@link RawComparator} comparator used to compare keys.
813       */
814      public RawComparator getOutputKeyComparator() {
815        Class<? extends RawComparator> theClass = getClass(
816          JobContext.KEY_COMPARATOR, null, RawComparator.class);
817        if (theClass != null)
818          return ReflectionUtils.newInstance(theClass, this);
819        return WritableComparator.get(getMapOutputKeyClass().asSubclass(WritableComparable.class));
820      }
821    
822      /**
823       * Set the {@link RawComparator} comparator used to compare keys.
824       * 
825       * @param theClass the {@link RawComparator} comparator used to 
826       *                 compare keys.
827       * @see #setOutputValueGroupingComparator(Class)                 
828       */
829      public void setOutputKeyComparatorClass(Class<? extends RawComparator> theClass) {
830        setClass(JobContext.KEY_COMPARATOR,
831                 theClass, RawComparator.class);
832      }
833    
834      /**
835       * Set the {@link KeyFieldBasedComparator} options used to compare keys.
836       * 
837       * @param keySpec the key specification of the form -k pos1[,pos2], where,
838       *  pos is of the form f[.c][opts], where f is the number
839       *  of the key field to use, and c is the number of the first character from
840       *  the beginning of the field. Fields and character posns are numbered 
841       *  starting with 1; a character position of zero in pos2 indicates the
842       *  field's last character. If '.c' is omitted from pos1, it defaults to 1
843       *  (the beginning of the field); if omitted from pos2, it defaults to 0 
844       *  (the end of the field). opts are ordering options. The supported options
845       *  are:
846       *    -n, (Sort numerically)
847       *    -r, (Reverse the result of comparison)                 
848       */
849      public void setKeyFieldComparatorOptions(String keySpec) {
850        setOutputKeyComparatorClass(KeyFieldBasedComparator.class);
851        set(KeyFieldBasedComparator.COMPARATOR_OPTIONS, keySpec);
852      }
853      
854      /**
855       * Get the {@link KeyFieldBasedComparator} options
856       */
857      public String getKeyFieldComparatorOption() {
858        return get(KeyFieldBasedComparator.COMPARATOR_OPTIONS);
859      }
860    
861      /**
862       * Set the {@link KeyFieldBasedPartitioner} options used for 
863       * {@link Partitioner}
864       * 
865       * @param keySpec the key specification of the form -k pos1[,pos2], where,
866       *  pos is of the form f[.c][opts], where f is the number
867       *  of the key field to use, and c is the number of the first character from
868       *  the beginning of the field. Fields and character posns are numbered 
869       *  starting with 1; a character position of zero in pos2 indicates the
870       *  field's last character. If '.c' is omitted from pos1, it defaults to 1
871       *  (the beginning of the field); if omitted from pos2, it defaults to 0 
872       *  (the end of the field).
873       */
874      public void setKeyFieldPartitionerOptions(String keySpec) {
875        setPartitionerClass(KeyFieldBasedPartitioner.class);
876        set(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS, keySpec);
877      }
878      
879      /**
880       * Get the {@link KeyFieldBasedPartitioner} options
881       */
882      public String getKeyFieldPartitionerOption() {
883        return get(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS);
884      }
885    
886      /** 
887       * Get the user defined {@link WritableComparable} comparator for 
888       * grouping keys of inputs to the reduce.
889       * 
890       * @return comparator set by the user for grouping values.
891       * @see #setOutputValueGroupingComparator(Class) for details.  
892       */
893      public RawComparator getOutputValueGroupingComparator() {
894        Class<? extends RawComparator> theClass = getClass(
895          JobContext.GROUP_COMPARATOR_CLASS, null, RawComparator.class);
896        if (theClass == null) {
897          return getOutputKeyComparator();
898        }
899        
900        return ReflectionUtils.newInstance(theClass, this);
901      }
902    
903      /** 
904       * Set the user defined {@link RawComparator} comparator for 
905       * grouping keys in the input to the reduce.
906       * 
907       * <p>This comparator should be provided if the equivalence rules for keys
908       * for sorting the intermediates are different from those for grouping keys
909       * before each call to 
910       * {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.</p>
911       *  
912       * <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
913       * in a single call to the reduce function if K1 and K2 compare as equal.</p>
914       * 
915       * <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to control 
916       * how keys are sorted, this can be used in conjunction to simulate 
917       * <i>secondary sort on values</i>.</p>
918       *  
919       * <p><i>Note</i>: This is not a guarantee of the reduce sort being 
920       * <i>stable</i> in any sense. (In any case, with the order of available 
921       * map-outputs to the reduce being non-deterministic, it wouldn't make 
922       * that much sense.)</p>
923       * 
924       * @param theClass the comparator class to be used for grouping keys. 
925       *                 It should implement <code>RawComparator</code>.
926       * @see #setOutputKeyComparatorClass(Class)                 
927       */
928      public void setOutputValueGroupingComparator(
929          Class<? extends RawComparator> theClass) {
930        setClass(JobContext.GROUP_COMPARATOR_CLASS,
931                 theClass, RawComparator.class);
932      }
933    
934      /**
935       * Should the framework use the new context-object code for running
936       * the mapper?
937       * @return true, if the new api should be used
938       */
939      public boolean getUseNewMapper() {
940        return getBoolean("mapred.mapper.new-api", false);
941      }
942      /**
943       * Set whether the framework should use the new api for the mapper.
944       * This is the default for jobs submitted with the new Job api.
945       * @param flag true, if the new api should be used
946       */
947      public void setUseNewMapper(boolean flag) {
948        setBoolean("mapred.mapper.new-api", flag);
949      }
950    
951      /**
952       * Should the framework use the new context-object code for running
953       * the reducer?
954       * @return true, if the new api should be used
955       */
956      public boolean getUseNewReducer() {
957        return getBoolean("mapred.reducer.new-api", false);
958      }
959      /**
960       * Set whether the framework should use the new api for the reducer. 
961       * This is the default for jobs submitted with the new Job api.
962       * @param flag true, if the new api should be used
963       */
964      public void setUseNewReducer(boolean flag) {
965        setBoolean("mapred.reducer.new-api", flag);
966      }
967    
968      /**
969       * Get the value class for job outputs.
970       * 
971       * @return the value class for job outputs.
972       */
973      public Class<?> getOutputValueClass() {
974        return getClass(JobContext.OUTPUT_VALUE_CLASS, Text.class, Object.class);
975      }
976      
977      /**
978       * Set the value class for job outputs.
979       * 
980       * @param theClass the value class for job outputs.
981       */
982      public void setOutputValueClass(Class<?> theClass) {
983        setClass(JobContext.OUTPUT_VALUE_CLASS, theClass, Object.class);
984      }
985    
986      /**
987       * Get the {@link Mapper} class for the job.
988       * 
989       * @return the {@link Mapper} class for the job.
990       */
991      public Class<? extends Mapper> getMapperClass() {
992        return getClass("mapred.mapper.class", IdentityMapper.class, Mapper.class);
993      }
994      
995      /**
996       * Set the {@link Mapper} class for the job.
997       * 
998       * @param theClass the {@link Mapper} class for the job.
999       */
1000      public void setMapperClass(Class<? extends Mapper> theClass) {
1001        setClass("mapred.mapper.class", theClass, Mapper.class);
1002      }
1003    
1004      /**
1005       * Get the {@link MapRunnable} class for the job.
1006       * 
1007       * @return the {@link MapRunnable} class for the job.
1008       */
1009      public Class<? extends MapRunnable> getMapRunnerClass() {
1010        return getClass("mapred.map.runner.class",
1011                        MapRunner.class, MapRunnable.class);
1012      }
1013      
1014      /**
1015       * Expert: Set the {@link MapRunnable} class for the job.
1016       * 
1017       * Typically used to exert greater control on {@link Mapper}s.
1018       * 
1019       * @param theClass the {@link MapRunnable} class for the job.
1020       */
1021      public void setMapRunnerClass(Class<? extends MapRunnable> theClass) {
1022        setClass("mapred.map.runner.class", theClass, MapRunnable.class);
1023      }
1024    
1025      /**
1026       * Get the {@link Partitioner} used to partition {@link Mapper}-outputs 
1027       * to be sent to the {@link Reducer}s.
1028       * 
1029       * @return the {@link Partitioner} used to partition map-outputs.
1030       */
1031      public Class<? extends Partitioner> getPartitionerClass() {
1032        return getClass("mapred.partitioner.class",
1033                        HashPartitioner.class, Partitioner.class);
1034      }
1035      
1036      /**
1037       * Set the {@link Partitioner} class used to partition 
1038       * {@link Mapper}-outputs to be sent to the {@link Reducer}s.
1039       * 
1040       * @param theClass the {@link Partitioner} used to partition map-outputs.
1041       */
1042      public void setPartitionerClass(Class<? extends Partitioner> theClass) {
1043        setClass("mapred.partitioner.class", theClass, Partitioner.class);
1044      }
1045    
1046      /**
1047       * Get the {@link Reducer} class for the job.
1048       * 
1049       * @return the {@link Reducer} class for the job.
1050       */
1051      public Class<? extends Reducer> getReducerClass() {
1052        return getClass("mapred.reducer.class",
1053                        IdentityReducer.class, Reducer.class);
1054      }
1055      
1056      /**
1057       * Set the {@link Reducer} class for the job.
1058       * 
1059       * @param theClass the {@link Reducer} class for the job.
1060       */
1061      public void setReducerClass(Class<? extends Reducer> theClass) {
1062        setClass("mapred.reducer.class", theClass, Reducer.class);
1063      }
1064    
1065      /**
1066       * Get the user-defined <i>combiner</i> class used to combine map-outputs 
1067       * before being sent to the reducers. Typically the combiner is same as the
1068       * the {@link Reducer} for the job i.e. {@link #getReducerClass()}.
1069       * 
1070       * @return the user-defined combiner class used to combine map-outputs.
1071       */
1072      public Class<? extends Reducer> getCombinerClass() {
1073        return getClass("mapred.combiner.class", null, Reducer.class);
1074      }
1075    
1076      /**
1077       * Set the user-defined <i>combiner</i> class used to combine map-outputs 
1078       * before being sent to the reducers. 
1079       * 
1080       * <p>The combiner is an application-specified aggregation operation, which
1081       * can help cut down the amount of data transferred between the 
1082       * {@link Mapper} and the {@link Reducer}, leading to better performance.</p>
1083       * 
1084       * <p>The framework may invoke the combiner 0, 1, or multiple times, in both
1085       * the mapper and reducer tasks. In general, the combiner is called as the
1086       * sort/merge result is written to disk. The combiner must:
1087       * <ul>
1088       *   <li> be side-effect free</li>
1089       *   <li> have the same input and output key types and the same input and 
1090       *        output value types</li>
1091       * </ul></p>
1092       * 
1093       * <p>Typically the combiner is same as the <code>Reducer</code> for the  
1094       * job i.e. {@link #setReducerClass(Class)}.</p>
1095       * 
1096       * @param theClass the user-defined combiner class used to combine 
1097       *                 map-outputs.
1098       */
1099      public void setCombinerClass(Class<? extends Reducer> theClass) {
1100        setClass("mapred.combiner.class", theClass, Reducer.class);
1101      }
1102      
1103      /**
1104       * Should speculative execution be used for this job? 
1105       * Defaults to <code>true</code>.
1106       * 
1107       * @return <code>true</code> if speculative execution be used for this job,
1108       *         <code>false</code> otherwise.
1109       */
1110      public boolean getSpeculativeExecution() { 
1111        return (getMapSpeculativeExecution() || getReduceSpeculativeExecution());
1112      }
1113      
1114      /**
1115       * Turn speculative execution on or off for this job. 
1116       * 
1117       * @param speculativeExecution <code>true</code> if speculative execution 
1118       *                             should be turned on, else <code>false</code>.
1119       */
1120      public void setSpeculativeExecution(boolean speculativeExecution) {
1121        setMapSpeculativeExecution(speculativeExecution);
1122        setReduceSpeculativeExecution(speculativeExecution);
1123      }
1124    
1125      /**
1126       * Should speculative execution be used for this job for map tasks? 
1127       * Defaults to <code>true</code>.
1128       * 
1129       * @return <code>true</code> if speculative execution be 
1130       *                           used for this job for map tasks,
1131       *         <code>false</code> otherwise.
1132       */
1133      public boolean getMapSpeculativeExecution() { 
1134        return getBoolean(JobContext.MAP_SPECULATIVE, true);
1135      }
1136      
1137      /**
1138       * Turn speculative execution on or off for this job for map tasks. 
1139       * 
1140       * @param speculativeExecution <code>true</code> if speculative execution 
1141       *                             should be turned on for map tasks,
1142       *                             else <code>false</code>.
1143       */
1144      public void setMapSpeculativeExecution(boolean speculativeExecution) {
1145        setBoolean(JobContext.MAP_SPECULATIVE, speculativeExecution);
1146      }
1147    
1148      /**
1149       * Should speculative execution be used for this job for reduce tasks? 
1150       * Defaults to <code>true</code>.
1151       * 
1152       * @return <code>true</code> if speculative execution be used 
1153       *                           for reduce tasks for this job,
1154       *         <code>false</code> otherwise.
1155       */
1156      public boolean getReduceSpeculativeExecution() { 
1157        return getBoolean(JobContext.REDUCE_SPECULATIVE, true);
1158      }
1159      
1160      /**
1161       * Turn speculative execution on or off for this job for reduce tasks. 
1162       * 
1163       * @param speculativeExecution <code>true</code> if speculative execution 
1164       *                             should be turned on for reduce tasks,
1165       *                             else <code>false</code>.
1166       */
1167      public void setReduceSpeculativeExecution(boolean speculativeExecution) {
1168        setBoolean(JobContext.REDUCE_SPECULATIVE, 
1169                   speculativeExecution);
1170      }
1171    
1172      /**
1173       * Get configured the number of reduce tasks for this job.
1174       * Defaults to <code>1</code>.
1175       * 
1176       * @return the number of reduce tasks for this job.
1177       */
1178      public int getNumMapTasks() { return getInt(JobContext.NUM_MAPS, 1); }
1179      
1180      /**
1181       * Set the number of map tasks for this job.
1182       * 
1183       * <p><i>Note</i>: This is only a <i>hint</i> to the framework. The actual 
1184       * number of spawned map tasks depends on the number of {@link InputSplit}s 
1185       * generated by the job's {@link InputFormat#getSplits(JobConf, int)}.
1186       *  
1187       * A custom {@link InputFormat} is typically used to accurately control 
1188       * the number of map tasks for the job.</p>
1189       * 
1190       * <h4 id="NoOfMaps">How many maps?</h4>
1191       * 
1192       * <p>The number of maps is usually driven by the total size of the inputs 
1193       * i.e. total number of blocks of the input files.</p>
1194       *  
1195       * <p>The right level of parallelism for maps seems to be around 10-100 maps 
1196       * per-node, although it has been set up to 300 or so for very cpu-light map 
1197       * tasks. Task setup takes awhile, so it is best if the maps take at least a 
1198       * minute to execute.</p>
1199       * 
1200       * <p>The default behavior of file-based {@link InputFormat}s is to split the 
1201       * input into <i>logical</i> {@link InputSplit}s based on the total size, in 
1202       * bytes, of input files. However, the {@link FileSystem} blocksize of the 
1203       * input files is treated as an upper bound for input splits. A lower bound 
1204       * on the split size can be set via 
1205       * <a href="{@docRoot}/../mapred-default.html#mapreduce.input.fileinputformat.split.minsize">
1206       * mapreduce.input.fileinputformat.split.minsize</a>.</p>
1207       *  
1208       * <p>Thus, if you expect 10TB of input data and have a blocksize of 128MB, 
1209       * you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is 
1210       * used to set it even higher.</p>
1211       * 
1212       * @param n the number of map tasks for this job.
1213       * @see InputFormat#getSplits(JobConf, int)
1214       * @see FileInputFormat
1215       * @see FileSystem#getDefaultBlockSize()
1216       * @see FileStatus#getBlockSize()
1217       */
1218      public void setNumMapTasks(int n) { setInt(JobContext.NUM_MAPS, n); }
1219    
1220      /**
1221       * Get configured the number of reduce tasks for this job. Defaults to 
1222       * <code>1</code>.
1223       * 
1224       * @return the number of reduce tasks for this job.
1225       */
1226      public int getNumReduceTasks() { return getInt(JobContext.NUM_REDUCES, 1); }
1227      
1228      /**
1229       * Set the requisite number of reduce tasks for this job.
1230       * 
1231       * <h4 id="NoOfReduces">How many reduces?</h4>
1232       * 
1233       * <p>The right number of reduces seems to be <code>0.95</code> or 
1234       * <code>1.75</code> multiplied by (&lt;<i>no. of nodes</i>&gt; * 
1235       * <a href="{@docRoot}/../mapred-default.html#mapreduce.tasktracker.reduce.tasks.maximum">
1236       * mapreduce.tasktracker.reduce.tasks.maximum</a>).
1237       * </p>
1238       * 
1239       * <p>With <code>0.95</code> all of the reduces can launch immediately and 
1240       * start transfering map outputs as the maps finish. With <code>1.75</code> 
1241       * the faster nodes will finish their first round of reduces and launch a 
1242       * second wave of reduces doing a much better job of load balancing.</p>
1243       * 
1244       * <p>Increasing the number of reduces increases the framework overhead, but 
1245       * increases load balancing and lowers the cost of failures.</p>
1246       * 
1247       * <p>The scaling factors above are slightly less than whole numbers to 
1248       * reserve a few reduce slots in the framework for speculative-tasks, failures
1249       * etc.</p> 
1250       *
1251       * <h4 id="ReducerNone">Reducer NONE</h4>
1252       * 
1253       * <p>It is legal to set the number of reduce-tasks to <code>zero</code>.</p>
1254       * 
1255       * <p>In this case the output of the map-tasks directly go to distributed 
1256       * file-system, to the path set by 
1257       * {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the 
1258       * framework doesn't sort the map-outputs before writing it out to HDFS.</p>
1259       * 
1260       * @param n the number of reduce tasks for this job.
1261       */
1262      public void setNumReduceTasks(int n) { setInt(JobContext.NUM_REDUCES, n); }
1263      
1264      /** 
1265       * Get the configured number of maximum attempts that will be made to run a
1266       * map task, as specified by the <code>mapreduce.map.maxattempts</code>
1267       * property. If this property is not already set, the default is 4 attempts.
1268       *  
1269       * @return the max number of attempts per map task.
1270       */
1271      public int getMaxMapAttempts() {
1272        return getInt(JobContext.MAP_MAX_ATTEMPTS, 4);
1273      }
1274      
1275      /** 
1276       * Expert: Set the number of maximum attempts that will be made to run a
1277       * map task.
1278       * 
1279       * @param n the number of attempts per map task.
1280       */
1281      public void setMaxMapAttempts(int n) {
1282        setInt(JobContext.MAP_MAX_ATTEMPTS, n);
1283      }
1284    
1285      /** 
1286       * Get the configured number of maximum attempts  that will be made to run a
1287       * reduce task, as specified by the <code>mapreduce.reduce.maxattempts</code>
1288       * property. If this property is not already set, the default is 4 attempts.
1289       * 
1290       * @return the max number of attempts per reduce task.
1291       */
1292      public int getMaxReduceAttempts() {
1293        return getInt(JobContext.REDUCE_MAX_ATTEMPTS, 4);
1294      }
1295      /** 
1296       * Expert: Set the number of maximum attempts that will be made to run a
1297       * reduce task.
1298       * 
1299       * @param n the number of attempts per reduce task.
1300       */
1301      public void setMaxReduceAttempts(int n) {
1302        setInt(JobContext.REDUCE_MAX_ATTEMPTS, n);
1303      }
1304      
1305      /**
1306       * Get the user-specified job name. This is only used to identify the 
1307       * job to the user.
1308       * 
1309       * @return the job's name, defaulting to "".
1310       */
1311      public String getJobName() {
1312        return get(JobContext.JOB_NAME, "");
1313      }
1314      
1315      /**
1316       * Set the user-specified job name.
1317       * 
1318       * @param name the job's new name.
1319       */
1320      public void setJobName(String name) {
1321        set(JobContext.JOB_NAME, name);
1322      }
1323      
1324      /**
1325       * Get the user-specified session identifier. The default is the empty string.
1326       *
1327       * The session identifier is used to tag metric data that is reported to some
1328       * performance metrics system via the org.apache.hadoop.metrics API.  The 
1329       * session identifier is intended, in particular, for use by Hadoop-On-Demand 
1330       * (HOD) which allocates a virtual Hadoop cluster dynamically and transiently. 
1331       * HOD will set the session identifier by modifying the mapred-site.xml file 
1332       * before starting the cluster.
1333       *
1334       * When not running under HOD, this identifer is expected to remain set to 
1335       * the empty string.
1336       *
1337       * @return the session identifier, defaulting to "".
1338       */
1339      @Deprecated
1340      public String getSessionId() {
1341          return get("session.id", "");
1342      }
1343      
1344      /**
1345       * Set the user-specified session identifier.  
1346       *
1347       * @param sessionId the new session id.
1348       */
1349      @Deprecated
1350      public void setSessionId(String sessionId) {
1351          set("session.id", sessionId);
1352      }
1353        
1354      /**
1355       * Set the maximum no. of failures of a given job per tasktracker.
1356       * If the no. of task failures exceeds <code>noFailures</code>, the 
1357       * tasktracker is <i>blacklisted</i> for this job. 
1358       * 
1359       * @param noFailures maximum no. of failures of a given job per tasktracker.
1360       */
1361      public void setMaxTaskFailuresPerTracker(int noFailures) {
1362        setInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, noFailures);
1363      }
1364      
1365      /**
1366       * Expert: Get the maximum no. of failures of a given job per tasktracker.
1367       * If the no. of task failures exceeds this, the tasktracker is
1368       * <i>blacklisted</i> for this job. 
1369       * 
1370       * @return the maximum no. of failures of a given job per tasktracker.
1371       */
1372      public int getMaxTaskFailuresPerTracker() {
1373        return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 4); 
1374      }
1375    
1376      /**
1377       * Get the maximum percentage of map tasks that can fail without 
1378       * the job being aborted. 
1379       * 
1380       * Each map task is executed a minimum of {@link #getMaxMapAttempts()} 
1381       * attempts before being declared as <i>failed</i>.
1382       *  
1383       * Defaults to <code>zero</code>, i.e. <i>any</i> failed map-task results in
1384       * the job being declared as {@link JobStatus#FAILED}.
1385       * 
1386       * @return the maximum percentage of map tasks that can fail without
1387       *         the job being aborted.
1388       */
1389      public int getMaxMapTaskFailuresPercent() {
1390        return getInt(JobContext.MAP_FAILURES_MAX_PERCENT, 0);
1391      }
1392    
1393      /**
1394       * Expert: Set the maximum percentage of map tasks that can fail without the
1395       * job being aborted. 
1396       * 
1397       * Each map task is executed a minimum of {@link #getMaxMapAttempts} attempts 
1398       * before being declared as <i>failed</i>.
1399       * 
1400       * @param percent the maximum percentage of map tasks that can fail without 
1401       *                the job being aborted.
1402       */
1403      public void setMaxMapTaskFailuresPercent(int percent) {
1404        setInt(JobContext.MAP_FAILURES_MAX_PERCENT, percent);
1405      }
1406      
1407      /**
1408       * Get the maximum percentage of reduce tasks that can fail without 
1409       * the job being aborted. 
1410       * 
1411       * Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
1412       * attempts before being declared as <i>failed</i>.
1413       * 
1414       * Defaults to <code>zero</code>, i.e. <i>any</i> failed reduce-task results 
1415       * in the job being declared as {@link JobStatus#FAILED}.
1416       * 
1417       * @return the maximum percentage of reduce tasks that can fail without
1418       *         the job being aborted.
1419       */
1420      public int getMaxReduceTaskFailuresPercent() {
1421        return getInt(JobContext.REDUCE_FAILURES_MAXPERCENT, 0);
1422      }
1423      
1424      /**
1425       * Set the maximum percentage of reduce tasks that can fail without the job
1426       * being aborted.
1427       * 
1428       * Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
1429       * attempts before being declared as <i>failed</i>.
1430       * 
1431       * @param percent the maximum percentage of reduce tasks that can fail without 
1432       *                the job being aborted.
1433       */
1434      public void setMaxReduceTaskFailuresPercent(int percent) {
1435        setInt(JobContext.REDUCE_FAILURES_MAXPERCENT, percent);
1436      }
1437      
1438      /**
1439       * Set {@link JobPriority} for this job.
1440       * 
1441       * @param prio the {@link JobPriority} for this job.
1442       */
1443      public void setJobPriority(JobPriority prio) {
1444        set(JobContext.PRIORITY, prio.toString());
1445      }
1446      
1447      /**
1448       * Get the {@link JobPriority} for this job.
1449       * 
1450       * @return the {@link JobPriority} for this job.
1451       */
1452      public JobPriority getJobPriority() {
1453        String prio = get(JobContext.PRIORITY);
1454        if(prio == null) {
1455          return JobPriority.NORMAL;
1456        }
1457        
1458        return JobPriority.valueOf(prio);
1459      }
1460    
1461      /**
1462       * Set JobSubmitHostName for this job.
1463       * 
1464       * @param hostname the JobSubmitHostName for this job.
1465       */
1466      void setJobSubmitHostName(String hostname) {
1467        set(MRJobConfig.JOB_SUBMITHOST, hostname);
1468      }
1469      
1470      /**
1471       * Get the  JobSubmitHostName for this job.
1472       * 
1473       * @return the JobSubmitHostName for this job.
1474       */
1475      String getJobSubmitHostName() {
1476        String hostname = get(MRJobConfig.JOB_SUBMITHOST);
1477        
1478        return hostname;
1479      }
1480    
1481      /**
1482       * Set JobSubmitHostAddress for this job.
1483       * 
1484       * @param hostadd the JobSubmitHostAddress for this job.
1485       */
1486      void setJobSubmitHostAddress(String hostadd) {
1487        set(MRJobConfig.JOB_SUBMITHOSTADDR, hostadd);
1488      }
1489      
1490      /**
1491       * Get JobSubmitHostAddress for this job.
1492       * 
1493       * @return  JobSubmitHostAddress for this job.
1494       */
1495      String getJobSubmitHostAddress() {
1496        String hostadd = get(MRJobConfig.JOB_SUBMITHOSTADDR);
1497        
1498        return hostadd;
1499      }
1500    
1501      /**
1502       * Get whether the task profiling is enabled.
1503       * @return true if some tasks will be profiled
1504       */
1505      public boolean getProfileEnabled() {
1506        return getBoolean(JobContext.TASK_PROFILE, false);
1507      }
1508    
1509      /**
1510       * Set whether the system should collect profiler information for some of 
1511       * the tasks in this job? The information is stored in the user log 
1512       * directory.
1513       * @param newValue true means it should be gathered
1514       */
1515      public void setProfileEnabled(boolean newValue) {
1516        setBoolean(JobContext.TASK_PROFILE, newValue);
1517      }
1518    
1519      /**
1520       * Get the profiler configuration arguments.
1521       *
1522       * The default value for this property is
1523       * "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s"
1524       * 
1525       * @return the parameters to pass to the task child to configure profiling
1526       */
1527      public String getProfileParams() {
1528        return get(JobContext.TASK_PROFILE_PARAMS,
1529                   "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y," +
1530                     "verbose=n,file=%s");
1531      }
1532    
1533      /**
1534       * Set the profiler configuration arguments. If the string contains a '%s' it
1535       * will be replaced with the name of the profiling output file when the task
1536       * runs.
1537       *
1538       * This value is passed to the task child JVM on the command line.
1539       *
1540       * @param value the configuration string
1541       */
1542      public void setProfileParams(String value) {
1543        set(JobContext.TASK_PROFILE_PARAMS, value);
1544      }
1545    
1546      /**
1547       * Get the range of maps or reduces to profile.
1548       * @param isMap is the task a map?
1549       * @return the task ranges
1550       */
1551      public IntegerRanges getProfileTaskRange(boolean isMap) {
1552        return getRange((isMap ? JobContext.NUM_MAP_PROFILES : 
1553                           JobContext.NUM_REDUCE_PROFILES), "0-2");
1554      }
1555    
1556      /**
1557       * Set the ranges of maps or reduces to profile. setProfileEnabled(true) 
1558       * must also be called.
1559       * @param newValue a set of integer ranges of the map ids
1560       */
1561      public void setProfileTaskRange(boolean isMap, String newValue) {
1562        // parse the value to make sure it is legal
1563          new Configuration.IntegerRanges(newValue);
1564        set((isMap ? JobContext.NUM_MAP_PROFILES : JobContext.NUM_REDUCE_PROFILES), 
1565              newValue);
1566      }
1567    
1568      /**
1569       * Set the debug script to run when the map tasks fail.
1570       * 
1571       * <p>The debug script can aid debugging of failed map tasks. The script is 
1572       * given task's stdout, stderr, syslog, jobconf files as arguments.</p>
1573       * 
1574       * <p>The debug command, run on the node where the map failed, is:</p>
1575       * <p><pre><blockquote> 
1576       * $script $stdout $stderr $syslog $jobconf.
1577       * </blockquote></pre></p>
1578       * 
1579       * <p> The script file is distributed through {@link DistributedCache} 
1580       * APIs. The script needs to be symlinked. </p>
1581       * 
1582       * <p>Here is an example on how to submit a script 
1583       * <p><blockquote><pre>
1584       * job.setMapDebugScript("./myscript");
1585       * DistributedCache.createSymlink(job);
1586       * DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
1587       * </pre></blockquote></p>
1588       * 
1589       * @param mDbgScript the script name
1590       */
1591      public void  setMapDebugScript(String mDbgScript) {
1592        set(JobContext.MAP_DEBUG_SCRIPT, mDbgScript);
1593      }
1594      
1595      /**
1596       * Get the map task's debug script.
1597       * 
1598       * @return the debug Script for the mapred job for failed map tasks.
1599       * @see #setMapDebugScript(String)
1600       */
1601      public String getMapDebugScript() {
1602        return get(JobContext.MAP_DEBUG_SCRIPT);
1603      }
1604      
1605      /**
1606       * Set the debug script to run when the reduce tasks fail.
1607       * 
1608       * <p>The debug script can aid debugging of failed reduce tasks. The script
1609       * is given task's stdout, stderr, syslog, jobconf files as arguments.</p>
1610       * 
1611       * <p>The debug command, run on the node where the map failed, is:</p>
1612       * <p><pre><blockquote> 
1613       * $script $stdout $stderr $syslog $jobconf.
1614       * </blockquote></pre></p>
1615       * 
1616       * <p> The script file is distributed through {@link DistributedCache} 
1617       * APIs. The script file needs to be symlinked </p>
1618       * 
1619       * <p>Here is an example on how to submit a script 
1620       * <p><blockquote><pre>
1621       * job.setReduceDebugScript("./myscript");
1622       * DistributedCache.createSymlink(job);
1623       * DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
1624       * </pre></blockquote></p>
1625       * 
1626       * @param rDbgScript the script name
1627       */
1628      public void  setReduceDebugScript(String rDbgScript) {
1629        set(JobContext.REDUCE_DEBUG_SCRIPT, rDbgScript);
1630      }
1631      
1632      /**
1633       * Get the reduce task's debug Script
1634       * 
1635       * @return the debug script for the mapred job for failed reduce tasks.
1636       * @see #setReduceDebugScript(String)
1637       */
1638      public String getReduceDebugScript() {
1639        return get(JobContext.REDUCE_DEBUG_SCRIPT);
1640      }
1641    
1642      /**
1643       * Get the uri to be invoked in-order to send a notification after the job 
1644       * has completed (success/failure). 
1645       * 
1646       * @return the job end notification uri, <code>null</code> if it hasn't
1647       *         been set.
1648       * @see #setJobEndNotificationURI(String)
1649       */
1650      public String getJobEndNotificationURI() {
1651        return get(JobContext.MR_JOB_END_NOTIFICATION_URL);
1652      }
1653    
1654      /**
1655       * Set the uri to be invoked in-order to send a notification after the job
1656       * has completed (success/failure).
1657       * 
1658       * <p>The uri can contain 2 special parameters: <tt>$jobId</tt> and 
1659       * <tt>$jobStatus</tt>. Those, if present, are replaced by the job's 
1660       * identifier and completion-status respectively.</p>
1661       * 
1662       * <p>This is typically used by application-writers to implement chaining of 
1663       * Map-Reduce jobs in an <i>asynchronous manner</i>.</p>
1664       * 
1665       * @param uri the job end notification uri
1666       * @see JobStatus
1667       * @see <a href="{@docRoot}/org/apache/hadoop/mapred/JobClient.html#
1668       *       JobCompletionAndChaining">Job Completion and Chaining</a>
1669       */
1670      public void setJobEndNotificationURI(String uri) {
1671        set(JobContext.MR_JOB_END_NOTIFICATION_URL, uri);
1672      }
1673    
1674      /**
1675       * Get job-specific shared directory for use as scratch space
1676       * 
1677       * <p>
1678       * When a job starts, a shared directory is created at location
1679       * <code>
1680       * ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ </code>.
1681       * This directory is exposed to the users through 
1682       * <code>mapreduce.job.local.dir </code>.
1683       * So, the tasks can use this space 
1684       * as scratch space and share files among them. </p>
1685       * This value is available as System property also.
1686       * 
1687       * @return The localized job specific shared directory
1688       */
1689      public String getJobLocalDir() {
1690        return get(JobContext.JOB_LOCAL_DIR);
1691      }
1692    
1693      /**
1694       * Get memory required to run a map task of the job, in MB.
1695       * 
1696       * If a value is specified in the configuration, it is returned.
1697       * Else, it returns {@link #DISABLED_MEMORY_LIMIT}.
1698       * <p/>
1699       * For backward compatibility, if the job configuration sets the
1700       * key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
1701       * from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
1702       * after converting it from bytes to MB.
1703       * @return memory required to run a map task of the job, in MB,
1704       *          or {@link #DISABLED_MEMORY_LIMIT} if unset.
1705       */
1706      public long getMemoryForMapTask() {
1707        long value = getDeprecatedMemoryValue();
1708        if (value == DISABLED_MEMORY_LIMIT) {
1709          value = normalizeMemoryConfigValue(
1710                    getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
1711                              DISABLED_MEMORY_LIMIT));
1712        }
1713        return value;
1714      }
1715    
1716      public void setMemoryForMapTask(long mem) {
1717        setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, mem);
1718      }
1719    
1720      /**
1721       * Get memory required to run a reduce task of the job, in MB.
1722       * 
1723       * If a value is specified in the configuration, it is returned.
1724       * Else, it returns {@link #DISABLED_MEMORY_LIMIT}.
1725       * <p/>
1726       * For backward compatibility, if the job configuration sets the
1727       * key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
1728       * from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
1729       * after converting it from bytes to MB.
1730       * @return memory required to run a reduce task of the job, in MB,
1731       *          or {@link #DISABLED_MEMORY_LIMIT} if unset.
1732       */
1733      public long getMemoryForReduceTask() {
1734        long value = getDeprecatedMemoryValue();
1735        if (value == DISABLED_MEMORY_LIMIT) {
1736          value = normalizeMemoryConfigValue(
1737                    getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
1738                            DISABLED_MEMORY_LIMIT));
1739        }
1740        return value;
1741      }
1742      
1743      // Return the value set to the key MAPRED_TASK_MAXVMEM_PROPERTY,
1744      // converted into MBs.
1745      // Returns DISABLED_MEMORY_LIMIT if unset, or set to a negative
1746      // value.
1747      private long getDeprecatedMemoryValue() {
1748        long oldValue = getLong(MAPRED_TASK_MAXVMEM_PROPERTY, 
1749            DISABLED_MEMORY_LIMIT);
1750        oldValue = normalizeMemoryConfigValue(oldValue);
1751        if (oldValue != DISABLED_MEMORY_LIMIT) {
1752          oldValue /= (1024*1024);
1753        }
1754        return oldValue;
1755      }
1756    
1757      public void setMemoryForReduceTask(long mem) {
1758        setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, mem);
1759      }
1760    
1761      /**
1762       * Return the name of the queue to which this job is submitted.
1763       * Defaults to 'default'.
1764       * 
1765       * @return name of the queue
1766       */
1767      public String getQueueName() {
1768        return get(JobContext.QUEUE_NAME, DEFAULT_QUEUE_NAME);
1769      }
1770      
1771      /**
1772       * Set the name of the queue to which this job should be submitted.
1773       * 
1774       * @param queueName Name of the queue
1775       */
1776      public void setQueueName(String queueName) {
1777        set(JobContext.QUEUE_NAME, queueName);
1778      }
1779      
1780      /**
1781       * Normalize the negative values in configuration
1782       * 
1783       * @param val
1784       * @return normalized value
1785       */
1786      public static long normalizeMemoryConfigValue(long val) {
1787        if (val < 0) {
1788          val = DISABLED_MEMORY_LIMIT;
1789        }
1790        return val;
1791      }
1792    
1793      /**
1794       * Compute the number of slots required to run a single map task-attempt
1795       * of this job.
1796       * @param slotSizePerMap cluster-wide value of the amount of memory required
1797       *                       to run a map-task
1798       * @return the number of slots required to run a single map task-attempt
1799       *          1 if memory parameters are disabled.
1800       */
1801      int computeNumSlotsPerMap(long slotSizePerMap) {
1802        if ((slotSizePerMap==DISABLED_MEMORY_LIMIT) ||
1803            (getMemoryForMapTask()==DISABLED_MEMORY_LIMIT)) {
1804          return 1;
1805        }
1806        return (int)(Math.ceil((float)getMemoryForMapTask() / (float)slotSizePerMap));
1807      }
1808      
1809      /**
1810       * Compute the number of slots required to run a single reduce task-attempt
1811       * of this job.
1812       * @param slotSizePerReduce cluster-wide value of the amount of memory 
1813       *                          required to run a reduce-task
1814       * @return the number of slots required to run a single reduce task-attempt
1815       *          1 if memory parameters are disabled
1816       */
1817      int computeNumSlotsPerReduce(long slotSizePerReduce) {
1818        if ((slotSizePerReduce==DISABLED_MEMORY_LIMIT) ||
1819            (getMemoryForReduceTask()==DISABLED_MEMORY_LIMIT)) {
1820          return 1;
1821        }
1822        return 
1823        (int)(Math.ceil((float)getMemoryForReduceTask() / (float)slotSizePerReduce));
1824      }
1825      
1826      /** 
1827       * Find a jar that contains a class of the same name, if any.
1828       * It will return a jar file, even if that is not the first thing
1829       * on the class path that has a class with the same name.
1830       * 
1831       * @param my_class the class to find.
1832       * @return a jar file that contains the class, or null.
1833       * @throws IOException
1834       */
1835      public static String findContainingJar(Class my_class) {
1836        ClassLoader loader = my_class.getClassLoader();
1837        String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
1838        try {
1839          for(Enumeration itr = loader.getResources(class_file);
1840              itr.hasMoreElements();) {
1841            URL url = (URL) itr.nextElement();
1842            if ("jar".equals(url.getProtocol())) {
1843              String toReturn = url.getPath();
1844              if (toReturn.startsWith("file:")) {
1845                toReturn = toReturn.substring("file:".length());
1846              }
1847              // URLDecoder is a misnamed class, since it actually decodes
1848              // x-www-form-urlencoded MIME type rather than actual
1849              // URL encoding (which the file path has). Therefore it would
1850              // decode +s to ' 's which is incorrect (spaces are actually
1851              // either unencoded or encoded as "%20"). Replace +s first, so
1852              // that they are kept sacred during the decoding process.
1853              toReturn = toReturn.replaceAll("\\+", "%2B");
1854              toReturn = URLDecoder.decode(toReturn, "UTF-8");
1855              return toReturn.replaceAll("!.*$", "");
1856            }
1857          }
1858        } catch (IOException e) {
1859          throw new RuntimeException(e);
1860        }
1861        return null;
1862      }
1863    
1864    
1865      /**
1866       * Get the memory required to run a task of this job, in bytes. See
1867       * {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
1868       * <p/>
1869       * This method is deprecated. Now, different memory limits can be
1870       * set for map and reduce tasks of a job, in MB. 
1871       * <p/>
1872       * For backward compatibility, if the job configuration sets the
1873       * key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
1874       * from {@link #DISABLED_MEMORY_LIMIT}, that value is returned. 
1875       * Otherwise, this method will return the larger of the values returned by 
1876       * {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()}
1877       * after converting them into bytes.
1878       *
1879       * @return Memory required to run a task of this job, in bytes,
1880       *          or {@link #DISABLED_MEMORY_LIMIT}, if unset.
1881       * @see #setMaxVirtualMemoryForTask(long)
1882       * @deprecated Use {@link #getMemoryForMapTask()} and
1883       *             {@link #getMemoryForReduceTask()}
1884       */
1885      @Deprecated
1886      public long getMaxVirtualMemoryForTask() {
1887        LOG.warn(
1888          "getMaxVirtualMemoryForTask() is deprecated. " +
1889          "Instead use getMemoryForMapTask() and getMemoryForReduceTask()");
1890    
1891        long value = getLong(MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
1892        value = normalizeMemoryConfigValue(value);
1893        if (value == DISABLED_MEMORY_LIMIT) {
1894          value = Math.max(getMemoryForMapTask(), getMemoryForReduceTask());
1895          value = normalizeMemoryConfigValue(value);
1896          if (value != DISABLED_MEMORY_LIMIT) {
1897            value *= 1024*1024;
1898          }
1899        }
1900        return value;
1901      }
1902    
1903      /**
1904       * Set the maximum amount of memory any task of this job can use. See
1905       * {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
1906       * <p/>
1907       * mapred.task.maxvmem is split into
1908       * mapreduce.map.memory.mb
1909       * and mapreduce.map.memory.mb,mapred
1910       * each of the new key are set
1911       * as mapred.task.maxvmem / 1024
1912       * as new values are in MB
1913       *
1914       * @param vmem Maximum amount of virtual memory in bytes any task of this job
1915       *             can use.
1916       * @see #getMaxVirtualMemoryForTask()
1917       * @deprecated
1918       *  Use {@link #setMemoryForMapTask(long mem)}  and
1919       *  Use {@link #setMemoryForReduceTask(long mem)}
1920       */
1921      @Deprecated
1922      public void setMaxVirtualMemoryForTask(long vmem) {
1923        LOG.warn("setMaxVirtualMemoryForTask() is deprecated."+
1924          "Instead use setMemoryForMapTask() and setMemoryForReduceTask()");
1925        if(vmem != DISABLED_MEMORY_LIMIT && vmem < 0) {
1926          setMemoryForMapTask(DISABLED_MEMORY_LIMIT);
1927          setMemoryForReduceTask(DISABLED_MEMORY_LIMIT);
1928        }
1929    
1930        if(get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) == null) {
1931          setMemoryForMapTask(vmem / (1024 * 1024)); //Changing bytes to mb
1932          setMemoryForReduceTask(vmem / (1024 * 1024));//Changing bytes to mb
1933        }else{
1934          this.setLong(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY,vmem);
1935        }
1936      }
1937    
1938      /**
1939       * @deprecated this variable is deprecated and nolonger in use.
1940       */
1941      @Deprecated
1942      public long getMaxPhysicalMemoryForTask() {
1943        LOG.warn("The API getMaxPhysicalMemoryForTask() is deprecated."
1944                  + " Refer to the APIs getMemoryForMapTask() and"
1945                  + " getMemoryForReduceTask() for details.");
1946        return -1;
1947      }
1948    
1949      /*
1950       * @deprecated this
1951       */
1952      @Deprecated
1953      public void setMaxPhysicalMemoryForTask(long mem) {
1954        LOG.warn("The API setMaxPhysicalMemoryForTask() is deprecated."
1955            + " The value set is ignored. Refer to "
1956            + " setMemoryForMapTask() and setMemoryForReduceTask() for details.");
1957      }
1958    
1959      static String deprecatedString(String key) {
1960        return "The variable " + key + " is no longer used.";
1961      }
1962    
1963      private void checkAndWarnDeprecation() {
1964        if(get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
1965          LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY)
1966                    + " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY
1967                    + " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
1968        }
1969      }
1970      
1971    
1972    }
1973