1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  
25  import junit.framework.TestCase;
26  
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.SmallTests;
31  import org.apache.hadoop.io.NullWritable;
32  import org.apache.hadoop.io.RawComparator;
33  import org.apache.hadoop.io.SequenceFile;
34  import org.apache.hadoop.io.Text;
35  import org.apache.hadoop.io.WritableComparable;
36  import org.apache.hadoop.io.WritableComparator;
37  import org.apache.hadoop.io.WritableUtils;
38  import org.apache.hadoop.mapred.JobConf;
39  import org.junit.experimental.categories.Category;
40  
41  /**
42   * The test is ported from Hadoop branch-0.23 with very small changes.
43   */
44  @Category(SmallTests.class)
45  public class TestTotalOrderPartitioner extends TestCase {
46  
47    private static final Text[] splitStrings = new Text[] {
48      // -inf            // 0
49      new Text("aabbb"), // 1
50      new Text("babbb"), // 2
51      new Text("daddd"), // 3
52      new Text("dddee"), // 4
53      new Text("ddhee"), // 5
54      new Text("dingo"), // 6
55      new Text("hijjj"), // 7
56      new Text("n"),     // 8
57      new Text("yak"),   // 9
58    };
59  
60    static class Check<T> {
61      T data;
62      int part;
63      Check(T data, int part) {
64        this.data = data;
65        this.part = part;
66      }
67    }
68  
69    private static final ArrayList<Check<Text>> testStrings =
70      new ArrayList<Check<Text>>();
71    static {
72      testStrings.add(new Check<Text>(new Text("aaaaa"), 0));
73      testStrings.add(new Check<Text>(new Text("aaabb"), 0));
74      testStrings.add(new Check<Text>(new Text("aabbb"), 1));
75      testStrings.add(new Check<Text>(new Text("aaaaa"), 0));
76      testStrings.add(new Check<Text>(new Text("babbb"), 2));
77      testStrings.add(new Check<Text>(new Text("baabb"), 1));
78      testStrings.add(new Check<Text>(new Text("yai"), 8));
79      testStrings.add(new Check<Text>(new Text("yak"), 9));
80      testStrings.add(new Check<Text>(new Text("z"), 9));
81      testStrings.add(new Check<Text>(new Text("ddngo"), 5));
82      testStrings.add(new Check<Text>(new Text("hi"), 6));
83    };
84  
85    private static <T extends WritableComparable<?>> Path writePartitionFile(
86        String testname, Configuration conf, T[] splits) throws IOException {
87      final FileSystem fs = FileSystem.getLocal(conf);
88      final Path testdir = new Path(System.getProperty("test.build.data", "/tmp")
89                                   ).makeQualified(fs);
90      Path p = new Path(testdir, testname + "/_partition.lst");
91      TotalOrderPartitioner.setPartitionFile(conf, p);
92      conf.setInt("mapreduce.job.reduces", splits.length + 1);
93      SequenceFile.Writer w = null;
94      try {
95        w = SequenceFile.createWriter(fs, conf, p,
96            splits[0].getClass(), NullWritable.class,
97            SequenceFile.CompressionType.NONE);
98        for (int i = 0; i < splits.length; ++i) {
99          w.append(splits[i], NullWritable.get());
100       }
101     } finally {
102       if (null != w)
103         w.close();
104     }
105     return p;
106   }
107 
108   public void testTotalOrderMemCmp() throws Exception {
109     TotalOrderPartitioner<Text,NullWritable> partitioner =
110       new TotalOrderPartitioner<Text,NullWritable>();
111 
112     // Need to use old JobConf-based variant here:
113     JobConf conf = new JobConf();
114     conf.setMapOutputKeyClass(Text.class);
115     conf.setNumReduceTasks(splitStrings.length + 1);
116 
117     Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
118         "totalordermemcmp", conf, splitStrings);
119     try {
120       partitioner.setConf(conf);
121       NullWritable nw = NullWritable.get();
122       for (Check<Text> chk : testStrings) {
123         assertEquals(chk.data.toString(), chk.part,
124             partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
125       }
126     } finally {
127       p.getFileSystem(conf).delete(p, true);
128     }
129   }
130 
131   public void testTotalOrderBinarySearch() throws Exception {
132     TotalOrderPartitioner<Text,NullWritable> partitioner =
133       new TotalOrderPartitioner<Text,NullWritable>();
134     JobConf conf = new JobConf();
135     conf.setMapOutputKeyClass(Text.class);
136     conf.setNumReduceTasks(splitStrings.length + 1);
137 
138     Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
139         "totalorderbinarysearch", conf, splitStrings);
140     conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
141     try {
142       partitioner.setConf(conf);
143       NullWritable nw = NullWritable.get();
144       for (Check<Text> chk : testStrings) {
145         assertEquals(chk.data.toString(), chk.part,
146             partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
147       }
148     } finally {
149       p.getFileSystem(conf).delete(p, true);
150     }
151   }
152 
153   public static class ReverseStringComparator implements RawComparator<Text> {
154     @Override
155     public int compare(Text a, Text b) {
156       return -a.compareTo(b);
157     }
158     @Override
159     public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
160       int n1 = WritableUtils.decodeVIntSize(b1[s1]);
161       int n2 = WritableUtils.decodeVIntSize(b2[s2]);
162       return -1 * WritableComparator.compareBytes(b1, s1+n1, l1-n1,
163                                                   b2, s2+n2, l2-n2);
164     }
165   }
166 
167   public void testTotalOrderCustomComparator() throws Exception {
168     TotalOrderPartitioner<Text,NullWritable> partitioner =
169       new TotalOrderPartitioner<Text,NullWritable>();
170 
171     final JobConf conf = new JobConf();
172     conf.setMapOutputKeyClass(Text.class);
173     conf.setNumReduceTasks(splitStrings.length + 1);
174 
175     Text[] revSplitStrings = Arrays.copyOf(splitStrings, splitStrings.length);
176     Arrays.sort(revSplitStrings, new ReverseStringComparator());
177     Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
178         "totalordercustomcomparator", conf, revSplitStrings);
179     conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
180     conf.setOutputKeyComparatorClass(ReverseStringComparator.class);
181 
182     ArrayList<Check<Text>> revCheck = new ArrayList<Check<Text>>();
183     revCheck.add(new Check<Text>(new Text("aaaaa"), 9));
184     revCheck.add(new Check<Text>(new Text("aaabb"), 9));
185     revCheck.add(new Check<Text>(new Text("aabbb"), 9));
186     revCheck.add(new Check<Text>(new Text("aaaaa"), 9));
187     revCheck.add(new Check<Text>(new Text("babbb"), 8));
188     revCheck.add(new Check<Text>(new Text("baabb"), 8));
189     revCheck.add(new Check<Text>(new Text("yai"), 1));
190     revCheck.add(new Check<Text>(new Text("yak"), 1));
191     revCheck.add(new Check<Text>(new Text("z"), 0));
192     revCheck.add(new Check<Text>(new Text("ddngo"), 4));
193     revCheck.add(new Check<Text>(new Text("hi"), 3));
194     try {
195       partitioner.setConf(conf);
196       NullWritable nw = NullWritable.get();
197       for (Check<Text> chk : revCheck) {
198         assertEquals(chk.data.toString(), chk.part,
199             partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
200       }
201     } finally {
202       p.getFileSystem(conf).delete(p, true);
203     }
204   }
205 }