1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce.hadoopbackport;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24
25 import junit.framework.TestCase;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.SmallTests;
31 import org.apache.hadoop.io.NullWritable;
32 import org.apache.hadoop.io.RawComparator;
33 import org.apache.hadoop.io.SequenceFile;
34 import org.apache.hadoop.io.Text;
35 import org.apache.hadoop.io.WritableComparable;
36 import org.apache.hadoop.io.WritableComparator;
37 import org.apache.hadoop.io.WritableUtils;
38 import org.apache.hadoop.mapred.JobConf;
39 import org.junit.experimental.categories.Category;
40
41
42
43
44 @Category(SmallTests.class)
45 public class TestTotalOrderPartitioner extends TestCase {
46
47 private static final Text[] splitStrings = new Text[] {
48
49 new Text("aabbb"),
50 new Text("babbb"),
51 new Text("daddd"),
52 new Text("dddee"),
53 new Text("ddhee"),
54 new Text("dingo"),
55 new Text("hijjj"),
56 new Text("n"),
57 new Text("yak"),
58 };
59
60 static class Check<T> {
61 T data;
62 int part;
63 Check(T data, int part) {
64 this.data = data;
65 this.part = part;
66 }
67 }
68
69 private static final ArrayList<Check<Text>> testStrings =
70 new ArrayList<Check<Text>>();
71 static {
72 testStrings.add(new Check<Text>(new Text("aaaaa"), 0));
73 testStrings.add(new Check<Text>(new Text("aaabb"), 0));
74 testStrings.add(new Check<Text>(new Text("aabbb"), 1));
75 testStrings.add(new Check<Text>(new Text("aaaaa"), 0));
76 testStrings.add(new Check<Text>(new Text("babbb"), 2));
77 testStrings.add(new Check<Text>(new Text("baabb"), 1));
78 testStrings.add(new Check<Text>(new Text("yai"), 8));
79 testStrings.add(new Check<Text>(new Text("yak"), 9));
80 testStrings.add(new Check<Text>(new Text("z"), 9));
81 testStrings.add(new Check<Text>(new Text("ddngo"), 5));
82 testStrings.add(new Check<Text>(new Text("hi"), 6));
83 };
84
85 private static <T extends WritableComparable<?>> Path writePartitionFile(
86 String testname, Configuration conf, T[] splits) throws IOException {
87 final FileSystem fs = FileSystem.getLocal(conf);
88 final Path testdir = new Path(System.getProperty("test.build.data", "/tmp")
89 ).makeQualified(fs);
90 Path p = new Path(testdir, testname + "/_partition.lst");
91 TotalOrderPartitioner.setPartitionFile(conf, p);
92 conf.setInt("mapreduce.job.reduces", splits.length + 1);
93 SequenceFile.Writer w = null;
94 try {
95 w = SequenceFile.createWriter(fs, conf, p,
96 splits[0].getClass(), NullWritable.class,
97 SequenceFile.CompressionType.NONE);
98 for (int i = 0; i < splits.length; ++i) {
99 w.append(splits[i], NullWritable.get());
100 }
101 } finally {
102 if (null != w)
103 w.close();
104 }
105 return p;
106 }
107
108 public void testTotalOrderMemCmp() throws Exception {
109 TotalOrderPartitioner<Text,NullWritable> partitioner =
110 new TotalOrderPartitioner<Text,NullWritable>();
111
112
113 JobConf conf = new JobConf();
114 conf.setMapOutputKeyClass(Text.class);
115 conf.setNumReduceTasks(splitStrings.length + 1);
116
117 Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
118 "totalordermemcmp", conf, splitStrings);
119 try {
120 partitioner.setConf(conf);
121 NullWritable nw = NullWritable.get();
122 for (Check<Text> chk : testStrings) {
123 assertEquals(chk.data.toString(), chk.part,
124 partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
125 }
126 } finally {
127 p.getFileSystem(conf).delete(p, true);
128 }
129 }
130
131 public void testTotalOrderBinarySearch() throws Exception {
132 TotalOrderPartitioner<Text,NullWritable> partitioner =
133 new TotalOrderPartitioner<Text,NullWritable>();
134 JobConf conf = new JobConf();
135 conf.setMapOutputKeyClass(Text.class);
136 conf.setNumReduceTasks(splitStrings.length + 1);
137
138 Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
139 "totalorderbinarysearch", conf, splitStrings);
140 conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
141 try {
142 partitioner.setConf(conf);
143 NullWritable nw = NullWritable.get();
144 for (Check<Text> chk : testStrings) {
145 assertEquals(chk.data.toString(), chk.part,
146 partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
147 }
148 } finally {
149 p.getFileSystem(conf).delete(p, true);
150 }
151 }
152
153 public static class ReverseStringComparator implements RawComparator<Text> {
154 @Override
155 public int compare(Text a, Text b) {
156 return -a.compareTo(b);
157 }
158 @Override
159 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
160 int n1 = WritableUtils.decodeVIntSize(b1[s1]);
161 int n2 = WritableUtils.decodeVIntSize(b2[s2]);
162 return -1 * WritableComparator.compareBytes(b1, s1+n1, l1-n1,
163 b2, s2+n2, l2-n2);
164 }
165 }
166
167 public void testTotalOrderCustomComparator() throws Exception {
168 TotalOrderPartitioner<Text,NullWritable> partitioner =
169 new TotalOrderPartitioner<Text,NullWritable>();
170
171 final JobConf conf = new JobConf();
172 conf.setMapOutputKeyClass(Text.class);
173 conf.setNumReduceTasks(splitStrings.length + 1);
174
175 Text[] revSplitStrings = Arrays.copyOf(splitStrings, splitStrings.length);
176 Arrays.sort(revSplitStrings, new ReverseStringComparator());
177 Path p = TestTotalOrderPartitioner.<Text>writePartitionFile(
178 "totalordercustomcomparator", conf, revSplitStrings);
179 conf.setBoolean(TotalOrderPartitioner.NATURAL_ORDER, false);
180 conf.setOutputKeyComparatorClass(ReverseStringComparator.class);
181
182 ArrayList<Check<Text>> revCheck = new ArrayList<Check<Text>>();
183 revCheck.add(new Check<Text>(new Text("aaaaa"), 9));
184 revCheck.add(new Check<Text>(new Text("aaabb"), 9));
185 revCheck.add(new Check<Text>(new Text("aabbb"), 9));
186 revCheck.add(new Check<Text>(new Text("aaaaa"), 9));
187 revCheck.add(new Check<Text>(new Text("babbb"), 8));
188 revCheck.add(new Check<Text>(new Text("baabb"), 8));
189 revCheck.add(new Check<Text>(new Text("yai"), 1));
190 revCheck.add(new Check<Text>(new Text("yak"), 1));
191 revCheck.add(new Check<Text>(new Text("z"), 0));
192 revCheck.add(new Check<Text>(new Text("ddngo"), 4));
193 revCheck.add(new Check<Text>(new Text("hi"), 3));
194 try {
195 partitioner.setConf(conf);
196 NullWritable nw = NullWritable.get();
197 for (Check<Text> chk : revCheck) {
198 assertEquals(chk.data.toString(), chk.part,
199 partitioner.getPartition(chk.data, nw, splitStrings.length + 1));
200 }
201 } finally {
202 p.getFileSystem(conf).delete(p, true);
203 }
204 }
205 }