1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import java.io.IOException;
21
22 import org.apache.hadoop.hbase.HConstants;
23 import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
24 import org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory;
25 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
26 import org.apache.hadoop.hbase.testclassification.SmallTests;
27 import org.junit.Test;
28 import org.junit.experimental.categories.Category;
29
30 @Category({ RegionServerTests.class, SmallTests.class })
31 public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompactionPolicy {
32
33 @Override
34 protected void config() {
35 super.config();
36
37
38 conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
39 "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
40 conf.setLong(CompactionConfiguration.DATE_TIERED_MAX_AGE_MILLIS_KEY, 100);
41 conf.setLong(CompactionConfiguration.DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 3);
42 conf.setLong(ExponentialCompactionWindowFactory.BASE_WINDOW_MILLIS_KEY, 6);
43 conf.setInt(ExponentialCompactionWindowFactory.WINDOWS_PER_TIER_KEY, 4);
44 conf.setBoolean(CompactionConfiguration.DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
45 false);
46
47
48 conf.setInt(CompactionConfiguration.MIN_KEY, 2);
49 conf.setInt(CompactionConfiguration.MAX_KEY, 12);
50 conf.setFloat(CompactionConfiguration.RATIO_KEY, 1.2F);
51
52 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
53 conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 10);
54 }
55
56
57
58
59
60 @Test
61 public void incomingWindow() throws IOException {
62 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
63 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
64 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
65
66 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
67 new long[] { Long.MIN_VALUE, 12 }, false, true);
68 }
69
70
71
72
73
74 @Test
75 public void NotIncomingWindow() throws IOException {
76 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
77 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
78 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
79
80 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22, 23,
81 24, 25 }, new long[] { Long.MIN_VALUE, 6}, false, true);
82 }
83
84
85
86
87
88 @Test
89 public void OnUpperBoundOfIncomingWindow() throws IOException {
90 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
91 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
92 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
93
94 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
95 new long[] { Long.MIN_VALUE, 12 }, false, true);
96 }
97
98
99
100
101
102 @Test
103 public void NewerThanIncomingWindow() throws IOException {
104 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
105 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 };
106 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
107
108 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
109 new long[] { Long.MIN_VALUE, 12}, false, true);
110 }
111
112
113
114
115
116 @Test
117 public void NoT2() throws IOException {
118 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
119 long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
120 long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
121
122 compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 },
123 new long[] { Long.MIN_VALUE, 96}, false, true);
124 }
125
126 @Test
127 public void T1() throws IOException {
128 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
129 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
130 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
131
132 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 },
133 new long[] { Long.MIN_VALUE, 120 }, false, true);
134 }
135
136
137
138
139
140 @Test
141 public void RatioT0() throws IOException {
142 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
143 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
144 long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
145
146 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 },
147 new long[] { Long.MIN_VALUE }, false, true);
148 }
149
150
151
152
153
154 @Test
155 public void RatioT2() throws IOException {
156 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
157 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
158 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
159
160 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 },
161 new long[] { Long.MIN_VALUE }, false, true);
162 }
163
164
165
166
167
168 @Test
169 public void RatioT0Next() throws IOException {
170 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
171 long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
172 long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
173
174 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 },
175 new long[] { Long.MIN_VALUE }, false, true);
176 }
177
178
179
180
181
182 @Test
183 public void olderThanMaxAge() throws IOException {
184 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
185 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
186 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
187
188 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 40, 41, 42, 33,
189 30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
190 }
191
192
193
194
195
196 @Test
197 public void outOfOrder() throws IOException {
198 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
199 long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
200 long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
201
202 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 31, 32, 33, 34,
203 22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false, true);
204 }
205
206
207
208
209
210 @Test
211 public void negativeEpochtime() throws IOException {
212 long[] minTimestamps =
213 new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 };
214 long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
215 long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
216
217 compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
218 new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 },
219 new long[] { Long.MIN_VALUE, -24 }, false, true);
220 }
221
222
223
224
225
226 @Test
227 public void majorCompation() throws IOException {
228 long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
229 long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
230 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
231
232 compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
233 new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 },
234 new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
235 }
236
237
238
239
240
241 @Test
242 public void negativeForMajor() throws IOException {
243 long[] minTimestamps =
244 new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
245 long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
246 long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
247
248 compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,
249 41, 42, 33, 30, 31, 2, 1 },
250 new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
251 }
252 }