View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  
22  import org.apache.hadoop.hbase.HConstants;
23  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
24  import org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory;
25  import org.apache.hadoop.hbase.testclassification.RegionServerTests;
26  import org.apache.hadoop.hbase.testclassification.SmallTests;
27  import org.junit.Test;
28  import org.junit.experimental.categories.Category;
29  
30  @Category({ RegionServerTests.class, SmallTests.class })
31  public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompactionPolicy {
32  
33    @Override
34    protected void config() {
35      super.config();
36  
37      // Set up policy
38      conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY,
39        "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
40      conf.setLong(CompactionConfiguration.DATE_TIERED_MAX_AGE_MILLIS_KEY, 100);
41      conf.setLong(CompactionConfiguration.DATE_TIERED_INCOMING_WINDOW_MIN_KEY, 3);
42      conf.setLong(ExponentialCompactionWindowFactory.BASE_WINDOW_MILLIS_KEY, 6);
43      conf.setInt(ExponentialCompactionWindowFactory.WINDOWS_PER_TIER_KEY, 4);
44      conf.setBoolean(CompactionConfiguration.DATE_TIERED_SINGLE_OUTPUT_FOR_MINOR_COMPACTION_KEY,
45        false);
46  
47      // Special settings for compaction policy per window
48      conf.setInt(CompactionConfiguration.MIN_KEY, 2);
49      conf.setInt(CompactionConfiguration.MAX_KEY, 12);
50      conf.setFloat(CompactionConfiguration.RATIO_KEY, 1.2F);
51  
52      conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 20);
53      conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 10);
54    }
55  
56    /**
57     * Test for incoming window
58     * @throws IOException with error
59     */
60    @Test
61    public void incomingWindow() throws IOException {
62      long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
63      long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
64      long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
65  
66      compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
67        new long[] { Long.MIN_VALUE, 12 }, false, true);
68    }
69  
70    /**
71     * Not enough files in incoming window
72     * @throws IOException with error
73     */
74    @Test
75    public void NotIncomingWindow() throws IOException {
76      long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
77      long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
78      long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
79  
80      compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22, 23,
81          24, 25 }, new long[] { Long.MIN_VALUE, 6}, false, true);
82    }
83  
84    /**
85     * Test for file on the upper bound of incoming window
86     * @throws IOException with error
87     */
88    @Test
89    public void OnUpperBoundOfIncomingWindow() throws IOException {
90      long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
91      long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
92      long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
93  
94      compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
95        new long[] { Long.MIN_VALUE, 12 }, false, true);
96    }
97  
98    /**
99     * Test for file newer than incoming window
100    * @throws IOException with error
101    */
102   @Test
103   public void NewerThanIncomingWindow() throws IOException {
104     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
105     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 19 };
106     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
107 
108     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 10, 11, 12, 13 },
109       new long[] { Long.MIN_VALUE, 12}, false, true);
110   }
111 
112   /**
113    * If there is no T1 window, we don't build T2
114    * @throws IOException with error
115    */
116   @Test
117   public void NoT2() throws IOException {
118     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
119     long[] maxTimestamps = new long[] { 44, 60, 61, 97, 100, 193 };
120     long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
121 
122     compactEquals(194, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 22, 23 },
123       new long[] { Long.MIN_VALUE, 96}, false, true);
124   }
125 
126   @Test
127   public void T1() throws IOException {
128     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
129     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
130     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
131 
132     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31, 32 },
133       new long[] { Long.MIN_VALUE, 120 }, false, true);
134   }
135 
136   /**
137    * Apply exploring logic on non-incoming window
138    * @throws IOException with error
139    */
140   @Test
141   public void RatioT0() throws IOException {
142     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
143     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
144     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
145 
146     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 20, 21, 22 },
147       new long[] { Long.MIN_VALUE }, false, true);
148   }
149 
150   /**
151    * Also apply ratio-based logic on t2 window
152    * @throws IOException with error
153    */
154   @Test
155   public void RatioT2() throws IOException {
156     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
157     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
158     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
159 
160     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 30, 31 },
161       new long[] { Long.MIN_VALUE }, false, true);
162   }
163 
164   /**
165    * The next compaction call after testTieredCompactionRatioT0 is compacted
166    * @throws IOException with error
167    */
168   @Test
169   public void RatioT0Next() throws IOException {
170     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
171     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
172     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
173 
174     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 23, 24 },
175       new long[] { Long.MIN_VALUE }, false, true);
176   }
177 
178   /**
179    * Older than now(161) - maxAge(100)
180    * @throws IOException with error
181    */
182   @Test
183   public void olderThanMaxAge() throws IOException {
184     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
185     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
186     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
187 
188     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 40, 41, 42, 33,
189         30, 31 }, new long[] { Long.MIN_VALUE, 96 }, false, true);
190   }
191 
192   /**
193    * Out-of-order data
194    * @throws IOException with error
195    */
196   @Test
197   public void outOfOrder() throws IOException {
198     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
199     long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
200     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
201 
202     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 31, 32, 33, 34,
203         22, 28, 23, 24, 1 }, new long[] { Long.MIN_VALUE, 12 }, false, true);
204   }
205 
206   /**
207    * Negative epoch time
208    * @throws IOException with error
209    */
210   @Test
211   public void negativeEpochtime() throws IOException {
212     long[] minTimestamps =
213         new long[] { -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000 };
214     long[] maxTimestamps = new long[] { -28, -11, -10, -9, -8, -7, -6, -5, -4, -3 };
215     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 25, 23, 24, 1 };
216 
217     compactEquals(1, sfCreate(minTimestamps, maxTimestamps, sizes),
218       new long[] { 31, 32, 33, 34, 22, 25, 23, 24, 1 },
219       new long[] { Long.MIN_VALUE, -24 }, false, true);
220   }
221 
222   /**
223    * Major compaction
224    * @throws IOException with error
225    */
226   @Test
227   public void majorCompation() throws IOException {
228     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
229     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
230     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
231 
232     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
233       new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 },
234       new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
235   }
236 
237   /**
238    * Major compaction with negative numbers
239    * @throws IOException with error
240    */
241   @Test
242   public void negativeForMajor() throws IOException {
243     long[] minTimestamps =
244         new long[] { -155, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100 };
245     long[] maxTimestamps = new long[] { -8, -7, -6, -5, -4, -3, -2, -1, 0, 6, 13 };
246     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
247 
248     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), new long[] { 0, 50, 51, 40,
249         41, 42, 33, 30, 31, 2, 1 },
250       new long[] { Long.MIN_VALUE, -144, -120, -96, -72, -48, -24, 0, 6, 12 }, true, true);
251   }
252 }