View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import com.google.common.collect.ImmutableList;
22  import com.google.common.collect.Lists;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Arrays;
27  import java.util.List;
28  
29  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
30  import org.apache.hadoop.hbase.regionserver.compactions.DateTieredCompactionPolicy;
31  import org.apache.hadoop.hbase.testclassification.SmallTests;
32  import org.junit.Assert;
33  import org.junit.Test;
34  import org.junit.experimental.categories.Category;
35  
36  @Category(SmallTests.class)
37  public class TestDateTieredCompaction extends TestCompactionPolicy {
38    ArrayList<StoreFile> sfCreate(long[] minTimestamps, long[] maxTimestamps, long[] sizes)
39        throws IOException {
40      ArrayList<Long> ageInDisk = new ArrayList<Long>();
41      for (int i = 0; i < sizes.length; i++) {
42        ageInDisk.add(0L);
43      }
44  
45      ArrayList<StoreFile> ret = Lists.newArrayList();
46      for (int i = 0; i < sizes.length; i++) {
47        MockStoreFile msf =
48            new MockStoreFile(TEST_UTIL, TEST_FILE, sizes[i], ageInDisk.get(i), false, i);
49        msf.setTimeRangeTracker(new TimeRangeTracker(minTimestamps[i], maxTimestamps[i]));
50        ret.add(msf);
51      }
52      return ret;
53    }
54  
55    @Override
56    protected void config() {
57      super.config();
58  
59      // Set up policy
60      conf.setLong(CompactionConfiguration.MAX_AGE_KEY, 100);
61      conf.setLong(CompactionConfiguration.INCOMING_WINDOW_MIN_KEY, 3);
62      conf.setLong(CompactionConfiguration.BASE_WINDOW_MILLIS_KEY, 6);
63      conf.setInt(CompactionConfiguration.WINDOWS_PER_TIER_KEY, 4);
64      conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
65        DateTieredCompactionPolicy.class.getName());
66  
67      // Special settings for compaction policy per window
68      this.conf.setInt(CompactionConfiguration.MIN_KEY, 2);
69      this.conf.setInt(CompactionConfiguration.MAX_KEY, 12);
70      this.conf.setFloat(CompactionConfiguration.RATIO_KEY, 1.2F);
71    }
72  
73    void compactEquals(long now, ArrayList<StoreFile> candidates, long... expected)
74        throws IOException {
75      Assert.assertTrue(((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
76          .needsCompaction(candidates, ImmutableList.<StoreFile> of(), now));
77  
78      List<StoreFile> actual =
79          ((DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy())
80              .applyCompactionPolicy(candidates, false, false, now);
81  
82      Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual)));
83    }
84  
85    /**
86     * Test for incoming window
87     * @throws IOException with error
88     */
89    @Test
90    public void incomingWindow() throws IOException {
91      long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
92      long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
93      long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
94  
95      compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 13, 12, 11, 10);
96    }
97  
98    /**
99     * Not enough files in incoming window
100    * @throws IOException with error
101    */
102   @Test
103   public void NotIncomingWindow() throws IOException {
104     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
105     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
106     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11 };
107 
108     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 25, 24, 23, 22, 21, 20);
109   }
110 
111   /**
112    * Test for file newer than incoming window
113    * @throws IOException with error
114    */
115   @Test
116   public void NewerThanIncomingWindow() throws IOException {
117     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
118     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18 };
119     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 23, 24, 25, 10, 11, 12, 13 };
120 
121     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 13, 12, 11, 10);
122   }
123 
124   /**
125    * If there is no T1 window, we don't build 2
126    * @throws IOException with error
127    */
128   @Test
129   public void NoT2() throws IOException {
130     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
131     long[] maxTimestamps = new long[] { 44, 60, 61, 92, 95, 100 };
132     long[] sizes = new long[] { 0, 20, 21, 22, 23, 1 };
133 
134     compactEquals(100, sfCreate(minTimestamps, maxTimestamps, sizes), 23, 22);
135   }
136 
137   @Test
138   public void T1() throws IOException {
139     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
140     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
141     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 30, 31, 32, 2, 1 };
142 
143     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 32, 31, 30);
144   }
145 
146   /**
147    * Apply exploring logic on non-incoming window
148    * @throws IOException with error
149    */
150   @Test
151   public void RatioT0() throws IOException {
152     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
153     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
154     long[] sizes = new long[] { 30, 31, 32, 33, 34, 20, 21, 22, 280, 23, 24, 1 };
155 
156     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 22, 21, 20);
157   }
158 
159   /**
160    * Also apply ratio-based logic on t2 window
161    * @throws IOException with error
162    */
163   @Test
164   public void RatioT2() throws IOException {
165     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
166     long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 120, 124, 143, 145, 157 };
167     long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 350, 30, 31, 2, 1 };
168 
169     compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 31, 30);
170   }
171 
172   /**
173    * The next compaction call after testTieredCompactionRatioT0 is compacted
174    * @throws IOException with error
175    */
176   @Test
177   public void RatioT0Next() throws IOException {
178     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
179     long[] maxTimestamps = new long[] { 1, 2, 3, 4, 5, 8, 9, 10, 11, 12 };
180     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 280, 23, 24, 1 };
181 
182     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 24, 23);
183   }
184 
185   /**
186   * Older than now(161) - maxAge(100)
187   * @throws IOException with error
188   */
189  @Test
190  public void olderThanMaxAge() throws IOException {
191    long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
192    long[] maxTimestamps = new long[] { 44, 60, 61, 96, 100, 104, 105, 106, 113, 145, 157 };
193    long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2, 1 };
194 
195    compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes), 31, 30, 33, 42, 41, 40);
196  }
197 
198   /**
199    * Out-of-order data
200    * @throws IOException with error
201    */
202   @Test
203   public void OutOfOrder() throws IOException {
204     long[] minTimestamps = new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
205     long[] maxTimestamps = new long[] { 0, 13, 3, 10, 11, 1, 2, 12, 14, 15 };
206     long[] sizes = new long[] { 30, 31, 32, 33, 34, 22, 28, 23, 24, 1 };
207 
208     compactEquals(16, sfCreate(minTimestamps, maxTimestamps, sizes), 1, 24, 23, 28, 22, 34,
209       33, 32, 31);
210   }
211 }