View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.hadoop.hbase.HConstants;
25  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
26  import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
27  import org.apache.hadoop.hbase.testclassification.SmallTests;
28  import org.junit.Assert;
29  import org.junit.Test;
30  import org.junit.experimental.categories.Category;
31  
32  @Category(SmallTests.class)
33  public class TestDefaultCompactSelection extends TestCompactionPolicy {
34  
35    @Test
36    public void testCompactionRatio() throws IOException {
37      /**
38       * NOTE: these tests are specific to describe the implementation of the
39       * current compaction algorithm.  Developed to ensure that refactoring
40       * doesn't implicitly alter this.
41       */
42      long tooBig = maxSize + 1;
43  
44      // default case. preserve user ratio on size
45      compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
46      // less than compact threshold = don't compact
47      compactEquals(sfCreate(100,50,25,12,12) /* empty */);
48      // greater than compact size = skip those
49      compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700);
50      // big size + threshold
51      compactEquals(sfCreate(tooBig, tooBig, 700,700) /* empty */);
52      // small files = don't care about ratio
53      compactEquals(sfCreate(7,1,1), 7,1,1);
54  
55      // don't exceed max file compact threshold
56      // note:  file selection starts with largest to smallest.
57      compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
58  
59      compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
60  
61      compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
62  
63      compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
64  
65      compactEquals(sfCreate(maxSize -1,maxSize -1,maxSize -1) /* empty */);
66  
67      // Always try and compact something to get below blocking storefile count
68      this.conf.setLong("hbase.hstore.compaction.min.size", 1);
69      store.storeEngine.getCompactionPolicy().setConf(conf);
70      compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1);
71      this.conf.setLong("hbase.hstore.compaction.min.size", minSize);
72      store.storeEngine.getCompactionPolicy().setConf(conf);
73  
74      /* MAJOR COMPACTION */
75      // if a major compaction has been forced, then compact everything
76      compactEquals(sfCreate(50,25,12,12), true, 50, 25, 12, 12);
77      // also choose files < threshold on major compaction
78      compactEquals(sfCreate(12,12), true, 12, 12);
79      // even if one of those files is too big
80      compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
81      // don't exceed max file compact threshold, even with major compaction
82      store.forceMajor = true;
83      compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
84      store.forceMajor = false;
85      // if we exceed maxCompactSize, downgrade to minor
86      // if not, it creates a 'snowball effect' when files >> maxCompactSize:
87      // the last file in compaction is the aggregate of all previous compactions
88      compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12);
89      conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
90      conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
91      store.storeEngine.getCompactionPolicy().setConf(conf);
92      try {
93        // trigger an aged major compaction
94        compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12);
95        // major sure exceeding maxCompactSize also downgrades aged minors
96        compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
97      } finally {
98        conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24);
99        conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
100     }
101 
102     /* REFERENCES == file is from a region that was split */
103     // treat storefiles that have references like a major compaction
104     compactEquals(sfCreate(true, 100,50,25,12,12), 100, 50, 25, 12, 12);
105     // reference files shouldn't obey max threshold
106     compactEquals(sfCreate(true, tooBig, 12,12), tooBig, 12, 12);
107     // reference files should obey max file compact to avoid OOM
108     compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
109 
110     // empty case
111     compactEquals(new ArrayList<StoreFile>() /* empty */);
112     // empty case (because all files are too big)
113     compactEquals(sfCreate(tooBig, tooBig) /* empty */);
114   }
115 
116   @Test
117   public void testOffPeakCompactionRatio() throws IOException {
118     /*
119      * NOTE: these tests are specific to describe the implementation of the
120      * current compaction algorithm.  Developed to ensure that refactoring
121      * doesn't implicitly alter this.
122      */
123     // set an off-peak compaction threshold
124     this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
125     store.storeEngine.getCompactionPolicy().setConf(this.conf);
126     // Test with and without the flag.
127     compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
128     compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
129   }
130 
131   @Test
132   public void testStuckStoreCompaction() throws IOException {
133     // Select the smallest compaction if the store is stuck.
134     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,30), 30, 30, 30);
135     // If not stuck, standard policy applies.
136     compactEquals(sfCreate(99,99,99,99,99, 30,30,30,30), 99, 30, 30, 30, 30);
137 
138     // Add sufficiently small files to compaction, though
139     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,15), 30, 30, 30, 15);
140     // Prefer earlier compaction to latter if the benefit is not significant
141     compactEquals(sfCreate(99,99,99,99, 30,26,26,29,25,25), 30, 26, 26);
142     // Prefer later compaction if the benefit is significant.
143     compactEquals(sfCreate(99,99,99,99, 27,27,27,20,20,20), 20, 20, 20);
144   }
145 
146   @Test
147   public void testCompactionEmptyHFile() throws IOException {
148     // Set TTL
149     ScanInfo oldScanInfo = store.getScanInfo();
150     ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getFamily(),
151         oldScanInfo.getMinVersions(), oldScanInfo.getMaxVersions(), 600,
152         oldScanInfo.getKeepDeletedCells(), oldScanInfo.getTimeToPurgeDeletes(),
153         oldScanInfo.getComparator());
154     store.setScanInfo(newScanInfo);
155     // Do not compact empty store file
156     List<StoreFile> candidates = sfCreate(0);
157     for (StoreFile file : candidates) {
158       if (file instanceof MockStoreFile) {
159         MockStoreFile mockFile = (MockStoreFile) file;
160         mockFile.setTimeRangeTracker(new TimeRangeTracker(-1, -1));
161         mockFile.setEntries(0);
162       }
163     }
164     // Test Default compactions
165     CompactionRequest result = ((RatioBasedCompactionPolicy) store.storeEngine
166         .getCompactionPolicy()).selectCompaction(candidates,
167         new ArrayList<StoreFile>(), false, false, false);
168     Assert.assertTrue(result.getFiles().size() == 0);
169     store.setScanInfo(oldScanInfo);
170   }
171 }