View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.hadoop.hbase.HConstants;
25  import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
26  import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
27  import org.apache.hadoop.hbase.testclassification.SmallTests;
28  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
29  import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
30  import org.junit.Assert;
31  import org.junit.Test;
32  import org.junit.experimental.categories.Category;
33  
34  @Category(SmallTests.class)
35  public class TestDefaultCompactSelection extends TestCompactionPolicy {
36  
37    @Test
38    public void testCompactionRatio() throws IOException {
39      /**
40       * NOTE: these tests are specific to describe the implementation of the
41       * current compaction algorithm.  Developed to ensure that refactoring
42       * doesn't implicitly alter this.
43       */
44      long tooBig = maxSize + 1;
45  
46      // default case. preserve user ratio on size
47      compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
48      // less than compact threshold = don't compact
49      compactEquals(sfCreate(100,50,25,12,12) /* empty */);
50      // greater than compact size = skip those
51      compactEquals(sfCreate(tooBig, tooBig, 700, 700, 700), 700, 700, 700);
52      // big size + threshold
53      compactEquals(sfCreate(tooBig, tooBig, 700,700) /* empty */);
54      // small files = don't care about ratio
55      compactEquals(sfCreate(7,1,1), 7,1,1);
56  
57      // don't exceed max file compact threshold
58      // note:  file selection starts with largest to smallest.
59      compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
60  
61      compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
62  
63      compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
64  
65      compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
66  
67      compactEquals(sfCreate(maxSize -1,maxSize -1,maxSize -1) /* empty */);
68  
69      // Always try and compact something to get below blocking storefile count
70      this.conf.setLong("hbase.hstore.compaction.min.size", 1);
71      store.storeEngine.getCompactionPolicy().setConf(conf);
72      compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1);
73      this.conf.setLong("hbase.hstore.compaction.min.size", minSize);
74      store.storeEngine.getCompactionPolicy().setConf(conf);
75  
76      /* MAJOR COMPACTION */
77      // if a major compaction has been forced, then compact everything
78      compactEquals(sfCreate(50,25,12,12), true, 50, 25, 12, 12);
79      // also choose files < threshold on major compaction
80      compactEquals(sfCreate(12,12), true, 12, 12);
81      // even if one of those files is too big
82      compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
83      // don't exceed max file compact threshold, even with major compaction
84      store.forceMajor = true;
85      compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
86      store.forceMajor = false;
87      // if we exceed maxCompactSize, downgrade to minor
88      // if not, it creates a 'snowball effect' when files >> maxCompactSize:
89      // the last file in compaction is the aggregate of all previous compactions
90      compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12);
91      conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
92      conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
93      store.storeEngine.getCompactionPolicy().setConf(conf);
94      ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
95      EnvironmentEdgeManager.injectEdge(timeMachine);
96      // Has to be  > 0 and < now.
97      timeMachine.setValue(1);
98      try {
99        // trigger an aged major compaction
100       compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12);
101       // major sure exceeding maxCompactSize also downgrades aged minors
102       compactEquals(sfCreate(100,50,23,12,12), 23, 12, 12);
103     } finally {
104       conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1000*60*60*24);
105       conf.setFloat("hbase.hregion.majorcompaction.jitter", 0.20F);
106     }
107 
108     /* REFERENCES == file is from a region that was split */
109     // treat storefiles that have references like a major compaction
110     compactEquals(sfCreate(true, 100,50,25,12,12), 100, 50, 25, 12, 12);
111     // reference files shouldn't obey max threshold
112     compactEquals(sfCreate(true, tooBig, 12,12), tooBig, 12, 12);
113     // reference files should obey max file compact to avoid OOM
114     compactEquals(sfCreate(true, 7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
115 
116     // empty case
117     compactEquals(new ArrayList<StoreFile>() /* empty */);
118     // empty case (because all files are too big)
119     compactEquals(sfCreate(tooBig, tooBig) /* empty */);
120   }
121 
122   @Test
123   public void testOffPeakCompactionRatio() throws IOException {
124     /*
125      * NOTE: these tests are specific to describe the implementation of the
126      * current compaction algorithm.  Developed to ensure that refactoring
127      * doesn't implicitly alter this.
128      */
129     // set an off-peak compaction threshold
130     this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
131     store.storeEngine.getCompactionPolicy().setConf(this.conf);
132     // Test with and without the flag.
133     compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
134     compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
135   }
136 
137   @Test
138   public void testStuckStoreCompaction() throws IOException {
139     // Select the smallest compaction if the store is stuck.
140     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,30), 30, 30, 30);
141     // If not stuck, standard policy applies.
142     compactEquals(sfCreate(99,99,99,99,99, 30,30,30,30), 99, 30, 30, 30, 30);
143 
144     // Add sufficiently small files to compaction, though
145     compactEquals(sfCreate(99,99,99,99,99,99, 30,30,30,15), 30, 30, 30, 15);
146     // Prefer earlier compaction to latter if the benefit is not significant
147     compactEquals(sfCreate(99,99,99,99, 30,26,26,29,25,25), 30, 26, 26);
148     // Prefer later compaction if the benefit is significant.
149     compactEquals(sfCreate(99,99,99,99, 27,27,27,20,20,20), 20, 20, 20);
150   }
151 
152   @Test
153   public void testCompactionEmptyHFile() throws IOException {
154     // Set TTL
155     ScanInfo oldScanInfo = store.getScanInfo();
156     ScanInfo newScanInfo = new ScanInfo(oldScanInfo.getFamily(),
157         oldScanInfo.getMinVersions(), oldScanInfo.getMaxVersions(), 600,
158         oldScanInfo.getKeepDeletedCells(), oldScanInfo.getTimeToPurgeDeletes(),
159         oldScanInfo.getComparator());
160     store.setScanInfo(newScanInfo);
161     // Do not compact empty store file
162     List<StoreFile> candidates = sfCreate(0);
163     for (StoreFile file : candidates) {
164       if (file instanceof MockStoreFile) {
165         MockStoreFile mockFile = (MockStoreFile) file;
166         mockFile.setTimeRangeTracker(new TimeRangeTracker(-1, -1));
167         mockFile.setEntries(0);
168       }
169     }
170     // Test Default compactions
171     CompactionRequest result = ((RatioBasedCompactionPolicy) store.storeEngine
172         .getCompactionPolicy()).selectCompaction(candidates,
173         new ArrayList<StoreFile>(), false, false, false);
174     Assert.assertTrue(result.getFiles().size() == 0);
175     store.setScanInfo(oldScanInfo);
176   }
177 }