1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.compactions;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.List;
25 import java.util.Random;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.MiniHBaseCluster;
33 import org.apache.hadoop.hbase.TableName;
34 import org.apache.hadoop.hbase.client.HBaseAdmin;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
38 import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
39 import org.apache.hadoop.hbase.regionserver.HRegion;
40 import org.apache.hadoop.hbase.regionserver.HRegionServer;
41 import org.apache.hadoop.hbase.regionserver.HStore;
42 import org.apache.hadoop.hbase.regionserver.Store;
43 import org.apache.hadoop.hbase.testclassification.MediumTests;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.EnvironmentEdge;
46 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
47 import org.apache.hadoop.hbase.util.JVMClusterUtil;
48 import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge;
49 import org.junit.AfterClass;
50 import org.junit.Assert;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 @Category({ MediumTests.class })
56 public class TestFIFOCompactionPolicy {
57
58 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59
60
61 private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
62
63 private final byte[] family = Bytes.toBytes("f");
64
65 private final byte[] qualifier = Bytes.toBytes("q");
66
67 private Store getStoreWithName(TableName tableName) {
68 MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster();
69 List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
70 for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
71 HRegionServer hrs = rsts.get(i).getRegionServer();
72 for (HRegion region : hrs.getOnlineRegions(tableName)) {
73 return region.getStores().values().iterator().next();
74 }
75 }
76 return null;
77 }
78
79 private Store prepareData() throws IOException, InterruptedException {
80 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
81 if (admin.tableExists(tableName)) {
82 admin.disableTable(tableName);
83 admin.deleteTable(tableName);
84 }
85 HTableDescriptor desc = new HTableDescriptor(tableName);
86 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
87 FIFOCompactionPolicy.class.getName());
88 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
89 DisabledRegionSplitPolicy.class.getName());
90 HColumnDescriptor colDesc = new HColumnDescriptor(family);
91 colDesc.setTimeToLive(20);
92 desc.addFamily(colDesc);
93
94 HTable table = TEST_UTIL.createTable(desc, new byte[][]{});
95 Random rand = new Random();
96 TimeOffsetEnvironmentEdge edge =
97 (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
98 for (int i = 0; i < 10; i++) {
99 for (int j = 0; j < 10; j++) {
100 byte[] value = new byte[128 * 1024];
101 rand.nextBytes(value);
102 table.put(new Put(Bytes.toBytes(i * 10 + j)).add(family, qualifier, value));
103 }
104 admin.flush(tableName.getNameAsString());
105 }
106 edge.increment(20001);
107 return getStoreWithName(tableName);
108 }
109
110 @BeforeClass
111 public static void setEnvironmentEdge()
112 {
113 EnvironmentEdge ee = new TimeOffsetEnvironmentEdge();
114 EnvironmentEdgeManager.injectEdge(ee);
115 }
116
117 @AfterClass
118 public static void resetEnvironmentEdge()
119 {
120 EnvironmentEdgeManager.reset();
121 }
122
123 @Test
124 public void testPurgeExpiredFiles() throws Exception {
125 Configuration conf = TEST_UTIL.getConfiguration();
126 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
127
128 TEST_UTIL.startMiniCluster(1);
129 try {
130 Store store = prepareData();
131 assertEquals(10, store.getStorefilesCount());
132 TEST_UTIL.getHBaseAdmin().majorCompact(tableName.getNameAsString());
133 while (store.getStorefilesCount() > 1) {
134 Thread.sleep(100);
135 }
136 assertTrue(store.getStorefilesCount() == 1);
137 } finally {
138 TEST_UTIL.shutdownMiniCluster();
139 }
140 }
141
142 @Test
143 public void testSanityCheckTTL() throws Exception
144 {
145 Configuration conf = TEST_UTIL.getConfiguration();
146 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
147 TEST_UTIL.startMiniCluster(1);
148
149 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
150 String tableName = this.tableName.getNameAsString()+"-TTL";
151 if (admin.tableExists(tableName)) {
152 admin.disableTable(tableName);
153 admin.deleteTable(tableName);
154 }
155 HTableDescriptor desc = new HTableDescriptor(tableName);
156 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
157 FIFOCompactionPolicy.class.getName());
158 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
159 DisabledRegionSplitPolicy.class.getName());
160 HColumnDescriptor colDesc = new HColumnDescriptor(family);
161 desc.addFamily(colDesc);
162 try{
163 admin.createTable(desc);
164 Assert.fail();
165 }catch(Exception e){
166 }finally{
167 TEST_UTIL.shutdownMiniCluster();
168 }
169 }
170
171 @Test
172 public void testSanityCheckMinVersion() throws Exception
173 {
174 Configuration conf = TEST_UTIL.getConfiguration();
175 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
176 TEST_UTIL.startMiniCluster(1);
177
178 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
179 String tableName = this.tableName.getNameAsString()+"-MinVersion";
180 if (admin.tableExists(tableName)) {
181 admin.disableTable(tableName);
182 admin.deleteTable(tableName);
183 }
184 HTableDescriptor desc = new HTableDescriptor(tableName);
185 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
186 FIFOCompactionPolicy.class.getName());
187 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
188 DisabledRegionSplitPolicy.class.getName());
189 HColumnDescriptor colDesc = new HColumnDescriptor(family);
190 colDesc.setTimeToLive(1);
191 colDesc.setMinVersions(1);
192 desc.addFamily(colDesc);
193 try{
194 admin.createTable(desc);
195 Assert.fail();
196 }catch(Exception e){
197 }finally{
198 TEST_UTIL.shutdownMiniCluster();
199 }
200 }
201
202 @Test
203 public void testSanityCheckBlockingStoreFiles() throws Exception
204 {
205 Configuration conf = TEST_UTIL.getConfiguration();
206 conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10);
207 TEST_UTIL.startMiniCluster(1);
208
209 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
210 String tableName = this.tableName.getNameAsString()+"-MinVersion";
211 if (admin.tableExists(tableName)) {
212 admin.disableTable(tableName);
213 admin.deleteTable(tableName);
214 }
215 HTableDescriptor desc = new HTableDescriptor(tableName);
216 desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
217 FIFOCompactionPolicy.class.getName());
218 desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
219 DisabledRegionSplitPolicy.class.getName());
220 HColumnDescriptor colDesc = new HColumnDescriptor(family);
221 colDesc.setTimeToLive(1);
222 desc.addFamily(colDesc);
223 try{
224 admin.createTable(desc);
225 Assert.fail();
226 }catch(Exception e){
227 }finally{
228 TEST_UTIL.shutdownMiniCluster();
229 }
230 }
231 }