1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.Random;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.LargeTests;
33 import org.apache.hadoop.hbase.client.HBaseAdmin;
34 import org.apache.hadoop.hbase.client.HTable;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
37 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.CompactionState;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44
45 @Category(LargeTests.class)
46 public class TestCompactionState {
47 final static Log LOG = LogFactory.getLog(TestCompactionState.class);
48 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49 private final static Random random = new Random();
50
51 @BeforeClass
52 public static void setUpBeforeClass() throws Exception {
53 TEST_UTIL.startMiniCluster();
54 }
55
56 @AfterClass
57 public static void tearDownAfterClass() throws Exception {
58 TEST_UTIL.shutdownMiniCluster();
59 }
60
61 @Test(timeout=60000)
62 public void testMajorCompaction() throws IOException, InterruptedException {
63 compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
64 }
65
66 @Test(timeout=60000)
67 public void testMinorCompaction() throws IOException, InterruptedException {
68 compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
69 }
70
71 @Test(timeout=60000)
72 public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
73 compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
74 }
75
76 @Test(timeout=60000)
77 public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
78 compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
79 }
80
81 @Test
82 public void testInvalidColumnFamily() throws IOException, InterruptedException {
83 byte [] table = Bytes.toBytes("testInvalidColumnFamily");
84 byte [] family = Bytes.toBytes("family");
85 byte [] fakecf = Bytes.toBytes("fakecf");
86 boolean caughtMinorCompact = false;
87 boolean caughtMajorCompact = false;
88 HTable ht = null;
89 try {
90 ht = TEST_UTIL.createTable(table, family);
91 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
92 try {
93 admin.compact(table, fakecf);
94 } catch (IOException ioe) {
95 caughtMinorCompact = true;
96 }
97 try {
98 admin.majorCompact(table, fakecf);
99 } catch (IOException ioe) {
100 caughtMajorCompact = true;
101 }
102 } finally {
103 if (ht != null) {
104 TEST_UTIL.deleteTable(table);
105 }
106 assertTrue(caughtMinorCompact);
107 assertTrue(caughtMajorCompact);
108 }
109 }
110
111
112
113
114
115
116
117
118
119
120
121
122 private void compaction(final String tableName, final int flushes,
123 final CompactionState expectedState, boolean singleFamily)
124 throws IOException, InterruptedException {
125
126 byte [] table = Bytes.toBytes(tableName);
127 byte [] family = Bytes.toBytes("family");
128 byte [][] families =
129 {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
130 HTable ht = null;
131 try {
132 ht = TEST_UTIL.createTable(table, families);
133 loadData(ht, families, 3000, flushes);
134 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
135 List<HRegion> regions = rs.getOnlineRegions(table);
136 int countBefore = countStoreFilesInFamilies(regions, families);
137 int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
138 assertTrue(countBefore > 0);
139 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
140 if (expectedState == CompactionState.MINOR) {
141 if (singleFamily) {
142 admin.compact(table, family);
143 } else {
144 admin.compact(table);
145 }
146 } else {
147 if (singleFamily) {
148 admin.majorCompact(table, family);
149 } else {
150 admin.majorCompact(table);
151 }
152 }
153 long curt = System.currentTimeMillis();
154 long waitTime = 5000;
155 long endt = curt + waitTime;
156 CompactionState state = admin.getCompactionState(table);
157 while (state == CompactionState.NONE && curt < endt) {
158 Thread.sleep(10);
159 state = admin.getCompactionState(table);
160 curt = System.currentTimeMillis();
161 }
162
163
164 if (expectedState != state) {
165 for (HRegion region: regions) {
166 state = CompactionRequest.getCompactionState(region.getRegionId());
167 assertEquals(CompactionState.NONE, state);
168 }
169 } else {
170 curt = System.currentTimeMillis();
171 waitTime = 20000;
172 endt = curt + waitTime;
173 state = admin.getCompactionState(table);
174 while (state != CompactionState.NONE && curt < endt) {
175 Thread.sleep(10);
176 state = admin.getCompactionState(table);
177 curt = System.currentTimeMillis();
178 }
179
180 assertEquals(CompactionState.NONE, state);
181 }
182 int countAfter = countStoreFilesInFamilies(regions, families);
183 int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
184 assertTrue(countAfter < countBefore);
185 if (!singleFamily) {
186 if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
187 else assertTrue(families.length < countAfter);
188 } else {
189 int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
190
191 assertTrue(singleFamDiff == (countBefore - countAfter));
192 if (expectedState == CompactionState.MAJOR) {
193 assertTrue(1 == countAfterSingleFamily);
194 } else {
195 assertTrue(1 < countAfterSingleFamily);
196 }
197 }
198 } finally {
199 if (ht != null) {
200 TEST_UTIL.deleteTable(table);
201 }
202 }
203 }
204
205 private static int countStoreFilesInFamily(
206 List<HRegion> regions, final byte[] family) {
207 return countStoreFilesInFamilies(regions, new byte[][]{family});
208 }
209
210 private static int countStoreFilesInFamilies(List<HRegion> regions, final byte[][] families) {
211 int count = 0;
212 for (HRegion region: regions) {
213 count += region.getStoreFileList(families).size();
214 }
215 return count;
216 }
217
218 private static void loadData(final HTable ht, final byte[][] families,
219 final int rows, final int flushes) throws IOException {
220 List<Put> puts = new ArrayList<Put>(rows);
221 byte[] qualifier = Bytes.toBytes("val");
222 for (int i = 0; i < flushes; i++) {
223 for (int k = 0; k < rows; k++) {
224 byte[] row = Bytes.toBytes(random.nextLong());
225 Put p = new Put(row);
226 for (int j = 0; j < families.length; ++j) {
227 p.add(families[ j ], qualifier, row);
228 }
229 puts.add(p);
230 }
231 ht.put(puts);
232 ht.flushCommits();
233 TEST_UTIL.flush();
234 puts.clear();
235 }
236 }
237
238 @org.junit.Rule
239 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
240 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
241 }