1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.concurrent.CountDownLatch;
27 import java.util.concurrent.atomic.AtomicLong;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.Admin;
35 import org.apache.hadoop.hbase.client.HBaseAdmin;
36 import org.apache.hadoop.hbase.client.Table;
37 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
38 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
39 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
40 import org.apache.hadoop.hbase.regionserver.HRegion;
41 import org.apache.hadoop.hbase.regionserver.HRegionServer;
42 import org.apache.hadoop.hbase.regionserver.HStore;
43 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
44 import org.apache.hadoop.hbase.regionserver.Store;
45 import org.apache.hadoop.hbase.regionserver.StoreFile;
46 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
47 import org.apache.hadoop.hbase.testclassification.MediumTests;
48 import org.apache.hadoop.hbase.wal.WAL;
49 import org.apache.hadoop.hbase.regionserver.wal.WALUtil;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 import com.google.common.collect.Lists;
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75 @Category(MediumTests.class)
76 public class TestIOFencing {
77 static final Log LOG = LogFactory.getLog(TestIOFencing.class);
78 static {
79
80
81
82
83
84
85
86
87 }
88
89 public abstract static class CompactionBlockerRegion extends HRegion {
90 volatile int compactCount = 0;
91 volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
92 volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
93
94 @SuppressWarnings("deprecation")
95 public CompactionBlockerRegion(Path tableDir, WAL log,
96 FileSystem fs, Configuration confParam, HRegionInfo info,
97 HTableDescriptor htd, RegionServerServices rsServices) {
98 super(tableDir, log, fs, confParam, info, htd, rsServices);
99 }
100
101 public void stopCompactions() {
102 compactionsBlocked = new CountDownLatch(1);
103 compactionsWaiting = new CountDownLatch(1);
104 }
105
106 public void allowCompactions() {
107 LOG.debug("allowing compactions");
108 compactionsBlocked.countDown();
109 }
110 public void waitForCompactionToBlock() throws IOException {
111 try {
112 LOG.debug("waiting for compaction to block");
113 compactionsWaiting.await();
114 LOG.debug("compaction block reached");
115 } catch (InterruptedException ex) {
116 throw new IOException(ex);
117 }
118 }
119 @Override
120 public boolean compact(CompactionContext compaction, Store store) throws IOException {
121 try {
122 return super.compact(compaction, store);
123 } finally {
124 compactCount++;
125 }
126 }
127 public int countStoreFiles() {
128 int count = 0;
129 for (Store store : stores.values()) {
130 count += store.getStorefilesCount();
131 }
132 return count;
133 }
134 }
135
136
137
138
139
140 public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
141
142 public BlockCompactionsInPrepRegion(Path tableDir, WAL log,
143 FileSystem fs, Configuration confParam, HRegionInfo info,
144 HTableDescriptor htd, RegionServerServices rsServices) {
145 super(tableDir, log, fs, confParam, info, htd, rsServices);
146 }
147 @Override
148 protected void doRegionCompactionPrep() throws IOException {
149 compactionsWaiting.countDown();
150 try {
151 compactionsBlocked.await();
152 } catch (InterruptedException ex) {
153 throw new IOException();
154 }
155 super.doRegionCompactionPrep();
156 }
157 }
158
159
160
161
162
163
164 public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
165 public BlockCompactionsInCompletionRegion(Path tableDir, WAL log,
166 FileSystem fs, Configuration confParam, HRegionInfo info,
167 HTableDescriptor htd, RegionServerServices rsServices) {
168 super(tableDir, log, fs, confParam, info, htd, rsServices);
169 }
170 @Override
171 protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
172 return new BlockCompactionsInCompletionHStore(this, family, this.conf);
173 }
174 }
175
176 public static class BlockCompactionsInCompletionHStore extends HStore {
177 CompactionBlockerRegion r;
178 protected BlockCompactionsInCompletionHStore(HRegion region, HColumnDescriptor family,
179 Configuration confParam) throws IOException {
180 super(region, family, confParam);
181 r = (CompactionBlockerRegion) region;
182 }
183
184 @Override
185 protected void completeCompaction(final Collection<StoreFile> compactedFiles,
186 boolean removeFiles) throws IOException {
187 try {
188 r.compactionsWaiting.countDown();
189 r.compactionsBlocked.await();
190 } catch (InterruptedException ex) {
191 throw new IOException(ex);
192 }
193 super.completeCompaction(compactedFiles, removeFiles);
194 }
195 @Override
196 protected void completeCompaction(Collection<StoreFile> compactedFiles) throws IOException {
197 try {
198 r.compactionsWaiting.countDown();
199 r.compactionsBlocked.await();
200 } catch (InterruptedException ex) {
201 throw new IOException(ex);
202 }
203 super.completeCompaction(compactedFiles);
204 }
205 }
206
207 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
208 private final static TableName TABLE_NAME =
209 TableName.valueOf("tabletest");
210 private final static byte[] FAMILY = Bytes.toBytes("family");
211 private static final int FIRST_BATCH_COUNT = 4000;
212 private static final int SECOND_BATCH_COUNT = FIRST_BATCH_COUNT;
213
214
215
216
217
218
219
220 @Test
221 public void testFencingAroundCompaction() throws Exception {
222 doTest(BlockCompactionsInPrepRegion.class, false);
223 doTest(BlockCompactionsInPrepRegion.class, true);
224 }
225
226
227
228
229
230
231
232 @Test
233 public void testFencingAroundCompactionAfterWALSync() throws Exception {
234 doTest(BlockCompactionsInCompletionRegion.class, false);
235 doTest(BlockCompactionsInCompletionRegion.class, true);
236 }
237
238 public void doTest(Class<?> regionClass, boolean distributedLogReplay) throws Exception {
239 Configuration c = TEST_UTIL.getConfiguration();
240 c.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, distributedLogReplay);
241
242 c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class);
243 c.setBoolean("dfs.support.append", true);
244
245 c.setLong("hbase.hregion.memstore.flush.size", 200000);
246 c.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
247
248 c.setInt("hbase.hstore.compactionThreshold", 1000);
249 c.setLong("hbase.hstore.blockingStoreFiles", 1000);
250
251 c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000);
252 LOG.info("Starting mini cluster");
253 TEST_UTIL.startMiniCluster(1);
254 CompactionBlockerRegion compactingRegion = null;
255 Admin admin = null;
256 try {
257 LOG.info("Creating admin");
258 admin = TEST_UTIL.getConnection().getAdmin();
259 LOG.info("Creating table");
260 TEST_UTIL.createTable(TABLE_NAME, FAMILY);
261 Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME);
262 LOG.info("Loading test table");
263
264 List<HRegion> testRegions = TEST_UTIL.getMiniHBaseCluster().findRegionsForTable(TABLE_NAME);
265 assertEquals(1, testRegions.size());
266 compactingRegion = (CompactionBlockerRegion)testRegions.get(0);
267 LOG.info("Blocking compactions");
268 compactingRegion.stopCompactions();
269 long lastFlushTime = compactingRegion.getLastFlushTime();
270
271 TEST_UTIL.loadNumericRows(table, FAMILY, 0, FIRST_BATCH_COUNT);
272
273
274
275 HRegionInfo oldHri = new HRegionInfo(table.getName(),
276 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
277 CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(oldHri,
278 FAMILY, Lists.newArrayList(new Path("/a")), Lists.newArrayList(new Path("/b")),
279 new Path("store_dir"));
280 WALUtil.writeCompactionMarker(compactingRegion.getWAL(), table.getTableDescriptor(),
281 oldHri, compactionDescriptor, new AtomicLong(Long.MAX_VALUE-100));
282
283
284 long startWaitTime = System.currentTimeMillis();
285 while (compactingRegion.getLastFlushTime() <= lastFlushTime ||
286 compactingRegion.countStoreFiles() <= 1) {
287 LOG.info("Waiting for the region to flush " + compactingRegion.getRegionNameAsString());
288 Thread.sleep(1000);
289 assertTrue("Timed out waiting for the region to flush",
290 System.currentTimeMillis() - startWaitTime < 30000);
291 }
292 assertTrue(compactingRegion.countStoreFiles() > 1);
293 final byte REGION_NAME[] = compactingRegion.getRegionName();
294 LOG.info("Asking for compaction");
295 ((HBaseAdmin)admin).majorCompact(TABLE_NAME.getName());
296 LOG.info("Waiting for compaction to be about to start");
297 compactingRegion.waitForCompactionToBlock();
298 LOG.info("Starting a new server");
299 RegionServerThread newServerThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer();
300 final HRegionServer newServer = newServerThread.getRegionServer();
301 LOG.info("Killing region server ZK lease");
302 TEST_UTIL.expireRegionServerSession(0);
303 CompactionBlockerRegion newRegion = null;
304 startWaitTime = System.currentTimeMillis();
305 LOG.info("Waiting for the new server to pick up the region " + Bytes.toString(REGION_NAME));
306
307
308 Waiter.waitFor(c, 60000, new Waiter.Predicate<Exception>() {
309 @Override
310 public boolean evaluate() throws Exception {
311 HRegion newRegion = newServer.getOnlineRegion(REGION_NAME);
312 return newRegion != null && !newRegion.isRecovering();
313 }
314 });
315
316 newRegion = (CompactionBlockerRegion)newServer.getOnlineRegion(REGION_NAME);
317
318 LOG.info("Allowing compaction to proceed");
319 compactingRegion.allowCompactions();
320 while (compactingRegion.compactCount == 0) {
321 Thread.sleep(1000);
322 }
323
324
325 LOG.info("Compaction finished");
326
327
328 FileSystem fs = newRegion.getFilesystem();
329 for (String f: newRegion.getStoreFileList(new byte [][] {FAMILY})) {
330 assertTrue("After compaction, does not exist: " + f, fs.exists(new Path(f)));
331 }
332
333
334 TEST_UTIL.loadNumericRows(table, FAMILY, FIRST_BATCH_COUNT, FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
335 ((HBaseAdmin)admin).majorCompact(TABLE_NAME.getName());
336 startWaitTime = System.currentTimeMillis();
337 while (newRegion.compactCount == 0) {
338 Thread.sleep(1000);
339 assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000);
340 }
341 assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table));
342 } finally {
343 if (compactingRegion != null) {
344 compactingRegion.allowCompactions();
345 }
346 admin.close();
347 TEST_UTIL.shutdownMiniCluster();
348 }
349 }
350 }