1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.concurrent.CountDownLatch;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.commons.logging.impl.Log4JLogger;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.HBaseAdmin;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37 import org.apache.hadoop.hbase.regionserver.HRegion;
38 import org.apache.hadoop.hbase.regionserver.HRegionServer;
39 import org.apache.hadoop.hbase.regionserver.HStore;
40 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
41 import org.apache.hadoop.hbase.regionserver.Store;
42 import org.apache.hadoop.hbase.regionserver.StoreFile;
43 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
44 import org.apache.hadoop.hbase.regionserver.wal.HLog;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
47 import org.apache.hadoop.hdfs.DFSClient;
48 import org.apache.hadoop.hdfs.server.datanode.DataNode;
49 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
50 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
51 import org.apache.log4j.Level;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73 @Category(MediumTests.class)
74 public class TestIOFencing {
75 static final Log LOG = LogFactory.getLog(TestIOFencing.class);
76 static {
77 ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
78 ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
79 ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
80 ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")).getLogger().setLevel(Level.ALL);
81 ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
82 ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL);
83 }
84
85 public abstract static class CompactionBlockerRegion extends HRegion {
86 volatile int compactCount = 0;
87 volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
88 volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
89
90 @SuppressWarnings("deprecation")
91 public CompactionBlockerRegion(Path tableDir, HLog log,
92 FileSystem fs, Configuration confParam, HRegionInfo info,
93 HTableDescriptor htd, RegionServerServices rsServices) {
94 super(tableDir, log, fs, confParam, info, htd, rsServices);
95 }
96
97 public void stopCompactions() {
98 compactionsBlocked = new CountDownLatch(1);
99 compactionsWaiting = new CountDownLatch(1);
100 }
101
102 public void allowCompactions() {
103 LOG.debug("allowing compactions");
104 compactionsBlocked.countDown();
105 }
106 public void waitForCompactionToBlock() throws IOException {
107 try {
108 LOG.debug("waiting for compaction to block");
109 compactionsWaiting.await();
110 LOG.debug("compaction block reached");
111 } catch (InterruptedException ex) {
112 throw new IOException(ex);
113 }
114 }
115 @Override
116 public boolean compact(CompactionContext compaction, Store store) throws IOException {
117 try {
118 return super.compact(compaction, store);
119 } finally {
120 compactCount++;
121 }
122 }
123 public int countStoreFiles() {
124 int count = 0;
125 for (Store store : stores.values()) {
126 count += store.getStorefilesCount();
127 }
128 return count;
129 }
130 }
131
132
133
134
135
136 public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
137
138 public BlockCompactionsInPrepRegion(Path tableDir, HLog log,
139 FileSystem fs, Configuration confParam, HRegionInfo info,
140 HTableDescriptor htd, RegionServerServices rsServices) {
141 super(tableDir, log, fs, confParam, info, htd, rsServices);
142 }
143 @Override
144 protected void doRegionCompactionPrep() throws IOException {
145 compactionsWaiting.countDown();
146 try {
147 compactionsBlocked.await();
148 } catch (InterruptedException ex) {
149 throw new IOException();
150 }
151 super.doRegionCompactionPrep();
152 }
153 }
154
155
156
157
158
159
160 public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
161 public BlockCompactionsInCompletionRegion(Path tableDir, HLog log,
162 FileSystem fs, Configuration confParam, HRegionInfo info,
163 HTableDescriptor htd, RegionServerServices rsServices) {
164 super(tableDir, log, fs, confParam, info, htd, rsServices);
165 }
166 @Override
167 protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
168 return new BlockCompactionsInCompletionHStore(this, family, this.conf);
169 }
170 }
171
172 public static class BlockCompactionsInCompletionHStore extends HStore {
173 CompactionBlockerRegion r;
174 protected BlockCompactionsInCompletionHStore(HRegion region, HColumnDescriptor family,
175 Configuration confParam) throws IOException {
176 super(region, family, confParam);
177 r = (CompactionBlockerRegion) region;
178 }
179
180 @Override
181 protected void completeCompaction(Collection<StoreFile> compactedFiles) throws IOException {
182 try {
183 r.compactionsWaiting.countDown();
184 r.compactionsBlocked.await();
185 } catch (InterruptedException ex) {
186 throw new IOException(ex);
187 }
188 super.completeCompaction(compactedFiles);
189 }
190 }
191
192 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
193 private final static TableName TABLE_NAME =
194 TableName.valueOf("tabletest");
195 private final static byte[] FAMILY = Bytes.toBytes("family");
196 private static final int FIRST_BATCH_COUNT = 4000;
197 private static final int SECOND_BATCH_COUNT = FIRST_BATCH_COUNT;
198
199
200
201
202
203
204
205 @Test
206 public void testFencingAroundCompaction() throws Exception {
207 doTest(BlockCompactionsInPrepRegion.class);
208 }
209
210
211
212
213
214
215
216 @Test
217 public void testFencingAroundCompactionAfterWALSync() throws Exception {
218 doTest(BlockCompactionsInCompletionRegion.class);
219 }
220
221 public void doTest(Class<?> regionClass) throws Exception {
222 Configuration c = TEST_UTIL.getConfiguration();
223 c.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
224
225 c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class);
226 c.setBoolean("dfs.support.append", true);
227
228 c.setLong("hbase.hregion.memstore.flush.size", 200000);
229 c.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
230
231 c.setInt("hbase.hstore.compactionThreshold", 1000);
232 c.setLong("hbase.hstore.blockingStoreFiles", 1000);
233
234 c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000);
235 LOG.info("Starting mini cluster");
236 TEST_UTIL.startMiniCluster(1);
237 CompactionBlockerRegion compactingRegion = null;
238 HBaseAdmin admin = null;
239 try {
240 LOG.info("Creating admin");
241 admin = new HBaseAdmin(c);
242 LOG.info("Creating table");
243 TEST_UTIL.createTable(TABLE_NAME, FAMILY);
244 HTable table = new HTable(c, TABLE_NAME);
245 LOG.info("Loading test table");
246
247 List<HRegion> testRegions = TEST_UTIL.getMiniHBaseCluster().findRegionsForTable(TABLE_NAME);
248 assertEquals(1, testRegions.size());
249 compactingRegion = (CompactionBlockerRegion)testRegions.get(0);
250 LOG.info("Blocking compactions");
251 compactingRegion.stopCompactions();
252 long lastFlushTime = compactingRegion.getLastFlushTime();
253
254 TEST_UTIL.loadNumericRows(table, FAMILY, 0, FIRST_BATCH_COUNT);
255
256
257 long startWaitTime = System.currentTimeMillis();
258 while (compactingRegion.getLastFlushTime() <= lastFlushTime ||
259 compactingRegion.countStoreFiles() <= 1) {
260 LOG.info("Waiting for the region to flush " + compactingRegion.getRegionNameAsString());
261 Thread.sleep(1000);
262 assertTrue("Timed out waiting for the region to flush",
263 System.currentTimeMillis() - startWaitTime < 30000);
264 }
265 assertTrue(compactingRegion.countStoreFiles() > 1);
266 final byte REGION_NAME[] = compactingRegion.getRegionName();
267 LOG.info("Asking for compaction");
268 admin.majorCompact(TABLE_NAME.getName());
269 LOG.info("Waiting for compaction to be about to start");
270 compactingRegion.waitForCompactionToBlock();
271 LOG.info("Starting a new server");
272 RegionServerThread newServerThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer();
273 HRegionServer newServer = newServerThread.getRegionServer();
274 LOG.info("Killing region server ZK lease");
275 TEST_UTIL.expireRegionServerSession(0);
276 CompactionBlockerRegion newRegion = null;
277 startWaitTime = System.currentTimeMillis();
278 while (newRegion == null) {
279 LOG.info("Waiting for the new server to pick up the region " + Bytes.toString(REGION_NAME));
280 Thread.sleep(1000);
281 newRegion = (CompactionBlockerRegion)newServer.getOnlineRegion(REGION_NAME);
282 assertTrue("Timed out waiting for new server to open region",
283 System.currentTimeMillis() - startWaitTime < 300000);
284 }
285 LOG.info("Allowing compaction to proceed");
286 compactingRegion.allowCompactions();
287 while (compactingRegion.compactCount == 0) {
288 Thread.sleep(1000);
289 }
290
291
292 LOG.info("Compaction finished");
293
294
295 FileSystem fs = newRegion.getFilesystem();
296 for (String f: newRegion.getStoreFileList(new byte [][] {FAMILY})) {
297 assertTrue("After compaction, does not exist: " + f, fs.exists(new Path(f)));
298 }
299
300
301 TEST_UTIL.loadNumericRows(table, FAMILY, FIRST_BATCH_COUNT, FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
302 admin.majorCompact(TABLE_NAME.getName());
303 startWaitTime = System.currentTimeMillis();
304 while (newRegion.compactCount == 0) {
305 Thread.sleep(1000);
306 assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000);
307 }
308 assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table));
309 } finally {
310 if (compactingRegion != null) {
311 compactingRegion.allowCompactions();
312 }
313 admin.close();
314 TEST_UTIL.shutdownMiniCluster();
315 }
316 }
317 }