1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.List;
26 import java.util.concurrent.CountDownLatch;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.commons.logging.impl.Log4JLogger;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.HBaseAdmin;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37 import org.apache.hadoop.hbase.regionserver.HRegion;
38 import org.apache.hadoop.hbase.regionserver.HRegionServer;
39 import org.apache.hadoop.hbase.regionserver.HStore;
40 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
41 import org.apache.hadoop.hbase.regionserver.Store;
42 import org.apache.hadoop.hbase.regionserver.StoreFile;
43 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
44 import org.apache.hadoop.hbase.regionserver.wal.HLog;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
47 import org.apache.hadoop.hdfs.DFSClient;
48 import org.apache.hadoop.hdfs.server.datanode.DataNode;
49 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
50 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
51 import org.apache.log4j.Level;
52 import org.junit.Ignore;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74 @Category(MediumTests.class)
75 public class TestIOFencing {
76 static final Log LOG = LogFactory.getLog(TestIOFencing.class);
77 static {
78 ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
79 ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
80 ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
81 ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")).getLogger().setLevel(Level.ALL);
82 ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
83 ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL);
84 }
85
86 public abstract static class CompactionBlockerRegion extends HRegion {
87 volatile int compactCount = 0;
88 volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
89 volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
90
91 @SuppressWarnings("deprecation")
92 public CompactionBlockerRegion(Path tableDir, HLog log,
93 FileSystem fs, Configuration confParam, HRegionInfo info,
94 HTableDescriptor htd, RegionServerServices rsServices) {
95 super(tableDir, log, fs, confParam, info, htd, rsServices);
96 }
97
98 public void stopCompactions() {
99 compactionsBlocked = new CountDownLatch(1);
100 compactionsWaiting = new CountDownLatch(1);
101 }
102
103 public void allowCompactions() {
104 LOG.debug("allowing compactions");
105 compactionsBlocked.countDown();
106 }
107 public void waitForCompactionToBlock() throws IOException {
108 try {
109 LOG.debug("waiting for compaction to block");
110 compactionsWaiting.await();
111 LOG.debug("compaction block reached");
112 } catch (InterruptedException ex) {
113 throw new IOException(ex);
114 }
115 }
116 @Override
117 public boolean compact(CompactionContext compaction, Store store) throws IOException {
118 try {
119 return super.compact(compaction, store);
120 } finally {
121 compactCount++;
122 }
123 }
124 public int countStoreFiles() {
125 int count = 0;
126 for (Store store : stores.values()) {
127 count += store.getStorefilesCount();
128 }
129 return count;
130 }
131 }
132
133
134
135
136
137 public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
138
139 public BlockCompactionsInPrepRegion(Path tableDir, HLog log,
140 FileSystem fs, Configuration confParam, HRegionInfo info,
141 HTableDescriptor htd, RegionServerServices rsServices) {
142 super(tableDir, log, fs, confParam, info, htd, rsServices);
143 }
144 @Override
145 protected void doRegionCompactionPrep() throws IOException {
146 compactionsWaiting.countDown();
147 try {
148 compactionsBlocked.await();
149 } catch (InterruptedException ex) {
150 throw new IOException();
151 }
152 super.doRegionCompactionPrep();
153 }
154 }
155
156
157
158
159
160
161 public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
162 public BlockCompactionsInCompletionRegion(Path tableDir, HLog log,
163 FileSystem fs, Configuration confParam, HRegionInfo info,
164 HTableDescriptor htd, RegionServerServices rsServices) {
165 super(tableDir, log, fs, confParam, info, htd, rsServices);
166 }
167 @Override
168 protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
169 return new BlockCompactionsInCompletionHStore(this, family, this.conf);
170 }
171 }
172
173 public static class BlockCompactionsInCompletionHStore extends HStore {
174 CompactionBlockerRegion r;
175 protected BlockCompactionsInCompletionHStore(HRegion region, HColumnDescriptor family,
176 Configuration confParam) throws IOException {
177 super(region, family, confParam);
178 r = (CompactionBlockerRegion) region;
179 }
180
181 @Override
182 protected void completeCompaction(Collection<StoreFile> compactedFiles) throws IOException {
183 try {
184 r.compactionsWaiting.countDown();
185 r.compactionsBlocked.await();
186 } catch (InterruptedException ex) {
187 throw new IOException(ex);
188 }
189 super.completeCompaction(compactedFiles);
190 }
191 }
192
193 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
194 private final static TableName TABLE_NAME =
195 TableName.valueOf("tabletest");
196 private final static byte[] FAMILY = Bytes.toBytes("family");
197 private static final int FIRST_BATCH_COUNT = 4000;
198 private static final int SECOND_BATCH_COUNT = FIRST_BATCH_COUNT;
199
200
201
202
203
204
205
206 @Ignore("See HBASE-10298")
207 @Test
208 public void testFencingAroundCompaction() throws Exception {
209 doTest(BlockCompactionsInPrepRegion.class);
210 }
211
212
213
214
215
216
217
218 @Ignore("See HBASE-10298")
219 @Test
220 public void testFencingAroundCompactionAfterWALSync() throws Exception {
221 doTest(BlockCompactionsInCompletionRegion.class);
222 }
223
224 public void doTest(Class<?> regionClass) throws Exception {
225 Configuration c = TEST_UTIL.getConfiguration();
226 c.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
227
228 c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class);
229 c.setBoolean("dfs.support.append", true);
230
231 c.setLong("hbase.hregion.memstore.flush.size", 200000);
232 c.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
233
234 c.setInt("hbase.hstore.compactionThreshold", 1000);
235 c.setLong("hbase.hstore.blockingStoreFiles", 1000);
236
237 c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000);
238 LOG.info("Starting mini cluster");
239 TEST_UTIL.startMiniCluster(1);
240 CompactionBlockerRegion compactingRegion = null;
241 HBaseAdmin admin = null;
242 try {
243 LOG.info("Creating admin");
244 admin = new HBaseAdmin(c);
245 LOG.info("Creating table");
246 TEST_UTIL.createTable(TABLE_NAME, FAMILY);
247 HTable table = new HTable(c, TABLE_NAME);
248 LOG.info("Loading test table");
249
250 List<HRegion> testRegions = TEST_UTIL.getMiniHBaseCluster().findRegionsForTable(TABLE_NAME);
251 assertEquals(1, testRegions.size());
252 compactingRegion = (CompactionBlockerRegion)testRegions.get(0);
253 LOG.info("Blocking compactions");
254 compactingRegion.stopCompactions();
255 long lastFlushTime = compactingRegion.getLastFlushTime();
256
257 TEST_UTIL.loadNumericRows(table, FAMILY, 0, FIRST_BATCH_COUNT);
258
259
260 long startWaitTime = System.currentTimeMillis();
261 while (compactingRegion.getLastFlushTime() <= lastFlushTime ||
262 compactingRegion.countStoreFiles() <= 1) {
263 LOG.info("Waiting for the region to flush " + compactingRegion.getRegionNameAsString());
264 Thread.sleep(1000);
265 assertTrue("Timed out waiting for the region to flush",
266 System.currentTimeMillis() - startWaitTime < 30000);
267 }
268 assertTrue(compactingRegion.countStoreFiles() > 1);
269 final byte REGION_NAME[] = compactingRegion.getRegionName();
270 LOG.info("Asking for compaction");
271 admin.majorCompact(TABLE_NAME.getName());
272 LOG.info("Waiting for compaction to be about to start");
273 compactingRegion.waitForCompactionToBlock();
274 LOG.info("Starting a new server");
275 RegionServerThread newServerThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer();
276 HRegionServer newServer = newServerThread.getRegionServer();
277 LOG.info("Killing region server ZK lease");
278 TEST_UTIL.expireRegionServerSession(0);
279 CompactionBlockerRegion newRegion = null;
280 startWaitTime = System.currentTimeMillis();
281 while (newRegion == null) {
282 LOG.info("Waiting for the new server to pick up the region " + Bytes.toString(REGION_NAME));
283 Thread.sleep(1000);
284 newRegion = (CompactionBlockerRegion)newServer.getOnlineRegion(REGION_NAME);
285 assertTrue("Timed out waiting for new server to open region",
286 System.currentTimeMillis() - startWaitTime < 300000);
287 }
288 LOG.info("Allowing compaction to proceed");
289 compactingRegion.allowCompactions();
290 while (compactingRegion.compactCount == 0) {
291 Thread.sleep(1000);
292 }
293
294
295 LOG.info("Compaction finished");
296
297
298 FileSystem fs = newRegion.getFilesystem();
299 for (String f: newRegion.getStoreFileList(new byte [][] {FAMILY})) {
300 assertTrue("After compaction, does not exist: " + f, fs.exists(new Path(f)));
301 }
302
303
304 TEST_UTIL.loadNumericRows(table, FAMILY, FIRST_BATCH_COUNT, FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
305 admin.majorCompact(TABLE_NAME.getName());
306 startWaitTime = System.currentTimeMillis();
307 while (newRegion.compactCount == 0) {
308 Thread.sleep(1000);
309 assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000);
310 }
311 assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table));
312 } finally {
313 if (compactingRegion != null) {
314 compactingRegion.allowCompactions();
315 }
316 admin.close();
317 TEST_UTIL.shutdownMiniCluster();
318 }
319 }
320 }