View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.List;
26  import java.util.concurrent.CountDownLatch;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.commons.logging.impl.Log4JLogger;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.HBaseAdmin;
35  import org.apache.hadoop.hbase.client.HTable;
36  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37  import org.apache.hadoop.hbase.regionserver.HRegion;
38  import org.apache.hadoop.hbase.regionserver.HRegionServer;
39  import org.apache.hadoop.hbase.regionserver.HStore;
40  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
41  import org.apache.hadoop.hbase.regionserver.Store;
42  import org.apache.hadoop.hbase.regionserver.StoreFile;
43  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
44  import org.apache.hadoop.hbase.regionserver.wal.HLog;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
47  import org.apache.hadoop.hdfs.DFSClient;
48  import org.apache.hadoop.hdfs.server.datanode.DataNode;
49  import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
50  import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
51  import org.apache.log4j.Level;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  /**
56   * Test for the case where a regionserver going down has enough cycles to do damage to regions
57   * that have actually been assigned elsehwere.
58   *
59   * <p>If we happen to assign a region before it fully done with in its old location -- i.e. it is on two servers at the
60   * same time -- all can work fine until the case where the region on the dying server decides to compact or otherwise
61   * change the region file set.  The region in its new location will then get a surprise when it tries to do something
62   * w/ a file removed by the region in its old location on dying server.
63   *
64   * <p>Making a test for this case is a little tough in that even if a file is deleted up on the namenode,
65   * if the file was opened before the delete, it will continue to let reads happen until something changes the
66   * state of cached blocks in the dfsclient that was already open (a block from the deleted file is cleaned
67   * from the datanode by NN).
68   *
69   * <p>What we will do below is do an explicit check for existence on the files listed in the region that
70   * has had some files removed because of a compaction.  This sort of hurry's along and makes certain what is a chance
71   * occurance.
72   */
73  @Category(MediumTests.class)
74  public class TestIOFencing {
75    static final Log LOG = LogFactory.getLog(TestIOFencing.class);
76    static {
77      ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
78      ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
79      ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
80      ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")).getLogger().setLevel(Level.ALL);
81      ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
82      ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL);
83    }
84  
85    public abstract static class CompactionBlockerRegion extends HRegion {
86      volatile int compactCount = 0;
87      volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
88      volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
89  
90      @SuppressWarnings("deprecation")
91      public CompactionBlockerRegion(Path tableDir, HLog log,
92          FileSystem fs, Configuration confParam, HRegionInfo info,
93          HTableDescriptor htd, RegionServerServices rsServices) {
94        super(tableDir, log, fs, confParam, info, htd, rsServices);
95      }
96  
97      public void stopCompactions() {
98        compactionsBlocked = new CountDownLatch(1);
99        compactionsWaiting = new CountDownLatch(1);
100     }
101 
102     public void allowCompactions() {
103       LOG.debug("allowing compactions");
104       compactionsBlocked.countDown();
105     }
106     public void waitForCompactionToBlock() throws IOException {
107       try {
108         LOG.debug("waiting for compaction to block");
109         compactionsWaiting.await();
110         LOG.debug("compaction block reached");
111       } catch (InterruptedException ex) {
112         throw new IOException(ex);
113       }
114     }
115     @Override
116     public boolean compact(CompactionContext compaction, Store store) throws IOException {
117       try {
118         return super.compact(compaction, store);
119       } finally {
120         compactCount++;
121       }
122     }
123     public int countStoreFiles() {
124       int count = 0;
125       for (Store store : stores.values()) {
126         count += store.getStorefilesCount();
127       }
128       return count;
129     }
130   }
131 
132   /**
133    * An override of HRegion that allows us park compactions in a holding pattern and
134    * then when appropriate for the test, allow them proceed again.
135    */
136   public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
137 
138     public BlockCompactionsInPrepRegion(Path tableDir, HLog log,
139         FileSystem fs, Configuration confParam, HRegionInfo info,
140         HTableDescriptor htd, RegionServerServices rsServices) {
141       super(tableDir, log, fs, confParam, info, htd, rsServices);
142     }
143     @Override
144     protected void doRegionCompactionPrep() throws IOException {
145       compactionsWaiting.countDown();
146       try {
147         compactionsBlocked.await();
148       } catch (InterruptedException ex) {
149         throw new IOException();
150       }
151       super.doRegionCompactionPrep();
152     }
153   }
154 
155   /**
156    * An override of HRegion that allows us park compactions in a holding pattern and
157    * then when appropriate for the test, allow them proceed again. This allows the compaction
158    * entry to go the WAL before blocking, but blocks afterwards
159    */
160   public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
161     public BlockCompactionsInCompletionRegion(Path tableDir, HLog log,
162         FileSystem fs, Configuration confParam, HRegionInfo info,
163         HTableDescriptor htd, RegionServerServices rsServices) {
164       super(tableDir, log, fs, confParam, info, htd, rsServices);
165     }
166     @Override
167     protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
168       return new BlockCompactionsInCompletionHStore(this, family, this.conf);
169     }
170   }
171 
172   public static class BlockCompactionsInCompletionHStore extends HStore {
173     CompactionBlockerRegion r;
174     protected BlockCompactionsInCompletionHStore(HRegion region, HColumnDescriptor family,
175         Configuration confParam) throws IOException {
176       super(region, family, confParam);
177       r = (CompactionBlockerRegion) region;
178     }
179 
180     @Override
181     protected void completeCompaction(Collection<StoreFile> compactedFiles) throws IOException {
182       try {
183         r.compactionsWaiting.countDown();
184         r.compactionsBlocked.await();
185       } catch (InterruptedException ex) {
186         throw new IOException(ex);
187       }
188       super.completeCompaction(compactedFiles);
189     }
190   }
191 
192   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
193   private final static TableName TABLE_NAME =
194       TableName.valueOf("tabletest");
195   private final static byte[] FAMILY = Bytes.toBytes("family");
196   private static final int FIRST_BATCH_COUNT = 4000;
197   private static final int SECOND_BATCH_COUNT = FIRST_BATCH_COUNT;
198 
199   /**
200    * Test that puts up a regionserver, starts a compaction on a loaded region but holds the
201    * compaction until after we have killed the server and the region has come up on
202    * a new regionserver altogether.  This fakes the double assignment case where region in one
203    * location changes the files out from underneath a region being served elsewhere.
204    */
205   @Test
206   public void testFencingAroundCompaction() throws Exception {
207     doTest(BlockCompactionsInPrepRegion.class);
208   }
209 
210   /**
211    * Test that puts up a regionserver, starts a compaction on a loaded region but holds the
212    * compaction completion until after we have killed the server and the region has come up on
213    * a new regionserver altogether.  This fakes the double assignment case where region in one
214    * location changes the files out from underneath a region being served elsewhere.
215    */
216   @Test
217   public void testFencingAroundCompactionAfterWALSync() throws Exception {
218     doTest(BlockCompactionsInCompletionRegion.class);
219   }
220 
221   public void doTest(Class<?> regionClass) throws Exception {
222     Configuration c = TEST_UTIL.getConfiguration();
223     c.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
224     // Insert our custom region
225     c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class);
226     c.setBoolean("dfs.support.append", true);
227     // Encourage plenty of flushes
228     c.setLong("hbase.hregion.memstore.flush.size", 200000);
229     c.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
230     // Only run compaction when we tell it to
231     c.setInt("hbase.hstore.compactionThreshold", 1000);
232     c.setLong("hbase.hstore.blockingStoreFiles", 1000);
233     // Compact quickly after we tell it to!
234     c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000);
235     LOG.info("Starting mini cluster");
236     TEST_UTIL.startMiniCluster(1);
237     CompactionBlockerRegion compactingRegion = null;
238     HBaseAdmin admin = null;
239     try {
240       LOG.info("Creating admin");
241       admin = new HBaseAdmin(c);
242       LOG.info("Creating table");
243       TEST_UTIL.createTable(TABLE_NAME, FAMILY);
244       HTable table = new HTable(c, TABLE_NAME);
245       LOG.info("Loading test table");
246       // Find the region
247       List<HRegion> testRegions = TEST_UTIL.getMiniHBaseCluster().findRegionsForTable(TABLE_NAME);
248       assertEquals(1, testRegions.size());
249       compactingRegion = (CompactionBlockerRegion)testRegions.get(0);
250       LOG.info("Blocking compactions");
251       compactingRegion.stopCompactions();
252       long lastFlushTime = compactingRegion.getLastFlushTime();
253       // Load some rows
254       TEST_UTIL.loadNumericRows(table, FAMILY, 0, FIRST_BATCH_COUNT);
255 
256       // Wait till flush has happened, otherwise there won't be multiple store files
257       long startWaitTime = System.currentTimeMillis();
258       while (compactingRegion.getLastFlushTime() <= lastFlushTime ||
259           compactingRegion.countStoreFiles() <= 1) {
260         LOG.info("Waiting for the region to flush " + compactingRegion.getRegionNameAsString());
261         Thread.sleep(1000);
262         assertTrue("Timed out waiting for the region to flush",
263           System.currentTimeMillis() - startWaitTime < 30000);
264       }
265       assertTrue(compactingRegion.countStoreFiles() > 1);
266       final byte REGION_NAME[] = compactingRegion.getRegionName();
267       LOG.info("Asking for compaction");
268       admin.majorCompact(TABLE_NAME.getName());
269       LOG.info("Waiting for compaction to be about to start");
270       compactingRegion.waitForCompactionToBlock();
271       LOG.info("Starting a new server");
272       RegionServerThread newServerThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer();
273       HRegionServer newServer = newServerThread.getRegionServer();
274       LOG.info("Killing region server ZK lease");
275       TEST_UTIL.expireRegionServerSession(0);
276       CompactionBlockerRegion newRegion = null;
277       startWaitTime = System.currentTimeMillis();
278       while (newRegion == null) {
279         LOG.info("Waiting for the new server to pick up the region " + Bytes.toString(REGION_NAME));
280         Thread.sleep(1000);
281         newRegion = (CompactionBlockerRegion)newServer.getOnlineRegion(REGION_NAME);
282         assertTrue("Timed out waiting for new server to open region",
283           System.currentTimeMillis() - startWaitTime < 300000);
284       }
285       LOG.info("Allowing compaction to proceed");
286       compactingRegion.allowCompactions();
287       while (compactingRegion.compactCount == 0) {
288         Thread.sleep(1000);
289       }
290       // The server we killed stays up until the compaction that was started before it was killed completes.  In logs
291       // you should see the old regionserver now going down.
292       LOG.info("Compaction finished");
293       // After compaction of old region finishes on the server that was going down, make sure that
294       // all the files we expect are still working when region is up in new location.
295       FileSystem fs = newRegion.getFilesystem();
296       for (String f: newRegion.getStoreFileList(new byte [][] {FAMILY})) {
297         assertTrue("After compaction, does not exist: " + f, fs.exists(new Path(f)));
298       }
299       // If we survive the split keep going...
300       // Now we make sure that the region isn't totally confused.  Load up more rows.
301       TEST_UTIL.loadNumericRows(table, FAMILY, FIRST_BATCH_COUNT, FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
302       admin.majorCompact(TABLE_NAME.getName());
303       startWaitTime = System.currentTimeMillis();
304       while (newRegion.compactCount == 0) {
305         Thread.sleep(1000);
306         assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000);
307       }
308       assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table));
309     } finally {
310       if (compactingRegion != null) {
311         compactingRegion.allowCompactions();
312       }
313       admin.close();
314       TEST_UTIL.shutdownMiniCluster();
315     }
316   }
317 }