View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.List;
26  import java.util.concurrent.CountDownLatch;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.commons.logging.impl.Log4JLogger;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.client.HBaseAdmin;
35  import org.apache.hadoop.hbase.client.HTable;
36  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37  import org.apache.hadoop.hbase.regionserver.HRegion;
38  import org.apache.hadoop.hbase.regionserver.HRegionServer;
39  import org.apache.hadoop.hbase.regionserver.HStore;
40  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
41  import org.apache.hadoop.hbase.regionserver.Store;
42  import org.apache.hadoop.hbase.regionserver.StoreFile;
43  import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
44  import org.apache.hadoop.hbase.regionserver.wal.HLog;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
47  import org.apache.hadoop.hdfs.DFSClient;
48  import org.apache.hadoop.hdfs.server.datanode.DataNode;
49  import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
50  import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
51  import org.apache.log4j.Level;
52  import org.junit.Ignore;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  
56  /**
57   * Test for the case where a regionserver going down has enough cycles to do damage to regions
58   * that have actually been assigned elsehwere.
59   *
60   * <p>If we happen to assign a region before it fully done with in its old location -- i.e. it is on two servers at the
61   * same time -- all can work fine until the case where the region on the dying server decides to compact or otherwise
62   * change the region file set.  The region in its new location will then get a surprise when it tries to do something
63   * w/ a file removed by the region in its old location on dying server.
64   *
65   * <p>Making a test for this case is a little tough in that even if a file is deleted up on the namenode,
66   * if the file was opened before the delete, it will continue to let reads happen until something changes the
67   * state of cached blocks in the dfsclient that was already open (a block from the deleted file is cleaned
68   * from the datanode by NN).
69   *
70   * <p>What we will do below is do an explicit check for existence on the files listed in the region that
71   * has had some files removed because of a compaction.  This sort of hurry's along and makes certain what is a chance
72   * occurance.
73   */
74  @Category(MediumTests.class)
75  public class TestIOFencing {
76    static final Log LOG = LogFactory.getLog(TestIOFencing.class);
77    static {
78      ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
79      ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
80      ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
81      ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem")).getLogger().setLevel(Level.ALL);
82      ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
83      ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL);
84    }
85  
86    public abstract static class CompactionBlockerRegion extends HRegion {
87      volatile int compactCount = 0;
88      volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
89      volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
90  
91      @SuppressWarnings("deprecation")
92      public CompactionBlockerRegion(Path tableDir, HLog log,
93          FileSystem fs, Configuration confParam, HRegionInfo info,
94          HTableDescriptor htd, RegionServerServices rsServices) {
95        super(tableDir, log, fs, confParam, info, htd, rsServices);
96      }
97  
98      public void stopCompactions() {
99        compactionsBlocked = new CountDownLatch(1);
100       compactionsWaiting = new CountDownLatch(1);
101     }
102 
103     public void allowCompactions() {
104       LOG.debug("allowing compactions");
105       compactionsBlocked.countDown();
106     }
107     public void waitForCompactionToBlock() throws IOException {
108       try {
109         LOG.debug("waiting for compaction to block");
110         compactionsWaiting.await();
111         LOG.debug("compaction block reached");
112       } catch (InterruptedException ex) {
113         throw new IOException(ex);
114       }
115     }
116     @Override
117     public boolean compact(CompactionContext compaction, Store store) throws IOException {
118       try {
119         return super.compact(compaction, store);
120       } finally {
121         compactCount++;
122       }
123     }
124     public int countStoreFiles() {
125       int count = 0;
126       for (Store store : stores.values()) {
127         count += store.getStorefilesCount();
128       }
129       return count;
130     }
131   }
132 
133   /**
134    * An override of HRegion that allows us park compactions in a holding pattern and
135    * then when appropriate for the test, allow them proceed again.
136    */
137   public static class BlockCompactionsInPrepRegion extends CompactionBlockerRegion {
138 
139     public BlockCompactionsInPrepRegion(Path tableDir, HLog log,
140         FileSystem fs, Configuration confParam, HRegionInfo info,
141         HTableDescriptor htd, RegionServerServices rsServices) {
142       super(tableDir, log, fs, confParam, info, htd, rsServices);
143     }
144     @Override
145     protected void doRegionCompactionPrep() throws IOException {
146       compactionsWaiting.countDown();
147       try {
148         compactionsBlocked.await();
149       } catch (InterruptedException ex) {
150         throw new IOException();
151       }
152       super.doRegionCompactionPrep();
153     }
154   }
155 
156   /**
157    * An override of HRegion that allows us park compactions in a holding pattern and
158    * then when appropriate for the test, allow them proceed again. This allows the compaction
159    * entry to go the WAL before blocking, but blocks afterwards
160    */
161   public static class BlockCompactionsInCompletionRegion extends CompactionBlockerRegion {
162     public BlockCompactionsInCompletionRegion(Path tableDir, HLog log,
163         FileSystem fs, Configuration confParam, HRegionInfo info,
164         HTableDescriptor htd, RegionServerServices rsServices) {
165       super(tableDir, log, fs, confParam, info, htd, rsServices);
166     }
167     @Override
168     protected HStore instantiateHStore(final HColumnDescriptor family) throws IOException {
169       return new BlockCompactionsInCompletionHStore(this, family, this.conf);
170     }
171   }
172 
173   public static class BlockCompactionsInCompletionHStore extends HStore {
174     CompactionBlockerRegion r;
175     protected BlockCompactionsInCompletionHStore(HRegion region, HColumnDescriptor family,
176         Configuration confParam) throws IOException {
177       super(region, family, confParam);
178       r = (CompactionBlockerRegion) region;
179     }
180 
181     @Override
182     protected void completeCompaction(Collection<StoreFile> compactedFiles) throws IOException {
183       try {
184         r.compactionsWaiting.countDown();
185         r.compactionsBlocked.await();
186       } catch (InterruptedException ex) {
187         throw new IOException(ex);
188       }
189       super.completeCompaction(compactedFiles);
190     }
191   }
192 
193   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
194   private final static TableName TABLE_NAME =
195       TableName.valueOf("tabletest");
196   private final static byte[] FAMILY = Bytes.toBytes("family");
197   private static final int FIRST_BATCH_COUNT = 4000;
198   private static final int SECOND_BATCH_COUNT = FIRST_BATCH_COUNT;
199 
200   /**
201    * Test that puts up a regionserver, starts a compaction on a loaded region but holds the
202    * compaction until after we have killed the server and the region has come up on
203    * a new regionserver altogether.  This fakes the double assignment case where region in one
204    * location changes the files out from underneath a region being served elsewhere.
205    */
206   @Ignore("See HBASE-10298")
207   @Test
208   public void testFencingAroundCompaction() throws Exception {
209     doTest(BlockCompactionsInPrepRegion.class);
210   }
211 
212   /**
213    * Test that puts up a regionserver, starts a compaction on a loaded region but holds the
214    * compaction completion until after we have killed the server and the region has come up on
215    * a new regionserver altogether.  This fakes the double assignment case where region in one
216    * location changes the files out from underneath a region being served elsewhere.
217    */
218   @Ignore("See HBASE-10298")
219   @Test
220   public void testFencingAroundCompactionAfterWALSync() throws Exception {
221     doTest(BlockCompactionsInCompletionRegion.class);
222   }
223 
224   public void doTest(Class<?> regionClass) throws Exception {
225     Configuration c = TEST_UTIL.getConfiguration();
226     c.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
227     // Insert our custom region
228     c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class);
229     c.setBoolean("dfs.support.append", true);
230     // Encourage plenty of flushes
231     c.setLong("hbase.hregion.memstore.flush.size", 200000);
232     c.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
233     // Only run compaction when we tell it to
234     c.setInt("hbase.hstore.compactionThreshold", 1000);
235     c.setLong("hbase.hstore.blockingStoreFiles", 1000);
236     // Compact quickly after we tell it to!
237     c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000);
238     LOG.info("Starting mini cluster");
239     TEST_UTIL.startMiniCluster(1);
240     CompactionBlockerRegion compactingRegion = null;
241     HBaseAdmin admin = null;
242     try {
243       LOG.info("Creating admin");
244       admin = new HBaseAdmin(c);
245       LOG.info("Creating table");
246       TEST_UTIL.createTable(TABLE_NAME, FAMILY);
247       HTable table = new HTable(c, TABLE_NAME);
248       LOG.info("Loading test table");
249       // Find the region
250       List<HRegion> testRegions = TEST_UTIL.getMiniHBaseCluster().findRegionsForTable(TABLE_NAME);
251       assertEquals(1, testRegions.size());
252       compactingRegion = (CompactionBlockerRegion)testRegions.get(0);
253       LOG.info("Blocking compactions");
254       compactingRegion.stopCompactions();
255       long lastFlushTime = compactingRegion.getLastFlushTime();
256       // Load some rows
257       TEST_UTIL.loadNumericRows(table, FAMILY, 0, FIRST_BATCH_COUNT);
258 
259       // Wait till flush has happened, otherwise there won't be multiple store files
260       long startWaitTime = System.currentTimeMillis();
261       while (compactingRegion.getLastFlushTime() <= lastFlushTime ||
262           compactingRegion.countStoreFiles() <= 1) {
263         LOG.info("Waiting for the region to flush " + compactingRegion.getRegionNameAsString());
264         Thread.sleep(1000);
265         assertTrue("Timed out waiting for the region to flush",
266           System.currentTimeMillis() - startWaitTime < 30000);
267       }
268       assertTrue(compactingRegion.countStoreFiles() > 1);
269       final byte REGION_NAME[] = compactingRegion.getRegionName();
270       LOG.info("Asking for compaction");
271       admin.majorCompact(TABLE_NAME.getName());
272       LOG.info("Waiting for compaction to be about to start");
273       compactingRegion.waitForCompactionToBlock();
274       LOG.info("Starting a new server");
275       RegionServerThread newServerThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer();
276       HRegionServer newServer = newServerThread.getRegionServer();
277       LOG.info("Killing region server ZK lease");
278       TEST_UTIL.expireRegionServerSession(0);
279       CompactionBlockerRegion newRegion = null;
280       startWaitTime = System.currentTimeMillis();
281       while (newRegion == null) {
282         LOG.info("Waiting for the new server to pick up the region " + Bytes.toString(REGION_NAME));
283         Thread.sleep(1000);
284         newRegion = (CompactionBlockerRegion)newServer.getOnlineRegion(REGION_NAME);
285         assertTrue("Timed out waiting for new server to open region",
286           System.currentTimeMillis() - startWaitTime < 300000);
287       }
288       LOG.info("Allowing compaction to proceed");
289       compactingRegion.allowCompactions();
290       while (compactingRegion.compactCount == 0) {
291         Thread.sleep(1000);
292       }
293       // The server we killed stays up until the compaction that was started before it was killed completes.  In logs
294       // you should see the old regionserver now going down.
295       LOG.info("Compaction finished");
296       // After compaction of old region finishes on the server that was going down, make sure that
297       // all the files we expect are still working when region is up in new location.
298       FileSystem fs = newRegion.getFilesystem();
299       for (String f: newRegion.getStoreFileList(new byte [][] {FAMILY})) {
300         assertTrue("After compaction, does not exist: " + f, fs.exists(new Path(f)));
301       }
302       // If we survive the split keep going...
303       // Now we make sure that the region isn't totally confused.  Load up more rows.
304       TEST_UTIL.loadNumericRows(table, FAMILY, FIRST_BATCH_COUNT, FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
305       admin.majorCompact(TABLE_NAME.getName());
306       startWaitTime = System.currentTimeMillis();
307       while (newRegion.compactCount == 0) {
308         Thread.sleep(1000);
309         assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000);
310       }
311       assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table));
312     } finally {
313       if (compactingRegion != null) {
314         compactingRegion.allowCompactions();
315       }
316       admin.close();
317       TEST_UTIL.shutdownMiniCluster();
318     }
319   }
320 }