1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.catalog;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.List;
29  import java.util.concurrent.atomic.AtomicBoolean;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.hbase.*;
35  import org.apache.hadoop.hbase.client.HBaseAdmin;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.util.Bytes;
38  import org.apache.hadoop.hbase.util.Pair;
39  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
40  import org.junit.AfterClass;
41  import org.junit.Before;
42  import org.junit.BeforeClass;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * Test {@link MetaReader}, {@link MetaEditor}, and {@link RootLocationEditor}.
48   */
49  @Category(MediumTests.class)
50  public class TestMetaReaderEditor {
51    private static final Log LOG = LogFactory.getLog(TestMetaReaderEditor.class);
52    private static final  HBaseTestingUtility UTIL = new HBaseTestingUtility();
53    private static ZooKeeperWatcher zkw;
54    private static CatalogTracker ct;
55    private final static Abortable ABORTABLE = new Abortable() {
56      private final AtomicBoolean abort = new AtomicBoolean(false);
57  
58      @Override
59      public void abort(String why, Throwable e) {
60        LOG.info(why, e);
61        abort.set(true);
62      }
63      
64      @Override
65      public boolean isAborted() {
66        return abort.get();
67      }
68      
69    };
70  
71    @BeforeClass public static void beforeClass() throws Exception {
72      UTIL.startMiniCluster(3);
73  
74      Configuration c = new Configuration(UTIL.getConfiguration());
75      // Tests to 4 retries every 5 seconds. Make it try every 1 second so more
76      // responsive.  1 second is default as is ten retries.
77      c.setLong("hbase.client.pause", 1000);
78      c.setInt("hbase.client.retries.number", 10);
79      zkw = new ZooKeeperWatcher(c, "TestMetaReaderEditor", ABORTABLE);
80      ct = new CatalogTracker(zkw, c, ABORTABLE);
81      ct.start();
82    }
83  
84    @AfterClass public static void afterClass() throws Exception {
85      ABORTABLE.abort("test ending", null);
86      ct.stop();
87      UTIL.shutdownMiniCluster();
88    }
89  
90    /**
91     * Does {@link MetaReader#getRegion(CatalogTracker, byte[])} and a write
92     * against .META. while its hosted server is restarted to prove our retrying
93     * works.
94     * @throws IOException
95     * @throws InterruptedException
96     */
97    @Test public void testRetrying()
98    throws IOException, InterruptedException {
99      final String name = "testRetrying";
100     LOG.info("Started " + name);
101     final byte [] nameBytes = Bytes.toBytes(name);
102     HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
103     int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
104     // Test it works getting a region from just made user table.
105     final List<HRegionInfo> regions =
106       testGettingTableRegions(this.ct, nameBytes, regionCount);
107     MetaTask reader = new MetaTask(this.ct, "reader") {
108       @Override
109       void metaTask() throws Throwable {
110         testGetRegion(this.ct, regions.get(0));
111         LOG.info("Read " + regions.get(0).getEncodedName());
112       }
113     };
114     MetaTask writer = new MetaTask(this.ct, "writer") {
115       @Override
116       void metaTask() throws Throwable {
117         MetaEditor.addRegionToMeta(this.ct, regions.get(0));
118         LOG.info("Wrote " + regions.get(0).getEncodedName());
119       }
120     };
121     reader.start();
122     writer.start();
123 
124     // We're gonna check how it takes. If it takes too long, we will consider
125     //  it as a fail. We can't put that in the @Test tag as we want to close
126     //  the threads nicely
127     final long timeOut = 180000;
128     long startTime = System.currentTimeMillis();
129 
130     try {
131       // Make sure reader and writer are working.
132       assertTrue(reader.isProgressing());
133       assertTrue(writer.isProgressing());
134 
135       // Kill server hosting meta -- twice  . See if our reader/writer ride over the
136       // meta moves.  They'll need to retry.
137       for (int i = 0; i < 2; i++) {
138         LOG.info("Restart=" + i);
139         UTIL.ensureSomeRegionServersAvailable(2);
140         int index = -1;
141         do {
142           index = UTIL.getMiniHBaseCluster().getServerWithMeta();
143         }while (index == -1 &&
144           startTime + timeOut < System.currentTimeMillis());
145 
146         if (index != -1){
147           UTIL.getMiniHBaseCluster().abortRegionServer(index);
148           UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
149         }
150       }
151 
152       assertTrue("reader: "+reader.toString(), reader.isProgressing());
153       assertTrue("writer: "+writer.toString(), writer.isProgressing());
154     } catch (IOException e) {
155       throw e;
156     } finally {
157       reader.stop = true;
158       writer.stop = true;
159       reader.join();
160       writer.join();
161       t.close();
162     }
163     long exeTime = System.currentTimeMillis() - startTime;
164     assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut);
165   }
166 
167   /**
168    * Thread that runs a MetaReader/MetaEditor task until asked stop.
169    */
170   abstract static class MetaTask extends Thread {
171     boolean stop = false;
172     int count = 0;
173     Throwable t = null;
174     final CatalogTracker ct;
175 
176     MetaTask(final CatalogTracker ct, final String name) {
177       super(name);
178       this.ct = ct;
179     }
180 
181     @Override
182     public void run() {
183       try {
184         while(!this.stop) {
185           LOG.info("Before " + this.getName()+ ", count=" + this.count);
186           metaTask();
187           this.count += 1;
188           LOG.info("After " + this.getName() + ", count=" + this.count);
189           Thread.sleep(100);
190         }
191       } catch (Throwable t) {
192         LOG.info(this.getName() + " failed", t);
193         this.t = t;
194       }
195     }
196 
197     boolean isProgressing() throws InterruptedException {
198       int currentCount = this.count;
199       while(currentCount == this.count) {
200         if (!isAlive()) return false;
201         if (this.t != null) return false;
202         Thread.sleep(10);
203       }
204       return true;
205     }
206 
207     @Override
208     public String toString() {
209       return "count=" + this.count + ", t=" +
210         (this.t == null? "null": this.t.toString());
211     }
212 
213     abstract void metaTask() throws Throwable;
214   }
215 
216   @Test public void testGetRegionsCatalogTables()
217   throws IOException, InterruptedException {
218     List<HRegionInfo> regions =
219       MetaReader.getTableRegions(ct, HConstants.META_TABLE_NAME);
220     assertTrue(regions.size() >= 1);
221     assertTrue(MetaReader.getTableRegionsAndLocations(ct,
222       Bytes.toString(HConstants.META_TABLE_NAME)).size() >= 1);
223     assertTrue(MetaReader.getTableRegionsAndLocations(ct,
224       Bytes.toString(HConstants.ROOT_TABLE_NAME)).size() == 1);
225   }
226 
227   @Test public void testTableExists() throws IOException {
228     final String name = "testTableExists";
229     final byte [] nameBytes = Bytes.toBytes(name);
230     assertFalse(MetaReader.tableExists(ct, name));
231     UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
232     assertTrue(MetaReader.tableExists(ct, name));
233     HBaseAdmin admin = UTIL.getHBaseAdmin();
234     admin.disableTable(name);
235     admin.deleteTable(name);
236     assertFalse(MetaReader.tableExists(ct, name));
237     assertTrue(MetaReader.tableExists(ct,
238       Bytes.toString(HConstants.META_TABLE_NAME)));
239     assertTrue(MetaReader.tableExists(ct,
240       Bytes.toString(HConstants.ROOT_TABLE_NAME)));
241   }
242 
243   @Test public void testGetRegion() throws IOException, InterruptedException {
244     final String name = "testGetRegion";
245     LOG.info("Started " + name);
246     // Test get on non-existent region.
247     Pair<HRegionInfo, ServerName> pair =
248       MetaReader.getRegion(ct, Bytes.toBytes("nonexistent-region"));
249     assertNull(pair);
250     // Test it works getting a region from meta/root.
251     pair =
252       MetaReader.getRegion(ct, HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
253     assertEquals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(),
254       pair.getFirst().getEncodedName());
255     LOG.info("Finished " + name);
256   }
257 
258   // Test for the optimization made in HBASE-3650
259   @Test public void testScanMetaForTable()
260   throws IOException, InterruptedException {
261     final String name = "testScanMetaForTable";
262     LOG.info("Started " + name);
263 
264     /** Create 2 tables
265      - testScanMetaForTable
266      - testScanMetaForTablf
267     **/
268 
269     UTIL.createTable(Bytes.toBytes(name), HConstants.CATALOG_FAMILY);
270     // name that is +1 greater than the first one (e+1=f)
271     byte[] greaterName = Bytes.toBytes("testScanMetaForTablf");
272     UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY);
273 
274     // Now make sure we only get the regions from 1 of the tables at a time
275 
276     assertEquals(1, MetaReader.getTableRegions(ct, Bytes.toBytes(name)).size());
277     assertEquals(1, MetaReader.getTableRegions(ct, greaterName).size());
278   }
279 
280   private static List<HRegionInfo> testGettingTableRegions(final CatalogTracker ct,
281       final byte [] nameBytes, final int regionCount)
282   throws IOException, InterruptedException {
283     List<HRegionInfo> regions = MetaReader.getTableRegions(ct, nameBytes);
284     assertEquals(regionCount, regions.size());
285     Pair<HRegionInfo, ServerName> pair =
286       MetaReader.getRegion(ct, regions.get(0).getRegionName());
287     assertEquals(regions.get(0).getEncodedName(),
288       pair.getFirst().getEncodedName());
289     return regions;
290   }
291 
292   private static void testGetRegion(final CatalogTracker ct,
293       final HRegionInfo region)
294   throws IOException, InterruptedException {
295     Pair<HRegionInfo, ServerName> pair =
296       MetaReader.getRegion(ct, region.getRegionName());
297     assertEquals(region.getEncodedName(),
298       pair.getFirst().getEncodedName());
299   }
300 
301   @org.junit.Rule
302   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
303     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
304 }
305