1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   * http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.coprocessor;
22  
23  import java.io.IOException;
24  import java.io.InterruptedIOException;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.*;
28  import org.apache.hadoop.hbase.client.HBaseAdmin;
29  import org.apache.hadoop.hbase.master.HMaster;
30  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
33  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
34  import org.junit.AfterClass;
35  import org.junit.BeforeClass;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  
39  import static org.junit.Assert.*;
40  
41  /**
42   * Tests unhandled exceptions thrown by coprocessors running on master.
43   * Expected result is that the master will remove the buggy coprocessor from
44   * its set of coprocessors and throw a org.apache.hadoop.hbase.DoNotRetryIOException
45   * back to the client.
46   * (HBASE-4014).
47   */
48  @Category(MediumTests.class)
49  public class TestMasterCoprocessorExceptionWithRemove {
50  
51    public static class MasterTracker extends ZooKeeperNodeTracker {
52      public boolean masterZKNodeWasDeleted = false;
53  
54      public MasterTracker(ZooKeeperWatcher zkw, String masterNode, Abortable abortable) {
55        super(zkw, masterNode, abortable);
56      }
57  
58      @Override
59      public synchronized void nodeDeleted(String path) {
60        if (path.equals("/hbase/master")) {
61          masterZKNodeWasDeleted = true;
62        }
63      }
64    }
65  
66    public static class BuggyMasterObserver extends BaseMasterObserver {
67      private boolean preCreateTableCalled;
68      private boolean postCreateTableCalled;
69      private boolean startCalled;
70      private boolean postStartMasterCalled;
71  
72      @Override
73      public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
74          HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
75        // Cause a NullPointerException and don't catch it: this should cause the
76        // master to throw an o.apache.hadoop.hbase.DoNotRetryIOException to the
77        // client.
78        Integer i;
79        i = null;
80        i = i++;
81      }
82  
83      public boolean wasCreateTableCalled() {
84        return preCreateTableCalled && postCreateTableCalled;
85      }
86  
87      @Override
88      public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
89          throws IOException {
90        postStartMasterCalled = true;
91      }
92  
93      public boolean wasStartMasterCalled() {
94        return postStartMasterCalled;
95      }
96  
97      @Override
98      public void start(CoprocessorEnvironment env) throws IOException {
99        startCalled = true;
100     }
101 
102     public boolean wasStarted() {
103       return startCalled;
104     }
105   }
106 
107   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
108 
109   private static byte[] TEST_TABLE1 = Bytes.toBytes("observed_table1");
110   private static byte[] TEST_FAMILY1 = Bytes.toBytes("fam1");
111 
112   private static byte[] TEST_TABLE2 = Bytes.toBytes("table2");
113   private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
114 
115   @BeforeClass
116   public static void setupBeforeClass() throws Exception {
117     Configuration conf = UTIL.getConfiguration();
118     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
119         BuggyMasterObserver.class.getName());
120     UTIL.startMiniCluster();
121   }
122 
123   @AfterClass
124   public static void teardownAfterClass() throws Exception {
125     UTIL.shutdownMiniCluster();
126   }
127 
128   @Test(timeout=30000)
129   public void testExceptionFromCoprocessorWhenCreatingTable()
130       throws IOException {
131     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
132 
133     HMaster master = cluster.getMaster();
134     MasterCoprocessorHost host = master.getCoprocessorHost();
135     BuggyMasterObserver cp = (BuggyMasterObserver)host.findCoprocessor(
136         BuggyMasterObserver.class.getName());
137     assertFalse("No table created yet", cp.wasCreateTableCalled());
138 
139     // Set a watch on the zookeeper /hbase/master node. If the master dies,
140     // the node will be deleted.
141     // Master should *NOT* die:
142     // we are testing that the default setting of hbase.coprocessor.abortonerror
143     // =false
144     // is respected.
145     ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(),
146       "unittest", new Abortable() {
147       @Override
148       public void abort(String why, Throwable e) {
149         throw new RuntimeException("Fatal ZK error: " + why, e);
150       }
151       @Override
152       public boolean isAborted() {
153         return false;
154       }
155     });
156 
157     MasterTracker masterTracker = new MasterTracker(zkw,"/hbase/master",
158         new Abortable() {
159           @Override
160           public void abort(String why, Throwable e) {
161             throw new RuntimeException("Fatal Zookeeper tracker error, why=", e);
162           }
163           @Override
164           public boolean isAborted() {
165             return false;
166           }
167         });
168 
169     masterTracker.start();
170     zkw.registerListener(masterTracker);
171 
172     // Test (part of the) output that should have be printed by master when it aborts:
173     // (namely the part that shows the set of loaded coprocessors).
174     // In this test, there is only a single coprocessor (BuggyMasterObserver).
175     String coprocessorName =
176         BuggyMasterObserver.class.getName();
177     assertTrue(master.getLoadedCoprocessors().equals("[" + coprocessorName + "]"));
178 
179     HTableDescriptor htd1 = new HTableDescriptor(TEST_TABLE1);
180     htd1.addFamily(new HColumnDescriptor(TEST_FAMILY1));
181 
182     boolean threwDNRE = false;
183     try {
184       HBaseAdmin admin = UTIL.getHBaseAdmin();
185       admin.createTable(htd1);
186     } catch (IOException e) {
187       if (e.getClass().getName().equals("org.apache.hadoop.hbase.DoNotRetryIOException")) {
188         threwDNRE = true;
189       }
190     } finally {
191       assertTrue(threwDNRE);
192     }
193 
194     // wait for a few seconds to make sure that the Master hasn't aborted.
195     try {
196       Thread.sleep(3000);
197     } catch (InterruptedException e) {
198       fail("InterruptedException while sleeping.");
199     }
200 
201     assertFalse("Master survived coprocessor NPE, as expected.",
202         masterTracker.masterZKNodeWasDeleted);
203 
204     String loadedCoprocessors = master.getLoadedCoprocessors();
205     assertTrue(loadedCoprocessors.equals("[" + coprocessorName + "]"));
206 
207     // Verify that BuggyMasterObserver has been removed due to its misbehavior
208     // by creating another table: should not have a problem this time.
209     HTableDescriptor htd2 = new HTableDescriptor(TEST_TABLE2);
210     htd2.addFamily(new HColumnDescriptor(TEST_FAMILY2));
211     HBaseAdmin admin = UTIL.getHBaseAdmin();
212     try {
213       admin.createTable(htd2);
214     } catch (IOException e) {
215       fail("Failed to create table after buggy coprocessor removal: " + e);
216     }
217   }
218 
219   @org.junit.Rule
220   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
221     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
222 }
223