View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.io.IOException;
28  import java.util.Collection;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.NavigableMap;
32  import java.util.concurrent.CountDownLatch;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.*;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.master.AssignmentManager;
41  import org.apache.hadoop.hbase.master.HMaster;
42  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
43  import org.apache.hadoop.hbase.master.RegionPlan;
44  import org.apache.hadoop.hbase.master.RegionState;
45  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46  import org.apache.hadoop.hbase.protobuf.generated.MasterMonitorProtos.GetTableDescriptorsRequest;
47  import org.apache.hadoop.hbase.protobuf.RequestConverter;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.regionserver.HRegionServer;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.Threads;
52  import org.junit.AfterClass;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  /**
58   * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver}
59   * interface hooks at all appropriate times during normal HMaster operations.
60   */
61  @Category(MediumTests.class)
62  public class TestMasterObserver {
63    private static final Log LOG = LogFactory.getLog(TestMasterObserver.class);
64  
65    public static CountDownLatch tableCreationLatch = new CountDownLatch(1);
66  
67    public static class CPMasterObserver implements MasterObserver {
68  
69      private boolean bypass = false;
70      private boolean preCreateTableCalled;
71      private boolean postCreateTableCalled;
72      private boolean preDeleteTableCalled;
73      private boolean postDeleteTableCalled;
74      private boolean preModifyTableCalled;
75      private boolean postModifyTableCalled;
76      private boolean preCreateNamespaceCalled;
77      private boolean postCreateNamespaceCalled;
78      private boolean preDeleteNamespaceCalled;
79      private boolean postDeleteNamespaceCalled;
80      private boolean preModifyNamespaceCalled;
81      private boolean postModifyNamespaceCalled;
82      private boolean preAddColumnCalled;
83      private boolean postAddColumnCalled;
84      private boolean preModifyColumnCalled;
85      private boolean postModifyColumnCalled;
86      private boolean preDeleteColumnCalled;
87      private boolean postDeleteColumnCalled;
88      private boolean preEnableTableCalled;
89      private boolean postEnableTableCalled;
90      private boolean preDisableTableCalled;
91      private boolean postDisableTableCalled;
92      private boolean preMoveCalled;
93      private boolean postMoveCalled;
94      private boolean preAssignCalled;
95      private boolean postAssignCalled;
96      private boolean preUnassignCalled;
97      private boolean postUnassignCalled;
98      private boolean preRegionOfflineCalled;
99      private boolean postRegionOfflineCalled;
100     private boolean preBalanceCalled;
101     private boolean postBalanceCalled;
102     private boolean preBalanceSwitchCalled;
103     private boolean postBalanceSwitchCalled;
104     private boolean preShutdownCalled;
105     private boolean preStopMasterCalled;
106     private boolean postStartMasterCalled;
107     private boolean startCalled;
108     private boolean stopCalled;
109     private boolean preSnapshotCalled;
110     private boolean postSnapshotCalled;
111     private boolean preCloneSnapshotCalled;
112     private boolean postCloneSnapshotCalled;
113     private boolean preRestoreSnapshotCalled;
114     private boolean postRestoreSnapshotCalled;
115     private boolean preDeleteSnapshotCalled;
116     private boolean postDeleteSnapshotCalled;
117     private boolean preCreateTableHandlerCalled;
118     private boolean postCreateTableHandlerCalled;
119     private boolean preDeleteTableHandlerCalled;
120     private boolean postDeleteTableHandlerCalled;
121     private boolean preAddColumnHandlerCalled;
122     private boolean postAddColumnHandlerCalled;
123     private boolean preModifyColumnHandlerCalled;
124     private boolean postModifyColumnHandlerCalled;
125     private boolean preDeleteColumnHandlerCalled;
126     private boolean postDeleteColumnHandlerCalled;
127     private boolean preEnableTableHandlerCalled;
128     private boolean postEnableTableHandlerCalled;
129     private boolean preDisableTableHandlerCalled;
130     private boolean postDisableTableHandlerCalled;
131     private boolean preModifyTableHandlerCalled;
132     private boolean postModifyTableHandlerCalled;
133     private boolean preGetTableDescriptorsCalled;
134     private boolean postGetTableDescriptorsCalled;
135 
136     public void enableBypass(boolean bypass) {
137       this.bypass = bypass;
138     }
139 
140     public void resetStates() {
141       preCreateTableCalled = false;
142       postCreateTableCalled = false;
143       preDeleteTableCalled = false;
144       postDeleteTableCalled = false;
145       preModifyTableCalled = false;
146       postModifyTableCalled = false;
147       preCreateNamespaceCalled = false;
148       postCreateNamespaceCalled = false;
149       preDeleteNamespaceCalled = false;
150       postDeleteNamespaceCalled = false;
151       preModifyNamespaceCalled = false;
152       postModifyNamespaceCalled = false;
153       preAddColumnCalled = false;
154       postAddColumnCalled = false;
155       preModifyColumnCalled = false;
156       postModifyColumnCalled = false;
157       preDeleteColumnCalled = false;
158       postDeleteColumnCalled = false;
159       preEnableTableCalled = false;
160       postEnableTableCalled = false;
161       preDisableTableCalled = false;
162       postDisableTableCalled = false;
163       preMoveCalled= false;
164       postMoveCalled = false;
165       preAssignCalled = false;
166       postAssignCalled = false;
167       preUnassignCalled = false;
168       postUnassignCalled = false;
169       preRegionOfflineCalled = false;
170       postRegionOfflineCalled = false;
171       preBalanceCalled = false;
172       postBalanceCalled = false;
173       preBalanceSwitchCalled = false;
174       postBalanceSwitchCalled = false;
175       preSnapshotCalled = false;
176       postSnapshotCalled = false;
177       preCloneSnapshotCalled = false;
178       postCloneSnapshotCalled = false;
179       preRestoreSnapshotCalled = false;
180       postRestoreSnapshotCalled = false;
181       preDeleteSnapshotCalled = false;
182       postDeleteSnapshotCalled = false;
183       preCreateTableHandlerCalled = false;
184       postCreateTableHandlerCalled = false;
185       preDeleteTableHandlerCalled = false;
186       postDeleteTableHandlerCalled = false;
187       preModifyTableHandlerCalled = false;
188       postModifyTableHandlerCalled = false;
189       preAddColumnHandlerCalled = false;
190       postAddColumnHandlerCalled = false;
191       preModifyColumnHandlerCalled = false;
192       postModifyColumnHandlerCalled = false;
193       preDeleteColumnHandlerCalled = false;
194       postDeleteColumnHandlerCalled = false;
195       preEnableTableHandlerCalled = false;
196       postEnableTableHandlerCalled = false;
197       preDisableTableHandlerCalled = false;
198       postDisableTableHandlerCalled = false;
199       preModifyTableHandlerCalled = false;
200       postModifyTableHandlerCalled = false;
201       preGetTableDescriptorsCalled = false;
202       postGetTableDescriptorsCalled = false;
203     }
204 
205     @Override
206     public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
207         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
208       if (bypass) {
209         env.bypass();
210       }
211       preCreateTableCalled = true;
212     }
213 
214     @Override
215     public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
216         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
217       postCreateTableCalled = true;
218     }
219 
220     public boolean wasCreateTableCalled() {
221       return preCreateTableCalled && postCreateTableCalled;
222     }
223 
224     public boolean preCreateTableCalledOnly() {
225       return preCreateTableCalled && !postCreateTableCalled;
226     }
227 
228     @Override
229     public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> env,
230         TableName tableName) throws IOException {
231       if (bypass) {
232         env.bypass();
233       }
234       preDeleteTableCalled = true;
235     }
236 
237     @Override
238     public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> env,
239         TableName tableName) throws IOException {
240       postDeleteTableCalled = true;
241     }
242 
243     public boolean wasDeleteTableCalled() {
244       return preDeleteTableCalled && postDeleteTableCalled;
245     }
246 
247     public boolean preDeleteTableCalledOnly() {
248       return preDeleteTableCalled && !postDeleteTableCalled;
249     }
250 
251     @Override
252     public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
253         TableName tableName, HTableDescriptor htd) throws IOException {
254       if (bypass) {
255         env.bypass();
256       }else{
257         env.shouldBypass();
258       }
259       preModifyTableCalled = true;
260     }
261 
262     @Override
263     public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
264         TableName tableName, HTableDescriptor htd) throws IOException {
265       postModifyTableCalled = true;
266     }
267 
268     public boolean wasModifyTableCalled() {
269       return preModifyTableCalled && postModifyTableCalled;
270     }
271 
272     public boolean preModifyTableCalledOnly() {
273       return preModifyTableCalled && !postModifyTableCalled;
274     }
275 
276     @Override
277     public void preCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
278         NamespaceDescriptor ns) throws IOException {
279       if (bypass) {
280         env.bypass();
281       }
282       preCreateNamespaceCalled = true;
283     }
284 
285     @Override
286     public void postCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
287         NamespaceDescriptor ns) throws IOException {
288       postCreateNamespaceCalled = true;
289     }
290 
291     public boolean wasCreateNamespaceCalled() {
292       return preCreateNamespaceCalled && postCreateNamespaceCalled;
293     }
294 
295     public boolean preCreateNamespaceCalledOnly() {
296       return preCreateNamespaceCalled && !postCreateNamespaceCalled;
297     }
298 
299     @Override
300     public void preDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
301         String name) throws IOException {
302       if (bypass) {
303         env.bypass();
304       }
305       preDeleteNamespaceCalled = true;
306     }
307 
308     @Override
309     public void postDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
310         String name) throws IOException {
311       postDeleteNamespaceCalled = true;
312     }
313 
314     public boolean wasDeleteNamespaceCalled() {
315       return preDeleteNamespaceCalled && postDeleteNamespaceCalled;
316     }
317 
318     public boolean preDeleteNamespaceCalledOnly() {
319       return preDeleteNamespaceCalled && !postDeleteNamespaceCalled;
320     }
321 
322     @Override
323     public void preModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
324         NamespaceDescriptor ns) throws IOException {
325       if (bypass) {
326         env.bypass();
327       }
328       preModifyNamespaceCalled = true;
329     }
330 
331     @Override
332     public void postModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> env,
333         NamespaceDescriptor ns) throws IOException {
334       postModifyNamespaceCalled = true;
335     }
336 
337     public boolean wasModifyNamespaceCalled() {
338       return preModifyNamespaceCalled && postModifyNamespaceCalled;
339     }
340 
341     public boolean preModifyNamespaceCalledOnly() {
342       return preModifyNamespaceCalled && !postModifyNamespaceCalled;
343     }
344 
345     @Override
346     public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> env,
347         TableName tableName, HColumnDescriptor column) throws IOException {
348       if (bypass) {
349         env.bypass();
350       }else{
351         env.shouldBypass();
352       }
353 
354       preAddColumnCalled = true;
355     }
356 
357     @Override
358     public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> env,
359         TableName tableName, HColumnDescriptor column) throws IOException {
360       postAddColumnCalled = true;
361     }
362 
363     public boolean wasAddColumnCalled() {
364       return preAddColumnCalled && postAddColumnCalled;
365     }
366 
367     public boolean preAddColumnCalledOnly() {
368       return preAddColumnCalled && !postAddColumnCalled;
369     }
370 
371     @Override
372     public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> env,
373         TableName tableName, HColumnDescriptor descriptor) throws IOException {
374       if (bypass) {
375         env.bypass();
376       }
377       preModifyColumnCalled = true;
378     }
379 
380     @Override
381     public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> env,
382         TableName tableName, HColumnDescriptor descriptor) throws IOException {
383       postModifyColumnCalled = true;
384     }
385 
386     public boolean wasModifyColumnCalled() {
387       return preModifyColumnCalled && postModifyColumnCalled;
388     }
389 
390     public boolean preModifyColumnCalledOnly() {
391       return preModifyColumnCalled && !postModifyColumnCalled;
392     }
393 
394     @Override
395     public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> env,
396         TableName tableName, byte[] c) throws IOException {
397       if (bypass) {
398         env.bypass();
399       }
400       preDeleteColumnCalled = true;
401     }
402 
403     @Override
404     public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> env,
405         TableName tableName, byte[] c) throws IOException {
406       postDeleteColumnCalled = true;
407     }
408 
409     public boolean wasDeleteColumnCalled() {
410       return preDeleteColumnCalled && postDeleteColumnCalled;
411     }
412 
413     public boolean preDeleteColumnCalledOnly() {
414       return preDeleteColumnCalled && !postDeleteColumnCalled;
415     }
416 
417     @Override
418     public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> env,
419         TableName tableName) throws IOException {
420       if (bypass) {
421         env.bypass();
422       }
423       preEnableTableCalled = true;
424     }
425 
426     @Override
427     public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> env,
428         TableName tableName) throws IOException {
429       postEnableTableCalled = true;
430     }
431 
432     public boolean wasEnableTableCalled() {
433       return preEnableTableCalled && postEnableTableCalled;
434     }
435 
436     public boolean preEnableTableCalledOnly() {
437       return preEnableTableCalled && !postEnableTableCalled;
438     }
439 
440     @Override
441     public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> env,
442         TableName tableName) throws IOException {
443       if (bypass) {
444         env.bypass();
445       }
446       preDisableTableCalled = true;
447     }
448 
449     @Override
450     public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> env,
451         TableName tableName) throws IOException {
452       postDisableTableCalled = true;
453     }
454 
455     public boolean wasDisableTableCalled() {
456       return preDisableTableCalled && postDisableTableCalled;
457     }
458 
459     public boolean preDisableTableCalledOnly() {
460       return preDisableTableCalled && !postDisableTableCalled;
461     }
462 
463     @Override
464     public void preMove(ObserverContext<MasterCoprocessorEnvironment> env,
465         HRegionInfo region, ServerName srcServer, ServerName destServer)
466     throws IOException {
467       if (bypass) {
468         env.bypass();
469       }
470       preMoveCalled = true;
471     }
472 
473     @Override
474     public void postMove(ObserverContext<MasterCoprocessorEnvironment> env, HRegionInfo region,
475         ServerName srcServer, ServerName destServer)
476     throws IOException {
477       postMoveCalled = true;
478     }
479 
480     public boolean wasMoveCalled() {
481       return preMoveCalled && postMoveCalled;
482     }
483 
484     public boolean preMoveCalledOnly() {
485       return preMoveCalled && !postMoveCalled;
486     }
487 
488     @Override
489     public void preAssign(ObserverContext<MasterCoprocessorEnvironment> env,
490         final HRegionInfo regionInfo) throws IOException {
491       if (bypass) {
492         env.bypass();
493       }
494       preAssignCalled = true;
495     }
496 
497     @Override
498     public void postAssign(ObserverContext<MasterCoprocessorEnvironment> env,
499         final HRegionInfo regionInfo) throws IOException {
500       postAssignCalled = true;
501     }
502 
503     public boolean wasAssignCalled() {
504       return preAssignCalled && postAssignCalled;
505     }
506 
507     public boolean preAssignCalledOnly() {
508       return preAssignCalled && !postAssignCalled;
509     }
510 
511     @Override
512     public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> env,
513         final HRegionInfo regionInfo, final boolean force) throws IOException {
514       if (bypass) {
515         env.bypass();
516       }
517       preUnassignCalled = true;
518     }
519 
520     @Override
521     public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> env,
522         final HRegionInfo regionInfo, final boolean force) throws IOException {
523       postUnassignCalled = true;
524     }
525 
526     public boolean wasUnassignCalled() {
527       return preUnassignCalled && postUnassignCalled;
528     }
529 
530     public boolean preUnassignCalledOnly() {
531       return preUnassignCalled && !postUnassignCalled;
532     }
533 
534     @Override
535     public void preRegionOffline(ObserverContext<MasterCoprocessorEnvironment> env,
536         final HRegionInfo regionInfo) throws IOException {
537       preRegionOfflineCalled = true;
538     }
539 
540     @Override
541     public void postRegionOffline(ObserverContext<MasterCoprocessorEnvironment> env,
542         final HRegionInfo regionInfo) throws IOException {
543       postRegionOfflineCalled = true;
544     }
545 
546     public boolean wasRegionOfflineCalled() {
547       return preRegionOfflineCalled && postRegionOfflineCalled;
548     }
549 
550     public boolean preRegionOfflineCalledOnly() {
551       return preRegionOfflineCalled && !postRegionOfflineCalled;
552     }
553 
554     @Override
555     public void preBalance(ObserverContext<MasterCoprocessorEnvironment> env)
556         throws IOException {
557       if (bypass) {
558         env.bypass();
559       }
560       preBalanceCalled = true;
561     }
562 
563     @Override
564     public void postBalance(ObserverContext<MasterCoprocessorEnvironment> env,
565         List<RegionPlan> plans) throws IOException {
566       postBalanceCalled = true;
567     }
568 
569     public boolean wasBalanceCalled() {
570       return preBalanceCalled && postBalanceCalled;
571     }
572 
573     public boolean preBalanceCalledOnly() {
574       return preBalanceCalled && !postBalanceCalled;
575     }
576 
577     @Override
578     public boolean preBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> env, boolean b)
579         throws IOException {
580       if (bypass) {
581         env.bypass();
582       }
583       preBalanceSwitchCalled = true;
584       return b;
585     }
586 
587     @Override
588     public void postBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> env,
589         boolean oldValue, boolean newValue) throws IOException {
590       postBalanceSwitchCalled = true;
591     }
592 
593     public boolean wasBalanceSwitchCalled() {
594       return preBalanceSwitchCalled && postBalanceSwitchCalled;
595     }
596 
597     public boolean preBalanceSwitchCalledOnly() {
598       return preBalanceSwitchCalled && !postBalanceSwitchCalled;
599     }
600 
601     @Override
602     public void preShutdown(ObserverContext<MasterCoprocessorEnvironment> env)
603         throws IOException {
604       preShutdownCalled = true;
605     }
606 
607     @Override
608     public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> env)
609         throws IOException {
610       preStopMasterCalled = true;
611     }
612 
613     @Override
614     public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx)
615         throws IOException {
616       postStartMasterCalled = true;
617     }
618 
619     public boolean wasStartMasterCalled() {
620       return postStartMasterCalled;
621     }
622 
623     @Override
624     public void start(CoprocessorEnvironment env) throws IOException {
625       startCalled = true;
626     }
627 
628     @Override
629     public void stop(CoprocessorEnvironment env) throws IOException {
630       stopCalled = true;
631     }
632 
633     public boolean wasStarted() { return startCalled; }
634 
635     public boolean wasStopped() { return stopCalled; }
636 
637     @Override
638     public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
639         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
640         throws IOException {
641       preSnapshotCalled = true;
642     }
643 
644     @Override
645     public void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
646         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
647         throws IOException {
648       postSnapshotCalled = true;
649     }
650 
651     public boolean wasSnapshotCalled() {
652       return preSnapshotCalled && postSnapshotCalled;
653     }
654 
655     @Override
656     public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
657         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
658         throws IOException {
659       preCloneSnapshotCalled = true;
660     }
661 
662     @Override
663     public void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
664         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
665         throws IOException {
666       postCloneSnapshotCalled = true;
667     }
668 
669     public boolean wasCloneSnapshotCalled() {
670       return preCloneSnapshotCalled && postCloneSnapshotCalled;
671     }
672 
673     @Override
674     public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
675         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
676         throws IOException {
677       preRestoreSnapshotCalled = true;
678     }
679 
680     @Override
681     public void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
682         final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
683         throws IOException {
684       postRestoreSnapshotCalled = true;
685     }
686 
687     public boolean wasRestoreSnapshotCalled() {
688       return preRestoreSnapshotCalled && postRestoreSnapshotCalled;
689     }
690 
691     @Override
692     public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
693         final SnapshotDescription snapshot) throws IOException {
694       preDeleteSnapshotCalled = true;
695     }
696 
697     @Override
698     public void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
699         final SnapshotDescription snapshot) throws IOException {
700       postDeleteSnapshotCalled = true;
701     }
702 
703     public boolean wasDeleteSnapshotCalled() {
704       return preDeleteSnapshotCalled && postDeleteSnapshotCalled;
705     }
706 
707     @Override
708     public void preCreateTableHandler(
709         ObserverContext<MasterCoprocessorEnvironment> env,
710         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
711       if (bypass) {
712         env.bypass();
713       }
714       preCreateTableHandlerCalled = true;
715     }
716 
717     @Override
718     public void postCreateTableHandler(
719         ObserverContext<MasterCoprocessorEnvironment> ctx,
720         HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
721       postCreateTableHandlerCalled = true;
722       tableCreationLatch.countDown();
723     }
724 
725     public boolean wasPreCreateTableHandlerCalled(){
726       return preCreateTableHandlerCalled;
727     }
728     public boolean wasCreateTableHandlerCalled() {
729       return preCreateTableHandlerCalled && postCreateTableHandlerCalled;
730     }
731 
732     public boolean wasCreateTableHandlerCalledOnly() {
733       return preCreateTableHandlerCalled && !postCreateTableHandlerCalled;
734     }
735 
736     @Override
737     public void preDeleteTableHandler(
738         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
739         throws IOException {
740       if (bypass) {
741         env.bypass();
742       }
743       preDeleteTableHandlerCalled = true;
744     }
745 
746     @Override
747     public void postDeleteTableHandler(
748         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
749         throws IOException {
750       postDeleteTableHandlerCalled = true;
751     }
752 
753     public boolean wasDeleteTableHandlerCalled() {
754       return preDeleteTableHandlerCalled && postDeleteTableHandlerCalled;
755     }
756 
757     public boolean wasDeleteTableHandlerCalledOnly() {
758       return preDeleteTableHandlerCalled && !postDeleteTableHandlerCalled;
759     }
760     @Override
761     public void preModifyTableHandler(
762         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
763         HTableDescriptor htd) throws IOException {
764       if (bypass) {
765         env.bypass();
766       }
767       preModifyTableHandlerCalled = true;
768     }
769 
770     @Override
771     public void postModifyTableHandler(
772         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
773         HTableDescriptor htd) throws IOException {
774       postModifyTableHandlerCalled = true;
775     }
776 
777     public boolean wasModifyTableHandlerCalled() {
778       return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled;
779     }
780 
781     public boolean wasModifyTableHandlerCalledOnly() {
782       return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled;
783     }
784 
785     @Override
786     public void preAddColumnHandler(
787         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
788         HColumnDescriptor column) throws IOException {
789       if (bypass) {
790         env.bypass();
791       }
792       preAddColumnHandlerCalled = true;
793     }
794 
795     @Override
796     public void postAddColumnHandler(
797         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
798         HColumnDescriptor column) throws IOException {
799       postAddColumnHandlerCalled = true;
800     }
801     public boolean wasAddColumnHandlerCalled() {
802       return preAddColumnHandlerCalled && postAddColumnHandlerCalled;
803     }
804 
805     public boolean preAddColumnHandlerCalledOnly() {
806       return preAddColumnHandlerCalled && !postAddColumnHandlerCalled;
807     }
808 
809     @Override
810     public void preModifyColumnHandler(
811         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
812         HColumnDescriptor descriptor) throws IOException {
813       if (bypass) {
814         env.bypass();
815       }
816       preModifyColumnHandlerCalled = true;
817     }
818 
819     @Override
820     public void postModifyColumnHandler(
821         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
822         HColumnDescriptor descriptor) throws IOException {
823       postModifyColumnHandlerCalled = true;
824     }
825 
826     public boolean wasModifyColumnHandlerCalled() {
827       return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled;
828     }
829 
830     public boolean preModifyColumnHandlerCalledOnly() {
831       return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled;
832     }
833     @Override
834     public void preDeleteColumnHandler(
835         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
836         byte[] c) throws IOException {
837       if (bypass) {
838         env.bypass();
839       }
840       preDeleteColumnHandlerCalled = true;
841     }
842 
843     @Override
844     public void postDeleteColumnHandler(
845         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
846         byte[] c) throws IOException {
847       postDeleteColumnHandlerCalled = true;
848     }
849 
850     public boolean wasDeleteColumnHandlerCalled() {
851       return preDeleteColumnHandlerCalled && postDeleteColumnHandlerCalled;
852     }
853 
854     public boolean preDeleteColumnHandlerCalledOnly() {
855       return preDeleteColumnHandlerCalled && !postDeleteColumnHandlerCalled;
856     }
857 
858     @Override
859     public void preEnableTableHandler(
860         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
861         throws IOException {
862       if (bypass) {
863         env.bypass();
864       }
865       preEnableTableHandlerCalled = true;
866     }
867 
868     @Override
869     public void postEnableTableHandler(
870         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
871         throws IOException {
872       postEnableTableHandlerCalled = true;
873     }
874 
875     public boolean wasEnableTableHandlerCalled() {
876       return preEnableTableHandlerCalled && postEnableTableHandlerCalled;
877     }
878 
879     public boolean preEnableTableHandlerCalledOnly() {
880       return preEnableTableHandlerCalled && !postEnableTableHandlerCalled;
881     }
882 
883     @Override
884     public void preDisableTableHandler(
885         ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
886         throws IOException {
887       if (bypass) {
888         env.bypass();
889       }
890       preDisableTableHandlerCalled = true;
891     }
892 
893     @Override
894     public void postDisableTableHandler(
895         ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
896         throws IOException {
897       postDisableTableHandlerCalled = true;
898     }
899 
900     public boolean wasDisableTableHandlerCalled() {
901       return preDisableTableHandlerCalled && postDisableTableHandlerCalled;
902     }
903 
904     public boolean preDisableTableHandlerCalledOnly() {
905       return preDisableTableHandlerCalled && !postDisableTableHandlerCalled;
906     }
907 
908     @Override
909     public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
910         List<TableName> tableNamesList, List<HTableDescriptor> descriptors)
911         throws IOException {
912       preGetTableDescriptorsCalled = true;
913     }
914 
915     @Override
916     public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
917         List<HTableDescriptor> descriptors) throws IOException {
918       postGetTableDescriptorsCalled = true;
919     }
920 
921     public boolean wasGetTableDescriptorsCalled() {
922       return preGetTableDescriptorsCalled && postGetTableDescriptorsCalled;
923     }
924   }
925 
926   private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
927   private static byte[] TEST_SNAPSHOT = Bytes.toBytes("observed_snapshot");
928   private static TableName TEST_TABLE =
929       TableName.valueOf("observed_table");
930   private static byte[] TEST_CLONE = Bytes.toBytes("observed_clone");
931   private static byte[] TEST_FAMILY = Bytes.toBytes("fam1");
932   private static byte[] TEST_FAMILY2 = Bytes.toBytes("fam2");
933   private static byte[] TEST_FAMILY3 = Bytes.toBytes("fam3");
934 
935   @BeforeClass
936   public static void setupBeforeClass() throws Exception {
937     Configuration conf = UTIL.getConfiguration();
938     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
939         CPMasterObserver.class.getName());
940     conf.set("hbase.master.hfilecleaner.plugins",
941       "org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner," +
942       "org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner");
943     conf.set("hbase.master.logcleaner.plugins",
944       "org.apache.hadoop.hbase.master.snapshot.SnapshotLogCleaner");
945     // We need more than one data server on this test
946     UTIL.startMiniCluster(2);
947   }
948 
949   @AfterClass
950   public static void tearDownAfterClass() throws Exception {
951     UTIL.shutdownMiniCluster();
952   }
953 
954   @Test
955   public void testStarted() throws Exception {
956     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
957 
958     HMaster master = cluster.getMaster();
959     assertTrue("Master should be active", master.isActiveMaster());
960     MasterCoprocessorHost host = master.getCoprocessorHost();
961     assertNotNull("CoprocessorHost should not be null", host);
962     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
963         CPMasterObserver.class.getName());
964     assertNotNull("CPMasterObserver coprocessor not found or not installed!", cp);
965 
966     // check basic lifecycle
967     assertTrue("MasterObserver should have been started", cp.wasStarted());
968     assertTrue("postStartMaster() hook should have been called",
969         cp.wasStartMasterCalled());
970   }
971 
972   @Test
973   public void testTableOperations() throws Exception {
974     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
975 
976     HMaster master = cluster.getMaster();
977     MasterCoprocessorHost host = master.getCoprocessorHost();
978     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
979         CPMasterObserver.class.getName());
980     cp.enableBypass(true);
981     cp.resetStates();
982     assertFalse("No table created yet", cp.wasCreateTableCalled());
983 
984     // create a table
985     HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
986     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
987     HBaseAdmin admin = UTIL.getHBaseAdmin();
988 
989     tableCreationLatch = new CountDownLatch(1);
990     admin.createTable(htd);
991     // preCreateTable can't bypass default action.
992     assertTrue("Test table should be created", cp.wasCreateTableCalled());
993     tableCreationLatch.await();
994     assertTrue("Table pre create handler called.", cp
995         .wasPreCreateTableHandlerCalled());
996     assertTrue("Table create handler should be called.",
997         cp.wasCreateTableHandlerCalled());
998 
999     tableCreationLatch = new CountDownLatch(1);
1000     admin.disableTable(TEST_TABLE);
1001     assertTrue(admin.isTableDisabled(TEST_TABLE));
1002     // preDisableTable can't bypass default action.
1003     assertTrue("Coprocessor should have been called on table disable",
1004       cp.wasDisableTableCalled());
1005     assertTrue("Disable table handler should be called.",
1006         cp.wasDisableTableHandlerCalled());
1007 
1008     // enable
1009     assertFalse(cp.wasEnableTableCalled());
1010     admin.enableTable(TEST_TABLE);
1011     assertTrue(admin.isTableEnabled(TEST_TABLE));
1012     // preEnableTable can't bypass default action.
1013     assertTrue("Coprocessor should have been called on table enable",
1014       cp.wasEnableTableCalled());
1015     assertTrue("Enable table handler should be called.",
1016         cp.wasEnableTableHandlerCalled());
1017 
1018     admin.disableTable(TEST_TABLE);
1019     assertTrue(admin.isTableDisabled(TEST_TABLE));
1020 
1021     // modify table
1022     htd.setMaxFileSize(512 * 1024 * 1024);
1023     modifyTableSync(admin, TEST_TABLE, htd);
1024     // preModifyTable can't bypass default action.
1025     assertTrue("Test table should have been modified",
1026       cp.wasModifyTableCalled());
1027 
1028     // add a column family
1029     admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
1030     assertTrue("New column family shouldn't have been added to test table",
1031       cp.preAddColumnCalledOnly());
1032 
1033     // modify a column family
1034     HColumnDescriptor hcd1 = new HColumnDescriptor(TEST_FAMILY2);
1035     hcd1.setMaxVersions(25);
1036     admin.modifyColumn(TEST_TABLE, hcd1);
1037     assertTrue("Second column family should be modified",
1038       cp.preModifyColumnCalledOnly());
1039 
1040     // delete table
1041     admin.deleteTable(TEST_TABLE);
1042     assertFalse("Test table should have been deleted",
1043         admin.tableExists(TEST_TABLE));
1044     // preDeleteTable can't bypass default action.
1045     assertTrue("Coprocessor should have been called on table delete",
1046         cp.wasDeleteTableCalled());
1047     assertTrue("Delete table handler should be called.",
1048         cp.wasDeleteTableHandlerCalled());
1049 
1050     // turn off bypass, run the tests again
1051     cp.enableBypass(false);
1052     cp.resetStates();
1053 
1054     admin.createTable(htd);
1055     assertTrue("Test table should be created", cp.wasCreateTableCalled());
1056     tableCreationLatch.await();
1057     assertTrue("Table pre create handler called.", cp
1058         .wasPreCreateTableHandlerCalled());
1059     assertTrue("Table create handler should be called.",
1060         cp.wasCreateTableHandlerCalled());
1061 
1062     // disable
1063     assertFalse(cp.wasDisableTableCalled());
1064     assertFalse(cp.wasDisableTableHandlerCalled());
1065     admin.disableTable(TEST_TABLE);
1066     assertTrue(admin.isTableDisabled(TEST_TABLE));
1067     assertTrue("Coprocessor should have been called on table disable",
1068       cp.wasDisableTableCalled());
1069     assertTrue("Disable table handler should be called.",
1070         cp.wasDisableTableHandlerCalled());
1071 
1072     // modify table
1073     htd.setMaxFileSize(512 * 1024 * 1024);
1074     modifyTableSync(admin, TEST_TABLE, htd);
1075     assertTrue("Test table should have been modified",
1076         cp.wasModifyTableCalled());
1077     // add a column family
1078     admin.addColumn(TEST_TABLE, new HColumnDescriptor(TEST_FAMILY2));
1079     assertTrue("New column family should have been added to test table",
1080         cp.wasAddColumnCalled());
1081     assertTrue("Add column handler should be called.",
1082         cp.wasAddColumnHandlerCalled());
1083 
1084     // modify a column family
1085     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
1086     hcd.setMaxVersions(25);
1087     admin.modifyColumn(TEST_TABLE, hcd);
1088     assertTrue("Second column family should be modified",
1089         cp.wasModifyColumnCalled());
1090     assertTrue("Modify table handler should be called.",
1091         cp.wasModifyColumnHandlerCalled());
1092 
1093     // enable
1094     assertFalse(cp.wasEnableTableCalled());
1095     assertFalse(cp.wasEnableTableHandlerCalled());
1096     admin.enableTable(TEST_TABLE);
1097     assertTrue(admin.isTableEnabled(TEST_TABLE));
1098     assertTrue("Coprocessor should have been called on table enable",
1099         cp.wasEnableTableCalled());
1100     assertTrue("Enable table handler should be called.",
1101         cp.wasEnableTableHandlerCalled());
1102 
1103     // disable again
1104     admin.disableTable(TEST_TABLE);
1105     assertTrue(admin.isTableDisabled(TEST_TABLE));
1106 
1107     // delete column
1108     assertFalse("No column family deleted yet", cp.wasDeleteColumnCalled());
1109     assertFalse("Delete table column handler should not be called.",
1110         cp.wasDeleteColumnHandlerCalled());
1111     admin.deleteColumn(TEST_TABLE, TEST_FAMILY2);
1112     HTableDescriptor tableDesc = admin.getTableDescriptor(TEST_TABLE);
1113     assertNull("'"+Bytes.toString(TEST_FAMILY2)+"' should have been removed",
1114         tableDesc.getFamily(TEST_FAMILY2));
1115     assertTrue("Coprocessor should have been called on column delete",
1116         cp.wasDeleteColumnCalled());
1117     assertTrue("Delete table column handler should be called.",
1118         cp.wasDeleteColumnHandlerCalled());
1119 
1120     // delete table
1121     assertFalse("No table deleted yet", cp.wasDeleteTableCalled());
1122     assertFalse("Delete table handler should not be called.",
1123         cp.wasDeleteTableHandlerCalled());
1124     admin.deleteTable(TEST_TABLE);
1125     assertFalse("Test table should have been deleted",
1126         admin.tableExists(TEST_TABLE));
1127     assertTrue("Coprocessor should have been called on table delete",
1128         cp.wasDeleteTableCalled());
1129     assertTrue("Delete table handler should be called.",
1130         cp.wasDeleteTableHandlerCalled());
1131   }
1132 
1133   @Test
1134   public void testSnapshotOperations() throws Exception {
1135     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1136     HMaster master = cluster.getMaster();
1137     MasterCoprocessorHost host = master.getCoprocessorHost();
1138     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1139         CPMasterObserver.class.getName());
1140     cp.resetStates();
1141 
1142     // create a table
1143     HTableDescriptor htd = new HTableDescriptor(TEST_TABLE);
1144     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
1145     HBaseAdmin admin = UTIL.getHBaseAdmin();
1146 
1147     tableCreationLatch = new CountDownLatch(1);
1148     admin.createTable(htd);
1149     tableCreationLatch.await();
1150     tableCreationLatch = new CountDownLatch(1);
1151 
1152     admin.disableTable(TEST_TABLE);
1153     assertTrue(admin.isTableDisabled(TEST_TABLE));
1154 
1155     try {
1156       // Test snapshot operation
1157       assertFalse("Coprocessor should not have been called yet",
1158         cp.wasSnapshotCalled());
1159       admin.snapshot(TEST_SNAPSHOT, TEST_TABLE);
1160       assertTrue("Coprocessor should have been called on snapshot",
1161         cp.wasSnapshotCalled());
1162 
1163       // Test clone operation
1164       admin.cloneSnapshot(TEST_SNAPSHOT, TEST_CLONE);
1165       assertTrue("Coprocessor should have been called on snapshot clone",
1166         cp.wasCloneSnapshotCalled());
1167       assertFalse("Coprocessor restore should not have been called on snapshot clone",
1168         cp.wasRestoreSnapshotCalled());
1169       admin.disableTable(TEST_CLONE);
1170       assertTrue(admin.isTableDisabled(TEST_TABLE));
1171       admin.deleteTable(TEST_CLONE);
1172 
1173       // Test restore operation
1174       cp.resetStates();
1175       admin.restoreSnapshot(TEST_SNAPSHOT);
1176       assertTrue("Coprocessor should have been called on snapshot restore",
1177         cp.wasRestoreSnapshotCalled());
1178       assertFalse("Coprocessor clone should not have been called on snapshot restore",
1179         cp.wasCloneSnapshotCalled());
1180 
1181       admin.deleteSnapshot(TEST_SNAPSHOT);
1182       assertTrue("Coprocessor should have been called on snapshot delete",
1183         cp.wasDeleteSnapshotCalled());
1184     } finally {
1185       admin.deleteTable(TEST_TABLE);
1186     }
1187   }
1188 
1189   @Test
1190   public void testNamespaceOperations() throws Exception {
1191     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1192     String testNamespace = "observed_ns";
1193     HMaster master = cluster.getMaster();
1194     MasterCoprocessorHost host = master.getCoprocessorHost();
1195     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1196         CPMasterObserver.class.getName());
1197 
1198     cp.enableBypass(false);
1199     cp.resetStates();
1200 
1201 
1202     // create a table
1203     HBaseAdmin admin = UTIL.getHBaseAdmin();
1204     admin.createNamespace(NamespaceDescriptor.create(testNamespace).build());
1205     assertTrue("Test namespace should be created", cp.wasCreateNamespaceCalled());
1206 
1207     assertNotNull(admin.getNamespaceDescriptor(testNamespace));
1208 
1209     // turn off bypass, run the tests again
1210     cp.enableBypass(true);
1211     cp.resetStates();
1212 
1213     admin.modifyNamespace(NamespaceDescriptor.create(testNamespace).build());
1214     assertTrue("Test namespace should not have been modified",
1215         cp.preModifyNamespaceCalledOnly());
1216 
1217     assertNotNull(admin.getNamespaceDescriptor(testNamespace));
1218 
1219     admin.deleteNamespace(testNamespace);
1220     assertTrue("Test namespace should not have been deleted", cp.preDeleteNamespaceCalledOnly());
1221 
1222     assertNotNull(admin.getNamespaceDescriptor(testNamespace));
1223 
1224     cp.enableBypass(false);
1225     cp.resetStates();
1226 
1227     // delete table
1228     admin.modifyNamespace(NamespaceDescriptor.create(testNamespace).build());
1229     assertTrue("Test namespace should have been modified", cp.wasModifyNamespaceCalled());
1230 
1231     admin.deleteNamespace(testNamespace);
1232     assertTrue("Test namespace should have been deleted", cp.wasDeleteNamespaceCalled());
1233 
1234     cp.enableBypass(true);
1235     cp.resetStates();
1236 
1237     admin.createNamespace(NamespaceDescriptor.create(testNamespace).build());
1238     assertTrue("Test namespace should not be created", cp.preCreateNamespaceCalledOnly());
1239   }
1240 
1241   private void modifyTableSync(HBaseAdmin admin, TableName tableName, HTableDescriptor htd)
1242       throws IOException {
1243     admin.modifyTable(tableName, htd);
1244     //wait until modify table finishes
1245     for (int t = 0; t < 100; t++) { //10 sec timeout
1246       HTableDescriptor td = admin.getTableDescriptor(htd.getTableName());
1247       if (td.equals(htd)) {
1248         break;
1249       }
1250       Threads.sleep(100);
1251     }
1252   }
1253 
1254   @Test
1255   public void testRegionTransitionOperations() throws Exception {
1256     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1257 
1258     HMaster master = cluster.getMaster();
1259     MasterCoprocessorHost host = master.getCoprocessorHost();
1260     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1261         CPMasterObserver.class.getName());
1262     cp.enableBypass(false);
1263     cp.resetStates();
1264 
1265     HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
1266 
1267     try {
1268       UTIL.createMultiRegions(table, TEST_FAMILY);
1269       UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
1270   
1271       NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
1272       Map.Entry<HRegionInfo, ServerName> firstGoodPair = null;
1273       for (Map.Entry<HRegionInfo, ServerName> e: regions.entrySet()) {
1274         if (e.getValue() != null) {
1275           firstGoodPair = e;
1276           break;
1277         }
1278       }
1279       assertNotNull("Found a non-null entry", firstGoodPair);
1280       LOG.info("Found " + firstGoodPair.toString());
1281       // Try to force a move
1282       Collection<ServerName> servers = master.getClusterStatus().getServers();
1283       String destName = null;
1284       String serverNameForFirstRegion = firstGoodPair.getValue().toString();
1285       LOG.info("serverNameForFirstRegion=" + serverNameForFirstRegion);
1286       boolean found = false;
1287       // Find server that is NOT carrying the first region
1288       for (ServerName info : servers) {
1289         LOG.info("ServerName=" + info);
1290         if (!serverNameForFirstRegion.equals(info.getServerName())) {
1291           destName = info.toString();
1292           found = true;
1293           break;
1294         }
1295       }
1296       assertTrue("Found server", found);
1297       LOG.info("Found " + destName);
1298       master.moveRegion(null,RequestConverter.buildMoveRegionRequest(
1299         firstGoodPair.getKey().getEncodedNameAsBytes(),Bytes.toBytes(destName)));
1300       assertTrue("Coprocessor should have been called on region move",
1301         cp.wasMoveCalled());
1302   
1303       // make sure balancer is on
1304       master.balanceSwitch(true);
1305       assertTrue("Coprocessor should have been called on balance switch",
1306           cp.wasBalanceSwitchCalled());
1307   
1308       // turn balancer off
1309       master.balanceSwitch(false);
1310   
1311       // wait for assignments to finish, if any
1312       AssignmentManager mgr = master.getAssignmentManager();
1313       Collection<RegionState> transRegions =
1314         mgr.getRegionStates().getRegionsInTransition().values();
1315       for (RegionState state : transRegions) {
1316         mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion());
1317       }
1318   
1319       // move half the open regions from RS 0 to RS 1
1320       HRegionServer rs = cluster.getRegionServer(0);
1321       byte[] destRS = Bytes.toBytes(cluster.getRegionServer(1).getServerName().toString());
1322       //Make sure no regions are in transition now
1323       waitForRITtoBeZero(master);
1324       List<HRegionInfo> openRegions = ProtobufUtil.getOnlineRegions(rs);
1325       int moveCnt = openRegions.size()/2;
1326       for (int i=0; i<moveCnt; i++) {
1327         HRegionInfo info = openRegions.get(i);
1328         if (!info.isMetaTable()) {
1329           master.moveRegion(null,RequestConverter.buildMoveRegionRequest(
1330             openRegions.get(i).getEncodedNameAsBytes(), destRS));
1331         }
1332       }
1333       //Make sure no regions are in transition now
1334       waitForRITtoBeZero(master);
1335       // now trigger a balance
1336       master.balanceSwitch(true);
1337       boolean balanceRun = master.balance();
1338       assertTrue("Coprocessor should be called on region rebalancing",
1339           cp.wasBalanceCalled());
1340     } finally {
1341       UTIL.deleteTable(TEST_TABLE);
1342     }
1343   }
1344 
1345   private void waitForRITtoBeZero(HMaster master) throws Exception {
1346     // wait for assignments to finish
1347     AssignmentManager mgr = master.getAssignmentManager();
1348     Collection<RegionState> transRegions =
1349       mgr.getRegionStates().getRegionsInTransition().values();
1350     for (RegionState state : transRegions) {
1351       mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion());
1352     }
1353   }
1354 
1355   @Test
1356   public void testTableDescriptorsEnumeration() throws Exception {
1357     MiniHBaseCluster cluster = UTIL.getHBaseCluster();
1358 
1359     HMaster master = cluster.getMaster();
1360     MasterCoprocessorHost host = master.getCoprocessorHost();
1361     CPMasterObserver cp = (CPMasterObserver)host.findCoprocessor(
1362         CPMasterObserver.class.getName());
1363     cp.resetStates();
1364 
1365     GetTableDescriptorsRequest req =
1366         RequestConverter.buildGetTableDescriptorsRequest((List<TableName>)null);
1367     master.getTableDescriptors(null, req);
1368 
1369     assertTrue("Coprocessor should be called on table descriptors request",
1370       cp.wasGetTableDescriptorsCalled());
1371   }
1372 
1373 }