View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.lang.Thread.UncaughtExceptionHandler;
23  import java.lang.annotation.Retention;
24  import java.lang.annotation.RetentionPolicy;
25  import java.lang.management.ManagementFactory;
26  import java.lang.management.MemoryUsage;
27  import java.lang.reflect.Constructor;
28  import java.net.BindException;
29  import java.net.InetSocketAddress;
30  import java.util.ArrayList;
31  import java.util.Collection;
32  import java.util.Collections;
33  import java.util.Comparator;
34  import java.util.HashMap;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.Map.Entry;
40  import java.util.Random;
41  import java.util.Set;
42  import java.util.SortedMap;
43  import java.util.TreeMap;
44  import java.util.TreeSet;
45  import java.util.concurrent.ConcurrentHashMap;
46  import java.util.concurrent.ConcurrentMap;
47  import java.util.concurrent.ConcurrentSkipListMap;
48  import java.util.concurrent.locks.ReentrantReadWriteLock;
49  
50  import javax.management.ObjectName;
51  
52  import org.apache.commons.logging.Log;
53  import org.apache.commons.logging.LogFactory;
54  import org.apache.hadoop.classification.InterfaceAudience;
55  import org.apache.hadoop.conf.Configuration;
56  import org.apache.hadoop.fs.FileSystem;
57  import org.apache.hadoop.fs.Path;
58  import org.apache.hadoop.hbase.Cell;
59  import org.apache.hadoop.hbase.CellScannable;
60  import org.apache.hadoop.hbase.CellScanner;
61  import org.apache.hadoop.hbase.CellUtil;
62  import org.apache.hadoop.hbase.Chore;
63  import org.apache.hadoop.hbase.ClockOutOfSyncException;
64  import org.apache.hadoop.hbase.DoNotRetryIOException;
65  import org.apache.hadoop.hbase.HBaseConfiguration;
66  import org.apache.hadoop.hbase.HBaseIOException;
67  import org.apache.hadoop.hbase.HConstants;
68  import org.apache.hadoop.hbase.HRegionInfo;
69  import org.apache.hadoop.hbase.HTableDescriptor;
70  import org.apache.hadoop.hbase.HealthCheckChore;
71  import org.apache.hadoop.hbase.KeyValueUtil;
72  import org.apache.hadoop.hbase.NotServingRegionException;
73  import org.apache.hadoop.hbase.RemoteExceptionHandler;
74  import org.apache.hadoop.hbase.ServerName;
75  import org.apache.hadoop.hbase.Stoppable;
76  import org.apache.hadoop.hbase.TableDescriptors;
77  import org.apache.hadoop.hbase.TableName;
78  import org.apache.hadoop.hbase.UnknownScannerException;
79  import org.apache.hadoop.hbase.YouAreDeadException;
80  import org.apache.hadoop.hbase.ZNodeClearer;
81  import org.apache.hadoop.hbase.catalog.CatalogTracker;
82  import org.apache.hadoop.hbase.catalog.MetaEditor;
83  import org.apache.hadoop.hbase.catalog.MetaReader;
84  import org.apache.hadoop.hbase.client.Append;
85  import org.apache.hadoop.hbase.client.Delete;
86  import org.apache.hadoop.hbase.client.Get;
87  import org.apache.hadoop.hbase.client.HConnectionManager;
88  import org.apache.hadoop.hbase.client.Increment;
89  import org.apache.hadoop.hbase.client.Mutation;
90  import org.apache.hadoop.hbase.client.Put;
91  import org.apache.hadoop.hbase.client.Result;
92  import org.apache.hadoop.hbase.client.RowMutations;
93  import org.apache.hadoop.hbase.client.Scan;
94  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
95  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
96  import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
97  import org.apache.hadoop.hbase.exceptions.RegionMovedException;
98  import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
99  import org.apache.hadoop.hbase.executor.ExecutorService;
100 import org.apache.hadoop.hbase.executor.ExecutorType;
101 import org.apache.hadoop.hbase.filter.ByteArrayComparable;
102 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
103 import org.apache.hadoop.hbase.fs.HFileSystem;
104 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
105 import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
106 import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
107 import org.apache.hadoop.hbase.ipc.RpcCallContext;
108 import org.apache.hadoop.hbase.ipc.RpcClient;
109 import org.apache.hadoop.hbase.ipc.RpcServer;
110 import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
111 import org.apache.hadoop.hbase.ipc.RpcServerInterface;
112 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
113 import org.apache.hadoop.hbase.ipc.ServerRpcController;
114 import org.apache.hadoop.hbase.master.SplitLogManager;
115 import org.apache.hadoop.hbase.master.TableLockManager;
116 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
117 import org.apache.hadoop.hbase.protobuf.RequestConverter;
118 import org.apache.hadoop.hbase.protobuf.ResponseConverter;
119 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
120 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
121 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
122 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest;
123 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionResponse;
124 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
125 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionResponse;
126 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
127 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
128 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
129 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
130 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
131 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
132 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
133 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
134 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
135 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse;
136 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
137 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
138 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
139 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState;
140 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
141 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
142 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
143 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
144 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
145 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionResponse;
146 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
147 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerResponse;
148 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
149 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
150 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WALEntry;
151 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
152 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
153 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest.FamilyPath;
154 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
155 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Condition;
156 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
157 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
158 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest;
159 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse;
160 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
161 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
162 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
163 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
164 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
165 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
166 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
167 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
168 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
169 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
170 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
171 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
172 import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
173 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.Coprocessor;
174 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
175 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
176 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
177 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
178 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
179 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
180 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
181 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
182 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
183 import org.apache.hadoop.hbase.regionserver.HRegion.Operation;
184 import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException;
185 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
186 import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
187 import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
188 import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler;
189 import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
190 import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
191 import org.apache.hadoop.hbase.regionserver.wal.HLog;
192 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
193 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
194 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
195 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
196 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
197 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
198 import org.apache.hadoop.hbase.security.UserProvider;
199 import org.apache.hadoop.hbase.trace.SpanReceiverHost;
200 import org.apache.hadoop.hbase.util.Bytes;
201 import org.apache.hadoop.hbase.util.CompressionTest;
202 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
203 import org.apache.hadoop.hbase.util.FSTableDescriptors;
204 import org.apache.hadoop.hbase.util.FSUtils;
205 import org.apache.hadoop.hbase.util.InfoServer;
206 import org.apache.hadoop.hbase.util.JvmPauseMonitor;
207 import org.apache.hadoop.hbase.util.Pair;
208 import org.apache.hadoop.hbase.util.Sleeper;
209 import org.apache.hadoop.hbase.util.Strings;
210 import org.apache.hadoop.hbase.util.Threads;
211 import org.apache.hadoop.hbase.util.VersionInfo;
212 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
213 import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
214 import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
215 import org.apache.hadoop.hbase.zookeeper.RecoveringRegionWatcher;
216 import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
217 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
218 import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
219 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
220 import org.apache.hadoop.ipc.RemoteException;
221 import org.apache.hadoop.metrics.util.MBeanUtil;
222 import org.apache.hadoop.net.DNS;
223 import org.apache.hadoop.util.ReflectionUtils;
224 import org.apache.hadoop.util.StringUtils;
225 import org.apache.zookeeper.KeeperException;
226 import org.apache.zookeeper.data.Stat;
227 import org.cliffc.high_scale_lib.Counter;
228 
229 import com.google.protobuf.BlockingRpcChannel;
230 import com.google.protobuf.ByteString;
231 import com.google.protobuf.Message;
232 import com.google.protobuf.RpcController;
233 import com.google.protobuf.ServiceException;
234 import com.google.protobuf.TextFormat;
235 import com.google.protobuf.ZeroCopyLiteralByteString;
236 
237 /**
238  * HRegionServer makes a set of HRegions available to clients. It checks in with
239  * the HMaster. There are many HRegionServers in a single HBase deployment.
240  */
241 @InterfaceAudience.Private
242 @SuppressWarnings("deprecation")
243 public class HRegionServer implements ClientProtos.ClientService.BlockingInterface,
244   AdminProtos.AdminService.BlockingInterface, Runnable, RegionServerServices,
245   HBaseRPCErrorHandler, LastSequenceId {
246 
247   public static final Log LOG = LogFactory.getLog(HRegionServer.class);
248 
249   private final Random rand;
250 
251   /*
252    * Strings to be used in forming the exception message for
253    * RegionsAlreadyInTransitionException.
254    */
255   protected static final String OPEN = "OPEN";
256   protected static final String CLOSE = "CLOSE";
257 
258   //RegionName vs current action in progress
259   //true - if open region action in progress
260   //false - if close region action in progress
261   protected final ConcurrentMap<byte[], Boolean> regionsInTransitionInRS =
262     new ConcurrentSkipListMap<byte[], Boolean>(Bytes.BYTES_COMPARATOR);
263 
264   protected long maxScannerResultSize;
265 
266   // Cache flushing
267   protected MemStoreFlusher cacheFlusher;
268 
269   // catalog tracker
270   protected CatalogTracker catalogTracker;
271 
272   // Watch if a region is out of recovering state from ZooKeeper
273   @SuppressWarnings("unused")
274   private RecoveringRegionWatcher recoveringRegionWatcher;
275 
276   /**
277    * Go here to get table descriptors.
278    */
279   protected TableDescriptors tableDescriptors;
280 
281   // Replication services. If no replication, this handler will be null.
282   protected ReplicationSourceService replicationSourceHandler;
283   protected ReplicationSinkService replicationSinkHandler;
284 
285   // Compactions
286   public CompactSplitThread compactSplitThread;
287 
288   final ConcurrentHashMap<String, RegionScannerHolder> scanners =
289       new ConcurrentHashMap<String, RegionScannerHolder>();
290 
291   /**
292    * Map of regions currently being served by this region server. Key is the
293    * encoded region name.  All access should be synchronized.
294    */
295   protected final Map<String, HRegion> onlineRegions =
296     new ConcurrentHashMap<String, HRegion>();
297 
298   /**
299    * Map of encoded region names to the DataNode locations they should be hosted on
300    * We store the value as InetSocketAddress since this is used only in HDFS
301    * API (create() that takes favored nodes as hints for placing file blocks).
302    * We could have used ServerName here as the value class, but we'd need to
303    * convert it to InetSocketAddress at some point before the HDFS API call, and
304    * it seems a bit weird to store ServerName since ServerName refers to RegionServers
305    * and here we really mean DataNode locations.
306    */
307   protected final Map<String, InetSocketAddress[]> regionFavoredNodesMap =
308       new ConcurrentHashMap<String, InetSocketAddress[]>();
309 
310   /**
311    * Set of regions currently being in recovering state which means it can accept writes(edits from
312    * previous failed region server) but not reads. A recovering region is also an online region.
313    */
314   protected final Map<String, HRegion> recoveringRegions = Collections
315       .synchronizedMap(new HashMap<String, HRegion>());
316 
317   // Leases
318   protected Leases leases;
319 
320   // Instance of the hbase executor service.
321   protected ExecutorService service;
322 
323   // Request counter. (Includes requests that are not serviced by regions.)
324   final Counter requestCount = new Counter();
325 
326   // If false, the file system has become unavailable
327   protected volatile boolean fsOk;
328   protected HFileSystem fs;
329 
330   // Set when a report to the master comes back with a message asking us to
331   // shutdown. Also set by call to stop when debugging or running unit tests
332   // of HRegionServer in isolation.
333   protected volatile boolean stopped = false;
334 
335   // Go down hard. Used if file system becomes unavailable and also in
336   // debugging and unit tests.
337   protected volatile boolean abortRequested;
338 
339   // Port we put up the webui on.
340   protected int webuiport = -1;
341 
342   ConcurrentMap<String, Integer> rowlocks = new ConcurrentHashMap<String, Integer>();
343 
344   // A state before we go into stopped state.  At this stage we're closing user
345   // space regions.
346   private boolean stopping = false;
347 
348   private volatile boolean killed = false;
349 
350   protected final Configuration conf;
351 
352   private Path rootDir;
353 
354   protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
355 
356   final int numRetries;
357   protected final int threadWakeFrequency;
358   private final int msgInterval;
359 
360   protected final int numRegionsToReport;
361 
362   // Stub to do region server status calls against the master.
363   private RegionServerStatusService.BlockingInterface rssStub;
364   // RPC client. Used to make the stub above that does region server status checking.
365   RpcClient rpcClient;
366 
367   // Server to handle client requests. Default access so can be accessed by
368   // unit tests.
369   RpcServerInterface rpcServer;
370 
371   private final InetSocketAddress isa;
372   private UncaughtExceptionHandler uncaughtExceptionHandler;
373 
374   // Info server. Default access so can be used by unit tests. REGIONSERVER
375   // is name of the webapp and the attribute name used stuffing this instance
376   // into web context.
377   InfoServer infoServer;
378   private JvmPauseMonitor pauseMonitor;
379 
380   /** region server process name */
381   public static final String REGIONSERVER = "regionserver";
382 
383   /** region server configuration name */
384   public static final String REGIONSERVER_CONF = "regionserver_conf";
385 
386   private MetricsRegionServer metricsRegionServer;
387   private SpanReceiverHost spanReceiverHost;
388 
389   /*
390    * Check for compactions requests.
391    */
392   Chore compactionChecker;
393 
394   /*
395    * Check for flushes
396    */
397   Chore periodicFlusher;
398 
399   // HLog and HLog roller. log is protected rather than private to avoid
400   // eclipse warning when accessed by inner classes
401   protected volatile HLog hlog;
402   // The meta updates are written to a different hlog. If this
403   // regionserver holds meta regions, then this field will be non-null.
404   protected volatile HLog hlogForMeta;
405 
406   LogRoller hlogRoller;
407   LogRoller metaHLogRoller;
408 
409   // flag set after we're done setting up server threads (used for testing)
410   protected volatile boolean isOnline;
411 
412   // zookeeper connection and watcher
413   private ZooKeeperWatcher zooKeeper;
414 
415   // master address manager and watcher
416   private MasterAddressTracker masterAddressManager;
417 
418   // Cluster Status Tracker
419   private ClusterStatusTracker clusterStatusTracker;
420 
421   // Log Splitting Worker
422   private SplitLogWorker splitLogWorker;
423 
424   // A sleeper that sleeps for msgInterval.
425   private final Sleeper sleeper;
426 
427   private final int rpcTimeout;
428 
429   private final RegionServerAccounting regionServerAccounting;
430 
431   // Cache configuration and block cache reference
432   final CacheConfig cacheConfig;
433 
434   /** The health check chore. */
435   private HealthCheckChore healthCheckChore;
436 
437   /**
438    * The server name the Master sees us as.  Its made from the hostname the
439    * master passes us, port, and server startcode. Gets set after registration
440    * against  Master.  The hostname can differ from the hostname in {@link #isa}
441    * but usually doesn't if both servers resolve .
442    */
443   private ServerName serverNameFromMasterPOV;
444 
445   /**
446    * This servers startcode.
447    */
448   private final long startcode;
449 
450   /**
451    * Unique identifier for the cluster we are a part of.
452    */
453   private String clusterId;
454 
455   /**
456    * MX Bean for RegionServerInfo
457    */
458   private ObjectName mxBean = null;
459 
460   /**
461    * Chore to clean periodically the moved region list
462    */
463   private MovedRegionsCleaner movedRegionsCleaner;
464 
465   /**
466    * The lease timeout period for client scanners (milliseconds).
467    */
468   private final int scannerLeaseTimeoutPeriod;
469 
470   /**
471    * The reference to the QosFunction
472    */
473   private final QosFunction qosFunction;
474 
475   private RegionServerCoprocessorHost rsHost;
476 
477   /** Handle all the snapshot requests to this server */
478   RegionServerSnapshotManager snapshotManager;
479 
480   // configuration setting on if replay WAL edits directly to another RS
481   private final boolean distributedLogReplay;
482 
483   // Table level lock manager for locking for region operations
484   private TableLockManager tableLockManager;
485 
486   private UserProvider userProvider;
487 
488   /**
489    * Starts a HRegionServer at the default location
490    *
491    * @param conf
492    * @throws IOException
493    * @throws InterruptedException
494    */
495   public HRegionServer(Configuration conf)
496   throws IOException, InterruptedException {
497     this.fsOk = true;
498     this.conf = conf;
499     this.isOnline = false;
500     checkCodecs(this.conf);
501     this.userProvider = UserProvider.instantiate(conf);
502 
503     FSUtils.setupShortCircuitRead(this.conf);
504 
505     // Config'ed params
506     this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
507         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
508     this.threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
509     this.msgInterval = conf.getInt("hbase.regionserver.msginterval", 3 * 1000);
510 
511     this.sleeper = new Sleeper(this.msgInterval, this);
512 
513     this.maxScannerResultSize = conf.getLong(
514       HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
515       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
516 
517     this.numRegionsToReport = conf.getInt(
518       "hbase.regionserver.numregionstoreport", 10);
519 
520     this.rpcTimeout = conf.getInt(
521       HConstants.HBASE_RPC_SHORTOPERATION_TIMEOUT_KEY,
522       HConstants.DEFAULT_HBASE_RPC_SHORTOPERATION_TIMEOUT);
523 
524     this.abortRequested = false;
525     this.stopped = false;
526 
527     this.scannerLeaseTimeoutPeriod = HBaseConfiguration.getInt(conf,
528       HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
529       HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
530       HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
531 
532     // Server to handle client requests.
533     String hostname = conf.get("hbase.regionserver.ipc.address",
534       Strings.domainNamePointerToHostName(DNS.getDefaultHost(
535         conf.get("hbase.regionserver.dns.interface", "default"),
536         conf.get("hbase.regionserver.dns.nameserver", "default"))));
537     int port = conf.getInt(HConstants.REGIONSERVER_PORT,
538       HConstants.DEFAULT_REGIONSERVER_PORT);
539     // Creation of a HSA will force a resolve.
540     InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
541     if (initialIsa.getAddress() == null) {
542       throw new IllegalArgumentException("Failed resolve of " + initialIsa);
543     }
544     this.rand = new Random(initialIsa.hashCode());
545     String name = "regionserver/" + initialIsa.toString();
546     // Set how many times to retry talking to another server over HConnection.
547     HConnectionManager.setServerSideHConnectionRetries(this.conf, name, LOG);
548     this.rpcServer = new RpcServer(this, name, getServices(),
549       /*HBaseRPCErrorHandler.class, OnlineRegions.class},*/
550       initialIsa, // BindAddress is IP we got for this server.
551       conf.getInt("hbase.regionserver.handler.count", 10),
552       conf.getInt("hbase.regionserver.metahandler.count", 10),
553       conf, HConstants.QOS_THRESHOLD);
554 
555     // Set our address.
556     this.isa = this.rpcServer.getListenerAddress();
557 
558     this.rpcServer.setErrorHandler(this);
559     this.rpcServer.setQosFunction((qosFunction = new QosFunction(this)));
560     this.startcode = System.currentTimeMillis();
561 
562     // login the zookeeper client principal (if using security)
563     ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
564       "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
565 
566     // login the server principal (if using secure Hadoop)
567     userProvider.login("hbase.regionserver.keytab.file",
568       "hbase.regionserver.kerberos.principal", this.isa.getHostName());
569     regionServerAccounting = new RegionServerAccounting();
570     cacheConfig = new CacheConfig(conf);
571     uncaughtExceptionHandler = new UncaughtExceptionHandler() {
572       @Override
573       public void uncaughtException(Thread t, Throwable e) {
574         abort("Uncaught exception in service thread " + t.getName(), e);
575       }
576     };
577     this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
578 
579     this.distributedLogReplay = this.conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
580       HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);
581   }
582 
583   /**
584    * @return list of blocking services and their security info classes that this server supports
585    */
586   private List<BlockingServiceAndInterface> getServices() {
587     List<BlockingServiceAndInterface> bssi = new ArrayList<BlockingServiceAndInterface>(2);
588     bssi.add(new BlockingServiceAndInterface(
589         ClientProtos.ClientService.newReflectiveBlockingService(this),
590         ClientProtos.ClientService.BlockingInterface.class));
591     bssi.add(new BlockingServiceAndInterface(
592         AdminProtos.AdminService.newReflectiveBlockingService(this),
593         AdminProtos.AdminService.BlockingInterface.class));
594     return bssi;
595   }
596 
597   /**
598    * Run test on configured codecs to make sure supporting libs are in place.
599    * @param c
600    * @throws IOException
601    */
602   private static void checkCodecs(final Configuration c) throws IOException {
603     // check to see if the codec list is available:
604     String [] codecs = c.getStrings("hbase.regionserver.codecs", (String[])null);
605     if (codecs == null) return;
606     for (String codec : codecs) {
607       if (!CompressionTest.testCompression(codec)) {
608         throw new IOException("Compression codec " + codec +
609           " not supported, aborting RS construction");
610       }
611     }
612   }
613 
614   String getClusterId() {
615     return this.clusterId;
616   }
617 
618   @Retention(RetentionPolicy.RUNTIME)
619   protected @interface QosPriority {
620     int priority() default 0;
621   }
622 
623   QosFunction getQosFunction() {
624     return qosFunction;
625   }
626 
627   RegionScanner getScanner(long scannerId) {
628     String scannerIdString = Long.toString(scannerId);
629     RegionScannerHolder scannerHolder = scanners.get(scannerIdString);
630     if (scannerHolder != null) {
631       return scannerHolder.s;
632     }
633     return null;
634   }
635 
636   /**
637    * All initialization needed before we go register with Master.
638    *
639    * @throws IOException
640    * @throws InterruptedException
641    */
642   private void preRegistrationInitialization(){
643     try {
644       initializeZooKeeper();
645       initializeThreads();
646     } catch (Throwable t) {
647       // Call stop if error or process will stick around for ever since server
648       // puts up non-daemon threads.
649       this.rpcServer.stop();
650       abort("Initialization of RS failed.  Hence aborting RS.", t);
651     }
652   }
653 
654   /**
655    * Bring up connection to zk ensemble and then wait until a master for this
656    * cluster and then after that, wait until cluster 'up' flag has been set.
657    * This is the order in which master does things.
658    * Finally put up a catalog tracker.
659    * @throws IOException
660    * @throws InterruptedException
661    */
662   private void initializeZooKeeper() throws IOException, InterruptedException {
663     // Open connection to zookeeper and set primary watcher
664     this.zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" +
665       this.isa.getPort(), this);
666 
667     // Create the master address manager, register with zk, and start it.  Then
668     // block until a master is available.  No point in starting up if no master
669     // running.
670     this.masterAddressManager = new MasterAddressTracker(this.zooKeeper, this);
671     this.masterAddressManager.start();
672     blockAndCheckIfStopped(this.masterAddressManager);
673 
674     // Wait on cluster being up.  Master will set this flag up in zookeeper
675     // when ready.
676     this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);
677     this.clusterStatusTracker.start();
678     blockAndCheckIfStopped(this.clusterStatusTracker);
679 
680     // Create the catalog tracker and start it;
681     this.catalogTracker = new CatalogTracker(this.zooKeeper, this.conf, this);
682     catalogTracker.start();
683 
684     // Retrieve clusterId
685     // Since cluster status is now up
686     // ID should have already been set by HMaster
687     try {
688       clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
689       if (clusterId == null) {
690         this.abort("Cluster ID has not been set");
691       }
692       LOG.info("ClusterId : "+clusterId);
693     } catch (KeeperException e) {
694       this.abort("Failed to retrieve Cluster ID",e);
695     }
696 
697     // watch for snapshots
698     try {
699       this.snapshotManager = new RegionServerSnapshotManager(this);
700     } catch (KeeperException e) {
701       this.abort("Failed to reach zk cluster when creating snapshot handler.");
702     }
703     this.tableLockManager = TableLockManager.createTableLockManager(conf, zooKeeper,
704         ServerName.valueOf(isa.getHostName(), isa.getPort(), startcode));
705 
706     // register watcher for recovering regions
707     if(this.distributedLogReplay) {
708       this.recoveringRegionWatcher = new RecoveringRegionWatcher(this.zooKeeper, this);
709     }
710   }
711 
712   /**
713    * Utilty method to wait indefinitely on a znode availability while checking
714    * if the region server is shut down
715    * @param tracker znode tracker to use
716    * @throws IOException any IO exception, plus if the RS is stopped
717    * @throws InterruptedException
718    */
719   private void blockAndCheckIfStopped(ZooKeeperNodeTracker tracker)
720       throws IOException, InterruptedException {
721     while (tracker.blockUntilAvailable(this.msgInterval, false) == null) {
722       if (this.stopped) {
723         throw new IOException("Received the shutdown message while waiting.");
724       }
725     }
726   }
727 
728   /**
729    * @return False if cluster shutdown in progress
730    */
731   private boolean isClusterUp() {
732     return this.clusterStatusTracker.isClusterUp();
733   }
734 
735   private void initializeThreads() throws IOException {
736     // Cache flushing thread.
737     this.cacheFlusher = new MemStoreFlusher(conf, this);
738 
739     // Compaction thread
740     this.compactSplitThread = new CompactSplitThread(this);
741 
742     // Background thread to check for compactions; needed if region has not gotten updates
743     // in a while. It will take care of not checking too frequently on store-by-store basis.
744     this.compactionChecker = new CompactionChecker(this, this.threadWakeFrequency, this);
745     this.periodicFlusher = new PeriodicMemstoreFlusher(this.threadWakeFrequency, this);
746     // Health checker thread.
747     int sleepTime = this.conf.getInt(HConstants.HEALTH_CHORE_WAKE_FREQ,
748       HConstants.DEFAULT_THREAD_WAKE_FREQUENCY);
749     if (isHealthCheckerConfigured()) {
750       healthCheckChore = new HealthCheckChore(sleepTime, this, getConfiguration());
751     }
752 
753     this.leases = new Leases(this.threadWakeFrequency);
754 
755     // Create the thread to clean the moved regions list
756     movedRegionsCleaner = MovedRegionsCleaner.createAndStart(this);
757 
758     // Setup RPC client for master communication
759     rpcClient = new RpcClient(conf, clusterId, new InetSocketAddress(
760         this.isa.getAddress(), 0));
761     this.pauseMonitor = new JvmPauseMonitor(conf);
762     pauseMonitor.start();
763   }
764 
765   /**
766    * The HRegionServer sticks in this loop until closed.
767    */
768   @Override
769   public void run() {
770     try {
771       // Do pre-registration initializations; zookeeper, lease threads, etc.
772       preRegistrationInitialization();
773     } catch (Throwable e) {
774       abort("Fatal exception during initialization", e);
775     }
776 
777     try {
778       // Set our ephemeral znode up in zookeeper now we have a name.
779       createMyEphemeralNode();
780 
781       // Try and register with the Master; tell it we are here.  Break if
782       // server is stopped or the clusterup flag is down or hdfs went wacky.
783       while (keepLooping()) {
784         RegionServerStartupResponse w = reportForDuty();
785         if (w == null) {
786           LOG.warn("reportForDuty failed; sleeping and then retrying.");
787           this.sleeper.sleep();
788         } else {
789           handleReportForDutyResponse(w);
790           break;
791         }
792       }
793 
794       if (!this.stopped && isHealthy()){
795         // start the snapshot handler, since the server is ready to run
796         this.snapshotManager.start();
797       }
798 
799       // We registered with the Master.  Go into run mode.
800       long lastMsg = 0;
801       long oldRequestCount = -1;
802       // The main run loop.
803       while (!this.stopped && isHealthy()) {
804         if (!isClusterUp()) {
805           if (isOnlineRegionsEmpty()) {
806             stop("Exiting; cluster shutdown set and not carrying any regions");
807           } else if (!this.stopping) {
808             this.stopping = true;
809             LOG.info("Closing user regions");
810             closeUserRegions(this.abortRequested);
811           } else if (this.stopping) {
812             boolean allUserRegionsOffline = areAllUserRegionsOffline();
813             if (allUserRegionsOffline) {
814               // Set stopped if no requests since last time we went around the loop.
815               // The remaining meta regions will be closed on our way out.
816               if (oldRequestCount == this.requestCount.get()) {
817                 stop("Stopped; only catalog regions remaining online");
818                 break;
819               }
820               oldRequestCount = this.requestCount.get();
821             } else {
822               // Make sure all regions have been closed -- some regions may
823               // have not got it because we were splitting at the time of
824               // the call to closeUserRegions.
825               closeUserRegions(this.abortRequested);
826             }
827             LOG.debug("Waiting on " + getOnlineRegionsAsPrintableString());
828           }
829         }
830         long now = System.currentTimeMillis();
831         if ((now - lastMsg) >= msgInterval) {
832           tryRegionServerReport(lastMsg, now);
833           lastMsg = System.currentTimeMillis();
834         }
835         if (!this.stopped) this.sleeper.sleep();
836       } // for
837     } catch (Throwable t) {
838       if (!checkOOME(t)) {
839         String prefix = t instanceof YouAreDeadException? "": "Unhandled: ";
840         abort(prefix + t.getMessage(), t);
841       }
842     }
843     // Run shutdown.
844     if (mxBean != null) {
845       MBeanUtil.unregisterMBean(mxBean);
846       mxBean = null;
847     }
848     if (this.leases != null) this.leases.closeAfterLeasesExpire();
849     this.rpcServer.stop();
850     if (this.splitLogWorker != null) {
851       splitLogWorker.stop();
852     }
853     if (this.infoServer != null) {
854       LOG.info("Stopping infoServer");
855       try {
856         this.infoServer.stop();
857       } catch (Exception e) {
858         e.printStackTrace();
859       }
860     }
861     // Send cache a shutdown.
862     if (cacheConfig.isBlockCacheEnabled()) {
863       cacheConfig.getBlockCache().shutdown();
864     }
865 
866     movedRegionsCleaner.stop("Region Server stopping");
867 
868     // Send interrupts to wake up threads if sleeping so they notice shutdown.
869     // TODO: Should we check they are alive? If OOME could have exited already
870     if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();
871     if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();
872     if (this.hlogRoller != null) this.hlogRoller.interruptIfNecessary();
873     if (this.metaHLogRoller != null) this.metaHLogRoller.interruptIfNecessary();
874     if (this.compactionChecker != null)
875       this.compactionChecker.interrupt();
876     if (this.healthCheckChore != null) {
877       this.healthCheckChore.interrupt();
878     }
879 
880     // Stop the snapshot handler, forcefully killing all running tasks
881     try {
882       if (snapshotManager != null) snapshotManager.stop(this.abortRequested || this.killed);
883     } catch (IOException e) {
884       LOG.warn("Failed to close snapshot handler cleanly", e);
885     }
886 
887     if (this.killed) {
888       // Just skip out w/o closing regions.  Used when testing.
889     } else if (abortRequested) {
890       if (this.fsOk) {
891         closeUserRegions(abortRequested); // Don't leave any open file handles
892       }
893       LOG.info("aborting server " + this.serverNameFromMasterPOV);
894     } else {
895       closeUserRegions(abortRequested);
896       closeAllScanners();
897       LOG.info("stopping server " + this.serverNameFromMasterPOV);
898     }
899     // Interrupt catalog tracker here in case any regions being opened out in
900     // handlers are stuck waiting on meta.
901     if (this.catalogTracker != null) this.catalogTracker.stop();
902 
903     // Closing the compactSplit thread before closing meta regions
904     if (!this.killed && containsMetaTableRegions()) {
905       if (!abortRequested || this.fsOk) {
906         if (this.compactSplitThread != null) {
907           this.compactSplitThread.join();
908           this.compactSplitThread = null;
909         }
910         closeMetaTableRegions(abortRequested);
911       }
912     }
913 
914     if (!this.killed && this.fsOk) {
915       waitOnAllRegionsToClose(abortRequested);
916       LOG.info("stopping server " + this.serverNameFromMasterPOV +
917         "; all regions closed.");
918     }
919 
920     //fsOk flag may be changed when closing regions throws exception.
921     if (this.fsOk) {
922       closeWAL(!abortRequested);
923     }
924 
925     // Make sure the proxy is down.
926     if (this.rssStub != null) {
927       this.rssStub = null;
928     }
929     this.rpcClient.stop();
930     this.leases.close();
931     if (this.pauseMonitor != null) {
932       this.pauseMonitor.stop();
933     }
934 
935     if (!killed) {
936       join();
937     }
938 
939     try {
940       deleteMyEphemeralNode();
941     } catch (KeeperException e) {
942       LOG.warn("Failed deleting my ephemeral node", e);
943     }
944     // We may have failed to delete the znode at the previous step, but
945     //  we delete the file anyway: a second attempt to delete the znode is likely to fail again.
946     ZNodeClearer.deleteMyEphemeralNodeOnDisk();
947     this.zooKeeper.close();
948     LOG.info("stopping server " + this.serverNameFromMasterPOV +
949       "; zookeeper connection closed.");
950 
951     LOG.info(Thread.currentThread().getName() + " exiting");
952   }
953 
954   private boolean containsMetaTableRegions() {
955     return onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
956   }
957 
958   private boolean areAllUserRegionsOffline() {
959     if (getNumberOfOnlineRegions() > 2) return false;
960     boolean allUserRegionsOffline = true;
961     for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
962       if (!e.getValue().getRegionInfo().isMetaTable()) {
963         allUserRegionsOffline = false;
964         break;
965       }
966     }
967     return allUserRegionsOffline;
968   }
969 
970   void tryRegionServerReport(long reportStartTime, long reportEndTime)
971   throws IOException {
972     if (this.rssStub == null) {
973       // the current server is stopping.
974       return;
975     }
976     ClusterStatusProtos.ServerLoad sl = buildServerLoad(reportStartTime, reportEndTime);
977     try {
978       RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder();
979       ServerName sn = ServerName.parseVersionedServerName(
980         this.serverNameFromMasterPOV.getVersionedBytes());
981       request.setServer(ProtobufUtil.toServerName(sn));
982       request.setLoad(sl);
983       this.rssStub.regionServerReport(null, request.build());
984     } catch (ServiceException se) {
985       IOException ioe = ProtobufUtil.getRemoteException(se);
986       if (ioe instanceof YouAreDeadException) {
987         // This will be caught and handled as a fatal error in run()
988         throw ioe;
989       }
990       // Couldn't connect to the master, get location from zk and reconnect
991       // Method blocks until new master is found or we are stopped
992       Pair<ServerName, RegionServerStatusService.BlockingInterface> p =
993         createRegionServerStatusStub();
994       this.rssStub = p.getSecond();
995     }
996   }
997 
998   ClusterStatusProtos.ServerLoad buildServerLoad(long reportStartTime, long reportEndTime) {
999     // We're getting the MetricsRegionServerWrapper here because the wrapper computes requests
1000     // per second, and other metrics  As long as metrics are part of ServerLoad it's best to use
1001     // the wrapper to compute those numbers in one place.
1002     // In the long term most of these should be moved off of ServerLoad and the heart beat.
1003     // Instead they should be stored in an HBase table so that external visibility into HBase is
1004     // improved; Additionally the load balancer will be able to take advantage of a more complete
1005     // history.
1006     MetricsRegionServerWrapper regionServerWrapper = this.metricsRegionServer.getRegionServerWrapper();
1007     Collection<HRegion> regions = getOnlineRegionsLocalContext();
1008     MemoryUsage memory =
1009       ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
1010 
1011     ClusterStatusProtos.ServerLoad.Builder serverLoad =
1012       ClusterStatusProtos.ServerLoad.newBuilder();
1013     serverLoad.setNumberOfRequests((int) regionServerWrapper.getRequestsPerSecond());
1014     serverLoad.setTotalNumberOfRequests((int) regionServerWrapper.getTotalRequestCount());
1015     serverLoad.setUsedHeapMB((int)(memory.getUsed() / 1024 / 1024));
1016     serverLoad.setMaxHeapMB((int) (memory.getMax() / 1024 / 1024));
1017     Set<String> coprocessors = this.hlog.getCoprocessorHost().getCoprocessors();
1018     for (String coprocessor : coprocessors) {
1019       serverLoad.addCoprocessors(
1020         Coprocessor.newBuilder().setName(coprocessor).build());
1021     }
1022     for (HRegion region : regions) {
1023       serverLoad.addRegionLoads(createRegionLoad(region));
1024     }
1025     serverLoad.setReportStartTime(reportStartTime);
1026     serverLoad.setReportEndTime(reportEndTime);
1027     if (this.infoServer != null) {
1028       serverLoad.setInfoServerPort(this.infoServer.getPort());
1029     } else {
1030       serverLoad.setInfoServerPort(-1);
1031     }
1032     return serverLoad.build();
1033   }
1034 
1035   String getOnlineRegionsAsPrintableString() {
1036     StringBuilder sb = new StringBuilder();
1037     for (HRegion r: this.onlineRegions.values()) {
1038       if (sb.length() > 0) sb.append(", ");
1039       sb.append(r.getRegionInfo().getEncodedName());
1040     }
1041     return sb.toString();
1042   }
1043 
1044   /**
1045    * Wait on regions close.
1046    */
1047   private void waitOnAllRegionsToClose(final boolean abort) {
1048     // Wait till all regions are closed before going out.
1049     int lastCount = -1;
1050     long previousLogTime = 0;
1051     Set<String> closedRegions = new HashSet<String>();
1052     while (!isOnlineRegionsEmpty()) {
1053       int count = getNumberOfOnlineRegions();
1054       // Only print a message if the count of regions has changed.
1055       if (count != lastCount) {
1056         // Log every second at most
1057         if (System.currentTimeMillis() > (previousLogTime + 1000)) {
1058           previousLogTime = System.currentTimeMillis();
1059           lastCount = count;
1060           LOG.info("Waiting on " + count + " regions to close");
1061           // Only print out regions still closing if a small number else will
1062           // swamp the log.
1063           if (count < 10 && LOG.isDebugEnabled()) {
1064             LOG.debug(this.onlineRegions);
1065           }
1066         }
1067       }
1068       // Ensure all user regions have been sent a close. Use this to
1069       // protect against the case where an open comes in after we start the
1070       // iterator of onlineRegions to close all user regions.
1071       for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
1072         HRegionInfo hri = e.getValue().getRegionInfo();
1073         if (!this.regionsInTransitionInRS.containsKey(hri.getEncodedNameAsBytes())
1074             && !closedRegions.contains(hri.getEncodedName())) {
1075           closedRegions.add(hri.getEncodedName());
1076           // Don't update zk with this close transition; pass false.
1077           closeRegionIgnoreErrors(hri, abort);
1078         }
1079       }
1080       // No regions in RIT, we could stop waiting now.
1081       if (this.regionsInTransitionInRS.isEmpty()) {
1082         if (!isOnlineRegionsEmpty()) {
1083           LOG.info("We were exiting though online regions are not empty," +
1084               " because some regions failed closing");
1085         }
1086         break;
1087       }
1088       Threads.sleep(200);
1089     }
1090   }
1091 
1092   private void closeWAL(final boolean delete) {
1093     if (this.hlogForMeta != null) {
1094       // All hlogs (meta and non-meta) are in the same directory. Don't call
1095       // closeAndDelete here since that would delete all hlogs not just the
1096       // meta ones. We will just 'close' the hlog for meta here, and leave
1097       // the directory cleanup to the follow-on closeAndDelete call.
1098       try {
1099         this.hlogForMeta.close();
1100       } catch (Throwable e) {
1101         LOG.error("Metalog close and delete failed", RemoteExceptionHandler.checkThrowable(e));
1102       }
1103     }
1104     if (this.hlog != null) {
1105       try {
1106         if (delete) {
1107           hlog.closeAndDelete();
1108         } else {
1109           hlog.close();
1110         }
1111       } catch (Throwable e) {
1112         LOG.error("Close and delete failed", RemoteExceptionHandler.checkThrowable(e));
1113       }
1114     }
1115   }
1116 
1117   private void closeAllScanners() {
1118     // Close any outstanding scanners. Means they'll get an UnknownScanner
1119     // exception next time they come in.
1120     for (Map.Entry<String, RegionScannerHolder> e : this.scanners.entrySet()) {
1121       try {
1122         e.getValue().s.close();
1123       } catch (IOException ioe) {
1124         LOG.warn("Closing scanner " + e.getKey(), ioe);
1125       }
1126     }
1127   }
1128 
1129   /*
1130    * Run init. Sets up hlog and starts up all server threads.
1131    *
1132    * @param c Extra configuration.
1133    */
1134   protected void handleReportForDutyResponse(final RegionServerStartupResponse c)
1135   throws IOException {
1136     try {
1137       for (NameStringPair e : c.getMapEntriesList()) {
1138         String key = e.getName();
1139         // The hostname the master sees us as.
1140         if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) {
1141           String hostnameFromMasterPOV = e.getValue();
1142           this.serverNameFromMasterPOV = ServerName.valueOf(hostnameFromMasterPOV,
1143               this.isa.getPort(), this.startcode);
1144           if (!hostnameFromMasterPOV.equals(this.isa.getHostName())) {
1145             LOG.info("Master passed us a different hostname to use; was=" +
1146               this.isa.getHostName() + ", but now=" + hostnameFromMasterPOV);
1147           }
1148           continue;
1149         }
1150         String value = e.getValue();
1151         if (LOG.isDebugEnabled()) {
1152           LOG.debug("Config from master: " + key + "=" + value);
1153         }
1154         this.conf.set(key, value);
1155       }
1156 
1157       // hack! Maps DFSClient => RegionServer for logs.  HDFS made this
1158       // config param for task trackers, but we can piggyback off of it.
1159       if (this.conf.get("mapred.task.id") == null) {
1160         this.conf.set("mapred.task.id", "hb_rs_" +
1161           this.serverNameFromMasterPOV.toString());
1162       }
1163 
1164       // Save it in a file, this will allow to see if we crash
1165       ZNodeClearer.writeMyEphemeralNodeOnDisk(getMyEphemeralNodePath());
1166 
1167       // Master sent us hbase.rootdir to use. Should be fully qualified
1168       // path with file system specification included. Set 'fs.defaultFS'
1169       // to match the filesystem on hbase.rootdir else underlying hadoop hdfs
1170       // accessors will be going against wrong filesystem (unless all is set
1171       // to defaults).
1172       FSUtils.setFsDefault(this.conf, FSUtils.getRootDir(this.conf));
1173       // Get fs instance used by this RS.  Do we use checksum verification in the hbase? If hbase
1174       // checksum verification enabled, then automatically switch off hdfs checksum verification.
1175       boolean useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, true);
1176       this.fs = new HFileSystem(this.conf, useHBaseChecksum);
1177       this.rootDir = FSUtils.getRootDir(this.conf);
1178       this.tableDescriptors = new FSTableDescriptors(this.fs, this.rootDir, true);
1179       this.hlog = setupWALAndReplication();
1180       // Init in here rather than in constructor after thread name has been set
1181       this.metricsRegionServer = new MetricsRegionServer(new MetricsRegionServerWrapperImpl(this));
1182 
1183       spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
1184 
1185       startServiceThreads();
1186       LOG.info("Serving as " + this.serverNameFromMasterPOV +
1187         ", RpcServer on " + this.isa +
1188         ", sessionid=0x" +
1189         Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
1190       isOnline = true;
1191     } catch (Throwable e) {
1192       this.isOnline = false;
1193       stop("Failed initialization");
1194       throw convertThrowableToIOE(cleanup(e, "Failed init"),
1195           "Region server startup failed");
1196     } finally {
1197       sleeper.skipSleepCycle();
1198     }
1199   }
1200 
1201   private void createMyEphemeralNode() throws KeeperException {
1202     ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper, getMyEphemeralNodePath(),
1203       HConstants.EMPTY_BYTE_ARRAY);
1204   }
1205 
1206   private void deleteMyEphemeralNode() throws KeeperException {
1207     ZKUtil.deleteNode(this.zooKeeper, getMyEphemeralNodePath());
1208   }
1209 
1210   @Override
1211   public RegionServerAccounting getRegionServerAccounting() {
1212     return regionServerAccounting;
1213   }
1214 
1215   @Override
1216   public TableLockManager getTableLockManager() {
1217     return tableLockManager;
1218   }
1219 
1220   /*
1221    * @param r Region to get RegionLoad for.
1222    *
1223    * @return RegionLoad instance.
1224    *
1225    * @throws IOException
1226    */
1227   private RegionLoad createRegionLoad(final HRegion r) {
1228     byte[] name = r.getRegionName();
1229     int stores = 0;
1230     int storefiles = 0;
1231     int storeUncompressedSizeMB = 0;
1232     int storefileSizeMB = 0;
1233     int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
1234     int storefileIndexSizeMB = 0;
1235     int rootIndexSizeKB = 0;
1236     int totalStaticIndexSizeKB = 0;
1237     int totalStaticBloomSizeKB = 0;
1238     long totalCompactingKVs = 0;
1239     long currentCompactedKVs = 0;
1240     synchronized (r.stores) {
1241       stores += r.stores.size();
1242       for (Store store : r.stores.values()) {
1243         storefiles += store.getStorefilesCount();
1244         storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed()
1245             / 1024 / 1024);
1246         storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
1247         storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
1248         CompactionProgress progress = store.getCompactionProgress();
1249         if (progress != null) {
1250           totalCompactingKVs += progress.totalCompactingKVs;
1251           currentCompactedKVs += progress.currentCompactedKVs;
1252         }
1253 
1254         rootIndexSizeKB +=
1255             (int) (store.getStorefilesIndexSize() / 1024);
1256 
1257         totalStaticIndexSizeKB +=
1258           (int) (store.getTotalStaticIndexSize() / 1024);
1259 
1260         totalStaticBloomSizeKB +=
1261           (int) (store.getTotalStaticBloomSize() / 1024);
1262       }
1263     }
1264     RegionLoad.Builder regionLoad = RegionLoad.newBuilder();
1265     RegionSpecifier.Builder regionSpecifier = RegionSpecifier.newBuilder();
1266     regionSpecifier.setType(RegionSpecifierType.REGION_NAME);
1267     regionSpecifier.setValue(ZeroCopyLiteralByteString.wrap(name));
1268     regionLoad.setRegionSpecifier(regionSpecifier.build())
1269       .setStores(stores)
1270       .setStorefiles(storefiles)
1271       .setStoreUncompressedSizeMB(storeUncompressedSizeMB)
1272       .setStorefileSizeMB(storefileSizeMB)
1273       .setMemstoreSizeMB(memstoreSizeMB)
1274       .setStorefileIndexSizeMB(storefileIndexSizeMB)
1275       .setRootIndexSizeKB(rootIndexSizeKB)
1276       .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB)
1277       .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB)
1278       .setReadRequestsCount((int) r.readRequestsCount.get())
1279       .setWriteRequestsCount((int) r.writeRequestsCount.get())
1280       .setTotalCompactingKVs(totalCompactingKVs)
1281       .setCurrentCompactedKVs(currentCompactedKVs)
1282       .setCompleteSequenceId(r.completeSequenceId);
1283 
1284     return regionLoad.build();
1285   }
1286 
1287   /**
1288    * @param encodedRegionName
1289    * @return An instance of RegionLoad.
1290    */
1291   public RegionLoad createRegionLoad(final String encodedRegionName) {
1292     HRegion r = null;
1293     r = this.onlineRegions.get(encodedRegionName);
1294     return r != null ? createRegionLoad(r) : null;
1295   }
1296 
1297   /*
1298    * Inner class that runs on a long period checking if regions need compaction.
1299    */
1300   private static class CompactionChecker extends Chore {
1301     private final HRegionServer instance;
1302     private final int majorCompactPriority;
1303     private final static int DEFAULT_PRIORITY = Integer.MAX_VALUE;
1304     private long iteration = 0;
1305 
1306     CompactionChecker(final HRegionServer h, final int sleepTime,
1307         final Stoppable stopper) {
1308       super("CompactionChecker", sleepTime, h);
1309       this.instance = h;
1310       LOG.info(this.getName() + " runs every " + StringUtils.formatTime(sleepTime));
1311 
1312       /* MajorCompactPriority is configurable.
1313        * If not set, the compaction will use default priority.
1314        */
1315       this.majorCompactPriority = this.instance.conf.
1316         getInt("hbase.regionserver.compactionChecker.majorCompactPriority",
1317         DEFAULT_PRIORITY);
1318     }
1319 
1320     @Override
1321     protected void chore() {
1322       for (HRegion r : this.instance.onlineRegions.values()) {
1323         if (r == null)
1324           continue;
1325         for (Store s : r.getStores().values()) {
1326           try {
1327             long multiplier = s.getCompactionCheckMultiplier();
1328             assert multiplier > 0;
1329             if (iteration % multiplier != 0) continue;
1330             if (s.needsCompaction()) {
1331               // Queue a compaction. Will recognize if major is needed.
1332               this.instance.compactSplitThread.requestSystemCompaction(r, s, getName()
1333                   + " requests compaction");
1334             } else if (s.isMajorCompaction()) {
1335               if (majorCompactPriority == DEFAULT_PRIORITY
1336                   || majorCompactPriority > r.getCompactPriority()) {
1337                 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1338                     + " requests major compaction; use default priority", null);
1339               } else {
1340                 this.instance.compactSplitThread.requestCompaction(r, s, getName()
1341                     + " requests major compaction; use configured priority",
1342                   this.majorCompactPriority, null);
1343               }
1344             }
1345           } catch (IOException e) {
1346             LOG.warn("Failed major compaction check on " + r, e);
1347           }
1348         }
1349       }
1350       iteration = (iteration == Long.MAX_VALUE) ? 0 : (iteration + 1);
1351     }
1352   }
1353 
1354   class PeriodicMemstoreFlusher extends Chore {
1355     final HRegionServer server;
1356     final static int RANGE_OF_DELAY = 20000; //millisec
1357     final static int MIN_DELAY_TIME = 3000; //millisec
1358     public PeriodicMemstoreFlusher(int cacheFlushInterval, final HRegionServer server) {
1359       super(server.getServerName() + "-MemstoreFlusherChore", cacheFlushInterval, server);
1360       this.server = server;
1361     }
1362 
1363     @Override
1364     protected void chore() {
1365       for (HRegion r : this.server.onlineRegions.values()) {
1366         if (r == null)
1367           continue;
1368         if (r.shouldFlush()) {
1369           FlushRequester requester = server.getFlushRequester();
1370           if (requester != null) {
1371             long randomDelay = rand.nextInt(RANGE_OF_DELAY) + MIN_DELAY_TIME;
1372             LOG.info(getName() + " requesting flush for region " + r.getRegionNameAsString() +
1373                 " after a delay of " + randomDelay);
1374             //Throttle the flushes by putting a delay. If we don't throttle, and there
1375             //is a balanced write-load on the regions in a table, we might end up
1376             //overwhelming the filesystem with too many flushes at once.
1377             requester.requestDelayedFlush(r, randomDelay);
1378           }
1379         }
1380       }
1381     }
1382   }
1383 
1384   /**
1385    * Report the status of the server. A server is online once all the startup is
1386    * completed (setting up filesystem, starting service threads, etc.). This
1387    * method is designed mostly to be useful in tests.
1388    *
1389    * @return true if online, false if not.
1390    */
1391   public boolean isOnline() {
1392     return isOnline;
1393   }
1394 
1395   /**
1396    * Setup WAL log and replication if enabled.
1397    * Replication setup is done in here because it wants to be hooked up to WAL.
1398    * @return A WAL instance.
1399    * @throws IOException
1400    */
1401   private HLog setupWALAndReplication() throws IOException {
1402     final Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
1403     final String logName
1404       = HLogUtil.getHLogDirectoryName(this.serverNameFromMasterPOV.toString());
1405 
1406     Path logdir = new Path(rootDir, logName);
1407     if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
1408     if (this.fs.exists(logdir)) {
1409       throw new RegionServerRunningException("Region server has already " +
1410         "created directory at " + this.serverNameFromMasterPOV.toString());
1411     }
1412 
1413     // Instantiate replication manager if replication enabled.  Pass it the
1414     // log directories.
1415     createNewReplicationInstance(conf, this, this.fs, logdir, oldLogDir);
1416 
1417     return instantiateHLog(rootDir, logName);
1418   }
1419 
1420   private HLog getMetaWAL() throws IOException {
1421     if (this.hlogForMeta != null) return this.hlogForMeta;
1422     final String logName = HLogUtil.getHLogDirectoryName(this.serverNameFromMasterPOV.toString());
1423     Path logdir = new Path(rootDir, logName);
1424     if (LOG.isDebugEnabled()) LOG.debug("logdir=" + logdir);
1425     this.hlogForMeta = HLogFactory.createMetaHLog(this.fs.getBackingFs(), rootDir, logName,
1426       this.conf, getMetaWALActionListeners(), this.serverNameFromMasterPOV.toString());
1427     return this.hlogForMeta;
1428   }
1429 
1430   /**
1431    * Called by {@link #setupWALAndReplication()} creating WAL instance.
1432    * @param rootdir
1433    * @param logName
1434    * @return WAL instance.
1435    * @throws IOException
1436    */
1437   protected HLog instantiateHLog(Path rootdir, String logName) throws IOException {
1438     return HLogFactory.createHLog(this.fs.getBackingFs(), rootdir, logName, this.conf,
1439       getWALActionListeners(), this.serverNameFromMasterPOV.toString());
1440   }
1441 
1442   /**
1443    * Called by {@link #instantiateHLog(Path, String)} setting up WAL instance.
1444    * Add any {@link WALActionsListener}s you want inserted before WAL startup.
1445    * @return List of WALActionsListener that will be passed in to
1446    * {@link org.apache.hadoop.hbase.regionserver.wal.FSHLog} on construction.
1447    */
1448   protected List<WALActionsListener> getWALActionListeners() {
1449     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
1450     // Log roller.
1451     this.hlogRoller = new LogRoller(this, this);
1452     listeners.add(this.hlogRoller);
1453     if (this.replicationSourceHandler != null &&
1454         this.replicationSourceHandler.getWALActionsListener() != null) {
1455       // Replication handler is an implementation of WALActionsListener.
1456       listeners.add(this.replicationSourceHandler.getWALActionsListener());
1457     }
1458     return listeners;
1459   }
1460 
1461   protected List<WALActionsListener> getMetaWALActionListeners() {
1462     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
1463     // Using a tmp log roller to ensure metaLogRoller is alive once it is not
1464     // null
1465     MetaLogRoller tmpLogRoller = new MetaLogRoller(this, this);
1466     String n = Thread.currentThread().getName();
1467     Threads.setDaemonThreadRunning(tmpLogRoller.getThread(),
1468         n + "-MetaLogRoller", uncaughtExceptionHandler);
1469     this.metaHLogRoller = tmpLogRoller;
1470     tmpLogRoller = null;
1471     listeners.add(this.metaHLogRoller);
1472     return listeners;
1473   }
1474 
1475   protected LogRoller getLogRoller() {
1476     return hlogRoller;
1477   }
1478 
1479   public MetricsRegionServer getMetrics() {
1480     return this.metricsRegionServer;
1481   }
1482 
1483   /**
1484    * @return Master address tracker instance.
1485    */
1486   public MasterAddressTracker getMasterAddressManager() {
1487     return this.masterAddressManager;
1488   }
1489 
1490   /*
1491    * Start maintenance Threads, Server, Worker and lease checker threads.
1492    * Install an UncaughtExceptionHandler that calls abort of RegionServer if we
1493    * get an unhandled exception. We cannot set the handler on all threads.
1494    * Server's internal Listener thread is off limits. For Server, if an OOME, it
1495    * waits a while then retries. Meantime, a flush or a compaction that tries to
1496    * run should trigger same critical condition and the shutdown will run. On
1497    * its way out, this server will shut down Server. Leases are sort of
1498    * inbetween. It has an internal thread that while it inherits from Chore, it
1499    * keeps its own internal stop mechanism so needs to be stopped by this
1500    * hosting server. Worker logs the exception and exits.
1501    */
1502   private void startServiceThreads() throws IOException {
1503     String n = Thread.currentThread().getName();
1504     // Start executor services
1505     this.service = new ExecutorService(getServerName().toShortString());
1506     this.service.startExecutorService(ExecutorType.RS_OPEN_REGION,
1507       conf.getInt("hbase.regionserver.executor.openregion.threads", 3));
1508     this.service.startExecutorService(ExecutorType.RS_OPEN_META,
1509       conf.getInt("hbase.regionserver.executor.openmeta.threads", 1));
1510     this.service.startExecutorService(ExecutorType.RS_CLOSE_REGION,
1511       conf.getInt("hbase.regionserver.executor.closeregion.threads", 3));
1512     this.service.startExecutorService(ExecutorType.RS_CLOSE_META,
1513       conf.getInt("hbase.regionserver.executor.closemeta.threads", 1));
1514     if (conf.getBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, false)) {
1515       this.service.startExecutorService(ExecutorType.RS_PARALLEL_SEEK,
1516         conf.getInt("hbase.storescanner.parallel.seek.threads", 10));
1517     }
1518     this.service.startExecutorService(ExecutorType.RS_LOG_REPLAY_OPS,
1519       conf.getInt("hbase.regionserver.wal.max.splitters", SplitLogWorker.DEFAULT_MAX_SPLITTERS));
1520 
1521     Threads.setDaemonThreadRunning(this.hlogRoller.getThread(), n + ".logRoller",
1522         uncaughtExceptionHandler);
1523     this.cacheFlusher.start(uncaughtExceptionHandler);
1524     Threads.setDaemonThreadRunning(this.compactionChecker.getThread(), n +
1525       ".compactionChecker", uncaughtExceptionHandler);
1526     Threads.setDaemonThreadRunning(this.periodicFlusher.getThread(), n +
1527         ".periodicFlusher", uncaughtExceptionHandler);
1528     if (this.healthCheckChore != null) {
1529     Threads
1530         .setDaemonThreadRunning(this.healthCheckChore.getThread(), n + ".healthChecker",
1531             uncaughtExceptionHandler);
1532     }
1533 
1534     // Leases is not a Thread. Internally it runs a daemon thread. If it gets
1535     // an unhandled exception, it will just exit.
1536     this.leases.setName(n + ".leaseChecker");
1537     this.leases.start();
1538 
1539     // Put up the webui.  Webui may come up on port other than configured if
1540     // that port is occupied. Adjust serverInfo if this is the case.
1541     this.webuiport = putUpWebUI();
1542 
1543     if (this.replicationSourceHandler == this.replicationSinkHandler &&
1544         this.replicationSourceHandler != null) {
1545       this.replicationSourceHandler.startReplicationService();
1546     } else {
1547       if (this.replicationSourceHandler != null) {
1548         this.replicationSourceHandler.startReplicationService();
1549       }
1550       if (this.replicationSinkHandler != null) {
1551         this.replicationSinkHandler.startReplicationService();
1552       }
1553     }
1554 
1555     // Start Server.  This service is like leases in that it internally runs
1556     // a thread.
1557     this.rpcServer.start();
1558 
1559     // Create the log splitting worker and start it
1560     // set a smaller retries to fast fail otherwise splitlogworker could be blocked for
1561     // quite a while inside HConnection layer. The worker won't be available for other
1562     // tasks even after current task is preempted after a split task times out.
1563     Configuration sinkConf = HBaseConfiguration.create(conf);
1564     sinkConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
1565       conf.getInt("hbase.log.replay.retries.number", 8)); // 8 retries take about 23 seconds
1566     sinkConf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
1567       conf.getInt("hbase.log.replay.rpc.timeout", 30000)); // default 30 seconds
1568     sinkConf.setInt("hbase.client.serverside.retries.multiplier", 1);
1569     this.splitLogWorker = new SplitLogWorker(this.zooKeeper, sinkConf, this, this);
1570     splitLogWorker.start();
1571   }
1572 
1573   /**
1574    * Puts up the webui.
1575    * @return Returns final port -- maybe different from what we started with.
1576    * @throws IOException
1577    */
1578   private int putUpWebUI() throws IOException {
1579     int port = this.conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 60030);
1580     // -1 is for disabling info server
1581     if (port < 0) return port;
1582     String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0");
1583     // check if auto port bind enabled
1584     boolean auto = this.conf.getBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,
1585         false);
1586     while (true) {
1587       try {
1588         this.infoServer = new InfoServer("regionserver", addr, port, false, this.conf);
1589         this.infoServer.addServlet("status", "/rs-status", RSStatusServlet.class);
1590         this.infoServer.addServlet("dump", "/dump", RSDumpServlet.class);
1591         this.infoServer.setAttribute(REGIONSERVER, this);
1592         this.infoServer.setAttribute(REGIONSERVER_CONF, conf);
1593         this.infoServer.start();
1594         break;
1595       } catch (BindException e) {
1596         if (!auto) {
1597           // auto bind disabled throw BindException
1598           LOG.error("Failed binding http info server to port: " + port);
1599           throw e;
1600         }
1601         // auto bind enabled, try to use another port
1602         LOG.info("Failed binding http info server to port: " + port);
1603         port++;
1604       }
1605     }
1606     return port;
1607   }
1608 
1609   /*
1610    * Verify that server is healthy
1611    */
1612   private boolean isHealthy() {
1613     if (!fsOk) {
1614       // File system problem
1615       return false;
1616     }
1617     // Verify that all threads are alive
1618     if (!(leases.isAlive()
1619         && cacheFlusher.isAlive() && hlogRoller.isAlive()
1620         && this.compactionChecker.isAlive())
1621         && this.periodicFlusher.isAlive()) {
1622       stop("One or more threads are no longer alive -- stop");
1623       return false;
1624     }
1625     if (metaHLogRoller != null && !metaHLogRoller.isAlive()) {
1626       stop("Meta HLog roller thread is no longer alive -- stop");
1627       return false;
1628     }
1629     return true;
1630   }
1631 
1632   public HLog getWAL() {
1633     try {
1634       return getWAL(null);
1635     } catch (IOException e) {
1636       LOG.warn("getWAL threw exception " + e);
1637       return null;
1638     }
1639   }
1640 
1641   @Override
1642   public HLog getWAL(HRegionInfo regionInfo) throws IOException {
1643     //TODO: at some point this should delegate to the HLogFactory
1644     //currently, we don't care about the region as much as we care about the
1645     //table.. (hence checking the tablename below)
1646     //_ROOT_ and hbase:meta regions have separate WAL.
1647     if (regionInfo != null && regionInfo.isMetaTable()) {
1648       return getMetaWAL();
1649     }
1650     return this.hlog;
1651   }
1652 
1653   @Override
1654   public CatalogTracker getCatalogTracker() {
1655     return this.catalogTracker;
1656   }
1657 
1658   @Override
1659   public void stop(final String msg) {
1660     try {
1661       this.rsHost.preStop(msg);
1662       this.stopped = true;
1663       LOG.info("STOPPED: " + msg);
1664       // Wakes run() if it is sleeping
1665       sleeper.skipSleepCycle();
1666     } catch (IOException exp) {
1667       LOG.warn("The region server did not stop", exp);
1668     }
1669   }
1670 
1671   public void waitForServerOnline(){
1672     while (!isOnline() && !isStopped()){
1673        sleeper.sleep();
1674     }
1675   }
1676 
1677   @Override
1678   public void postOpenDeployTasks(final HRegion r, final CatalogTracker ct)
1679   throws KeeperException, IOException {
1680     checkOpen();
1681     LOG.info("Post open deploy tasks for region=" + r.getRegionNameAsString());
1682     // Do checks to see if we need to compact (references or too many files)
1683     for (Store s : r.getStores().values()) {
1684       if (s.hasReferences() || s.needsCompaction()) {
1685        this.compactSplitThread.requestSystemCompaction(r, s, "Opening Region");
1686       }
1687     }
1688     long openSeqNum = r.getOpenSeqNum();
1689     if (openSeqNum == HConstants.NO_SEQNUM) {
1690       // If we opened a region, we should have read some sequence number from it.
1691       LOG.error("No sequence number found when opening " + r.getRegionNameAsString());
1692       openSeqNum = 0;
1693     }
1694 
1695     // Update flushed sequence id of a recovering region in ZK
1696     updateRecoveringRegionLastFlushedSequenceId(r);
1697 
1698     // Update ZK, or META
1699     if (r.getRegionInfo().isMetaRegion()) {
1700       MetaRegionTracker.setMetaLocation(getZooKeeper(),
1701           this.serverNameFromMasterPOV);
1702     } else {
1703       MetaEditor.updateRegionLocation(ct, r.getRegionInfo(),
1704         this.serverNameFromMasterPOV, openSeqNum);
1705     }
1706     LOG.info("Finished post open deploy task for " + r.getRegionNameAsString());
1707 
1708   }
1709 
1710   @Override
1711   public RpcServerInterface getRpcServer() {
1712     return rpcServer;
1713   }
1714 
1715   /**
1716    * Cause the server to exit without closing the regions it is serving, the log
1717    * it is using and without notifying the master. Used unit testing and on
1718    * catastrophic events such as HDFS is yanked out from under hbase or we OOME.
1719    *
1720    * @param reason
1721    *          the reason we are aborting
1722    * @param cause
1723    *          the exception that caused the abort, or null
1724    */
1725   @Override
1726   public void abort(String reason, Throwable cause) {
1727     String msg = "ABORTING region server " + this + ": " + reason;
1728     if (cause != null) {
1729       LOG.fatal(msg, cause);
1730     } else {
1731       LOG.fatal(msg);
1732     }
1733     this.abortRequested = true;
1734     // HBASE-4014: show list of coprocessors that were loaded to help debug
1735     // regionserver crashes.Note that we're implicitly using
1736     // java.util.HashSet's toString() method to print the coprocessor names.
1737     LOG.fatal("RegionServer abort: loaded coprocessors are: " +
1738         CoprocessorHost.getLoadedCoprocessors());
1739     // Do our best to report our abort to the master, but this may not work
1740     try {
1741       if (cause != null) {
1742         msg += "\nCause:\n" + StringUtils.stringifyException(cause);
1743       }
1744       // Report to the master but only if we have already registered with the master.
1745       if (rssStub != null && this.serverNameFromMasterPOV != null) {
1746         ReportRSFatalErrorRequest.Builder builder =
1747           ReportRSFatalErrorRequest.newBuilder();
1748         ServerName sn =
1749           ServerName.parseVersionedServerName(this.serverNameFromMasterPOV.getVersionedBytes());
1750         builder.setServer(ProtobufUtil.toServerName(sn));
1751         builder.setErrorMessage(msg);
1752         rssStub.reportRSFatalError(null, builder.build());
1753       }
1754     } catch (Throwable t) {
1755       LOG.warn("Unable to report fatal error to master", t);
1756     }
1757     stop(reason);
1758   }
1759 
1760   /**
1761    * @see HRegionServer#abort(String, Throwable)
1762    */
1763   public void abort(String reason) {
1764     abort(reason, null);
1765   }
1766 
1767   @Override
1768   public boolean isAborted() {
1769     return this.abortRequested;
1770   }
1771 
1772   /*
1773    * Simulate a kill -9 of this server. Exits w/o closing regions or cleaninup
1774    * logs but it does close socket in case want to bring up server on old
1775    * hostname+port immediately.
1776    */
1777   protected void kill() {
1778     this.killed = true;
1779     abort("Simulated kill");
1780   }
1781 
1782   /**
1783    * Wait on all threads to finish. Presumption is that all closes and stops
1784    * have already been called.
1785    */
1786   protected void join() {
1787     Threads.shutdown(this.compactionChecker.getThread());
1788     Threads.shutdown(this.periodicFlusher.getThread());
1789     this.cacheFlusher.join();
1790     if (this.healthCheckChore != null) {
1791       Threads.shutdown(this.healthCheckChore.getThread());
1792     }
1793     if (this.spanReceiverHost != null) {
1794       this.spanReceiverHost.closeReceivers();
1795     }
1796     if (this.hlogRoller != null) {
1797       Threads.shutdown(this.hlogRoller.getThread());
1798     }
1799     if (this.metaHLogRoller != null) {
1800       Threads.shutdown(this.metaHLogRoller.getThread());
1801     }
1802     if (this.compactSplitThread != null) {
1803       this.compactSplitThread.join();
1804     }
1805     if (this.service != null) this.service.shutdown();
1806     if (this.replicationSourceHandler != null &&
1807         this.replicationSourceHandler == this.replicationSinkHandler) {
1808       this.replicationSourceHandler.stopReplicationService();
1809     } else {
1810       if (this.replicationSourceHandler != null) {
1811         this.replicationSourceHandler.stopReplicationService();
1812       }
1813       if (this.replicationSinkHandler != null) {
1814         this.replicationSinkHandler.stopReplicationService();
1815       }
1816     }
1817   }
1818 
1819   /**
1820    * @return Return the object that implements the replication
1821    * source service.
1822    */
1823   ReplicationSourceService getReplicationSourceService() {
1824     return replicationSourceHandler;
1825   }
1826 
1827   /**
1828    * @return Return the object that implements the replication
1829    * sink service.
1830    */
1831   ReplicationSinkService getReplicationSinkService() {
1832     return replicationSinkHandler;
1833   }
1834 
1835   /**
1836    * Get the current master from ZooKeeper and open the RPC connection to it.
1837    *
1838    * Method will block until a master is available. You can break from this
1839    * block by requesting the server stop.
1840    *
1841    * @return master + port, or null if server has been stopped
1842    */
1843   private Pair<ServerName, RegionServerStatusService.BlockingInterface>
1844   createRegionServerStatusStub() {
1845     ServerName sn = null;
1846     long previousLogTime = 0;
1847     RegionServerStatusService.BlockingInterface master = null;
1848     boolean refresh = false; // for the first time, use cached data
1849     RegionServerStatusService.BlockingInterface intf = null;
1850     while (keepLooping() && master == null) {
1851       sn = this.masterAddressManager.getMasterAddress(refresh);
1852       if (sn == null) {
1853         if (!keepLooping()) {
1854           // give up with no connection.
1855           LOG.debug("No master found and cluster is stopped; bailing out");
1856           return null;
1857         }
1858         LOG.debug("No master found; retry");
1859         previousLogTime = System.currentTimeMillis();
1860         refresh = true; // let's try pull it from ZK directly
1861         sleeper.sleep();
1862         continue;
1863       }
1864 
1865       new InetSocketAddress(sn.getHostname(), sn.getPort());
1866       try {
1867         BlockingRpcChannel channel =
1868             this.rpcClient.createBlockingRpcChannel(sn, userProvider.getCurrent(), this.rpcTimeout);
1869         intf = RegionServerStatusService.newBlockingStub(channel);
1870         break;
1871       } catch (IOException e) {
1872         e = e instanceof RemoteException ?
1873             ((RemoteException)e).unwrapRemoteException() : e;
1874         if (e instanceof ServerNotRunningYetException) {
1875           if (System.currentTimeMillis() > (previousLogTime+1000)){
1876             LOG.info("Master isn't available yet, retrying");
1877             previousLogTime = System.currentTimeMillis();
1878           }
1879         } else {
1880           if (System.currentTimeMillis() > (previousLogTime + 1000)) {
1881             LOG.warn("Unable to connect to master. Retrying. Error was:", e);
1882             previousLogTime = System.currentTimeMillis();
1883           }
1884         }
1885         try {
1886           Thread.sleep(200);
1887         } catch (InterruptedException ignored) {
1888         }
1889       }
1890     }
1891     return new Pair<ServerName, RegionServerStatusService.BlockingInterface>(sn, intf);
1892   }
1893 
1894   /**
1895    * @return True if we should break loop because cluster is going down or
1896    * this server has been stopped or hdfs has gone bad.
1897    */
1898   private boolean keepLooping() {
1899     return !this.stopped && isClusterUp();
1900   }
1901 
1902   /*
1903    * Let the master know we're here Run initialization using parameters passed
1904    * us by the master.
1905    * @return A Map of key/value configurations we got from the Master else
1906    * null if we failed to register.
1907    * @throws IOException
1908    */
1909   private RegionServerStartupResponse reportForDuty() throws IOException {
1910     RegionServerStartupResponse result = null;
1911     Pair<ServerName, RegionServerStatusService.BlockingInterface> p =
1912       createRegionServerStatusStub();
1913     this.rssStub = p.getSecond();
1914     ServerName masterServerName = p.getFirst();
1915     if (masterServerName == null) return result;
1916     try {
1917       this.requestCount.set(0);
1918       LOG.info("reportForDuty to master=" + masterServerName + " with port=" + this.isa.getPort() +
1919         ", startcode=" + this.startcode);
1920       long now = EnvironmentEdgeManager.currentTimeMillis();
1921       int port = this.isa.getPort();
1922       RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder();
1923       request.setPort(port);
1924       request.setServerStartCode(this.startcode);
1925       request.setServerCurrentTime(now);
1926       result = this.rssStub.regionServerStartup(null, request.build());
1927     } catch (ServiceException se) {
1928       IOException ioe = ProtobufUtil.getRemoteException(se);
1929       if (ioe instanceof ClockOutOfSyncException) {
1930         LOG.fatal("Master rejected startup because clock is out of sync", ioe);
1931         // Re-throw IOE will cause RS to abort
1932         throw ioe;
1933       } else if (ioe instanceof ServerNotRunningYetException) {
1934         LOG.debug("Master is not running yet");
1935       } else {
1936         LOG.warn("error telling master we are up", se);
1937       }
1938     }
1939     return result;
1940   }
1941 
1942   @Override
1943   public long getLastSequenceId(byte[] region) {
1944     Long lastFlushedSequenceId = -1l;
1945     try {
1946       GetLastFlushedSequenceIdRequest req = RequestConverter
1947           .buildGetLastFlushedSequenceIdRequest(region);
1948       lastFlushedSequenceId = rssStub.getLastFlushedSequenceId(null, req)
1949           .getLastFlushedSequenceId();
1950     } catch (ServiceException e) {
1951       lastFlushedSequenceId = -1l;
1952       LOG.warn("Unable to connect to the master to check " + "the last flushed sequence id", e);
1953     }
1954     return lastFlushedSequenceId;
1955   }
1956 
1957   /**
1958    * Closes all regions.  Called on our way out.
1959    * Assumes that its not possible for new regions to be added to onlineRegions
1960    * while this method runs.
1961    */
1962   protected void closeAllRegions(final boolean abort) {
1963     closeUserRegions(abort);
1964     closeMetaTableRegions(abort);
1965   }
1966 
1967   /**
1968    * Close meta region if we carry it
1969    * @param abort Whether we're running an abort.
1970    */
1971   void closeMetaTableRegions(final boolean abort) {
1972     HRegion meta = null;
1973     this.lock.writeLock().lock();
1974     try {
1975       for (Map.Entry<String, HRegion> e: onlineRegions.entrySet()) {
1976         HRegionInfo hri = e.getValue().getRegionInfo();
1977         if (hri.isMetaRegion()) {
1978           meta = e.getValue();
1979         }
1980         if (meta != null) break;
1981       }
1982     } finally {
1983       this.lock.writeLock().unlock();
1984     }
1985     if (meta != null) closeRegionIgnoreErrors(meta.getRegionInfo(), abort);
1986   }
1987 
1988   /**
1989    * Schedule closes on all user regions.
1990    * Should be safe calling multiple times because it wont' close regions
1991    * that are already closed or that are closing.
1992    * @param abort Whether we're running an abort.
1993    */
1994   void closeUserRegions(final boolean abort) {
1995     this.lock.writeLock().lock();
1996     try {
1997       for (Map.Entry<String, HRegion> e: this.onlineRegions.entrySet()) {
1998         HRegion r = e.getValue();
1999         if (!r.getRegionInfo().isMetaTable() && r.isAvailable()) {
2000           // Don't update zk with this close transition; pass false.
2001           closeRegionIgnoreErrors(r.getRegionInfo(), abort);
2002         }
2003       }
2004     } finally {
2005       this.lock.writeLock().unlock();
2006     }
2007   }
2008 
2009   /** @return the info server */
2010   public InfoServer getInfoServer() {
2011     return infoServer;
2012   }
2013 
2014   /**
2015    * @return true if a stop has been requested.
2016    */
2017   @Override
2018   public boolean isStopped() {
2019     return this.stopped;
2020   }
2021 
2022   @Override
2023   public boolean isStopping() {
2024     return this.stopping;
2025   }
2026 
2027   @Override
2028   public Map<String, HRegion> getRecoveringRegions() {
2029     return this.recoveringRegions;
2030   }
2031 
2032   /**
2033    *
2034    * @return the configuration
2035    */
2036   @Override
2037   public Configuration getConfiguration() {
2038     return conf;
2039   }
2040 
2041   /** @return the write lock for the server */
2042   ReentrantReadWriteLock.WriteLock getWriteLock() {
2043     return lock.writeLock();
2044   }
2045 
2046   public int getNumberOfOnlineRegions() {
2047     return this.onlineRegions.size();
2048   }
2049 
2050   boolean isOnlineRegionsEmpty() {
2051     return this.onlineRegions.isEmpty();
2052   }
2053 
2054   /**
2055    * For tests, web ui and metrics.
2056    * This method will only work if HRegionServer is in the same JVM as client;
2057    * HRegion cannot be serialized to cross an rpc.
2058    */
2059   public Collection<HRegion> getOnlineRegionsLocalContext() {
2060     Collection<HRegion> regions = this.onlineRegions.values();
2061     return Collections.unmodifiableCollection(regions);
2062   }
2063 
2064   @Override
2065   public void addToOnlineRegions(HRegion region) {
2066     this.onlineRegions.put(region.getRegionInfo().getEncodedName(), region);
2067   }
2068 
2069   /**
2070    * @return A new Map of online regions sorted by region size with the first entry being the
2071    * biggest.  If two regions are the same size, then the last one found wins; i.e. this method
2072    * may NOT return all regions.
2073    */
2074   SortedMap<Long, HRegion> getCopyOfOnlineRegionsSortedBySize() {
2075     // we'll sort the regions in reverse
2076     SortedMap<Long, HRegion> sortedRegions = new TreeMap<Long, HRegion>(
2077         new Comparator<Long>() {
2078           @Override
2079           public int compare(Long a, Long b) {
2080             return -1 * a.compareTo(b);
2081           }
2082         });
2083     // Copy over all regions. Regions are sorted by size with biggest first.
2084     for (HRegion region : this.onlineRegions.values()) {
2085       sortedRegions.put(region.memstoreSize.get(), region);
2086     }
2087     return sortedRegions;
2088   }
2089 
2090   /**
2091    * @return time stamp in millis of when this region server was started
2092    */
2093   public long getStartcode() {
2094     return this.startcode;
2095   }
2096 
2097   /** @return reference to FlushRequester */
2098   @Override
2099   public FlushRequester getFlushRequester() {
2100     return this.cacheFlusher;
2101   }
2102 
2103   /**
2104    * Get the top N most loaded regions this server is serving so we can tell the
2105    * master which regions it can reallocate if we're overloaded. TODO: actually
2106    * calculate which regions are most loaded. (Right now, we're just grabbing
2107    * the first N regions being served regardless of load.)
2108    */
2109   protected HRegionInfo[] getMostLoadedRegions() {
2110     ArrayList<HRegionInfo> regions = new ArrayList<HRegionInfo>();
2111     for (HRegion r : onlineRegions.values()) {
2112       if (!r.isAvailable()) {
2113         continue;
2114       }
2115       if (regions.size() < numRegionsToReport) {
2116         regions.add(r.getRegionInfo());
2117       } else {
2118         break;
2119       }
2120     }
2121     return regions.toArray(new HRegionInfo[regions.size()]);
2122   }
2123 
2124   @Override
2125   public Leases getLeases() {
2126     return leases;
2127   }
2128 
2129   /**
2130    * @return Return the rootDir.
2131    */
2132   protected Path getRootDir() {
2133     return rootDir;
2134   }
2135 
2136   /**
2137    * @return Return the fs.
2138    */
2139   @Override
2140   public FileSystem getFileSystem() {
2141     return fs;
2142   }
2143 
2144   @Override
2145   public String toString() {
2146     return getServerName().toString();
2147   }
2148 
2149   /**
2150    * Interval at which threads should run
2151    *
2152    * @return the interval
2153    */
2154   public int getThreadWakeFrequency() {
2155     return threadWakeFrequency;
2156   }
2157 
2158   @Override
2159   public ZooKeeperWatcher getZooKeeper() {
2160     return zooKeeper;
2161   }
2162 
2163   @Override
2164   public ServerName getServerName() {
2165     // Our servername could change after we talk to the master.
2166     return this.serverNameFromMasterPOV == null?
2167         ServerName.valueOf(this.isa.getHostName(), this.isa.getPort(), this.startcode) :
2168         this.serverNameFromMasterPOV;
2169   }
2170 
2171   @Override
2172   public CompactionRequestor getCompactionRequester() {
2173     return this.compactSplitThread;
2174   }
2175 
2176   public ZooKeeperWatcher getZooKeeperWatcher() {
2177     return this.zooKeeper;
2178   }
2179 
2180   public RegionServerCoprocessorHost getCoprocessorHost(){
2181     return this.rsHost;
2182   }
2183 
2184   @Override
2185   public ConcurrentMap<byte[], Boolean> getRegionsInTransitionInRS() {
2186     return this.regionsInTransitionInRS;
2187   }
2188 
2189   @Override
2190   public ExecutorService getExecutorService() {
2191     return service;
2192   }
2193 
2194   //
2195   // Main program and support routines
2196   //
2197 
2198   /**
2199    * Load the replication service objects, if any
2200    */
2201   static private void createNewReplicationInstance(Configuration conf,
2202     HRegionServer server, FileSystem fs, Path logDir, Path oldLogDir) throws IOException{
2203 
2204     // If replication is not enabled, then return immediately.
2205     if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
2206         HConstants.REPLICATION_ENABLE_DEFAULT)) {
2207       return;
2208     }
2209 
2210     // read in the name of the source replication class from the config file.
2211     String sourceClassname = conf.get(HConstants.REPLICATION_SOURCE_SERVICE_CLASSNAME,
2212                                HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2213 
2214     // read in the name of the sink replication class from the config file.
2215     String sinkClassname = conf.get(HConstants.REPLICATION_SINK_SERVICE_CLASSNAME,
2216                              HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);
2217 
2218     // If both the sink and the source class names are the same, then instantiate
2219     // only one object.
2220     if (sourceClassname.equals(sinkClassname)) {
2221       server.replicationSourceHandler = (ReplicationSourceService)
2222                                          newReplicationInstance(sourceClassname,
2223                                          conf, server, fs, logDir, oldLogDir);
2224       server.replicationSinkHandler = (ReplicationSinkService)
2225                                          server.replicationSourceHandler;
2226     } else {
2227       server.replicationSourceHandler = (ReplicationSourceService)
2228                                          newReplicationInstance(sourceClassname,
2229                                          conf, server, fs, logDir, oldLogDir);
2230       server.replicationSinkHandler = (ReplicationSinkService)
2231                                          newReplicationInstance(sinkClassname,
2232                                          conf, server, fs, logDir, oldLogDir);
2233     }
2234   }
2235 
2236   static private ReplicationService newReplicationInstance(String classname,
2237     Configuration conf, HRegionServer server, FileSystem fs, Path logDir,
2238     Path oldLogDir) throws IOException{
2239 
2240     Class<?> clazz = null;
2241     try {
2242       ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
2243       clazz = Class.forName(classname, true, classLoader);
2244     } catch (java.lang.ClassNotFoundException nfe) {
2245       throw new IOException("Could not find class for " + classname);
2246     }
2247 
2248     // create an instance of the replication object.
2249     ReplicationService service = (ReplicationService)
2250                               ReflectionUtils.newInstance(clazz, conf);
2251     service.initialize(server, fs, logDir, oldLogDir);
2252     return service;
2253   }
2254 
2255   /**
2256    * @param hrs
2257    * @return Thread the RegionServer is running in correctly named.
2258    * @throws IOException
2259    */
2260   public static Thread startRegionServer(final HRegionServer hrs)
2261       throws IOException {
2262     return startRegionServer(hrs, "regionserver" + hrs.isa.getPort());
2263   }
2264 
2265   /**
2266    * @param hrs
2267    * @param name
2268    * @return Thread the RegionServer is running in correctly named.
2269    * @throws IOException
2270    */
2271   public static Thread startRegionServer(final HRegionServer hrs,
2272       final String name) throws IOException {
2273     Thread t = new Thread(hrs);
2274     t.setName(name);
2275     t.start();
2276     // Install shutdown hook that will catch signals and run an orderly shutdown
2277     // of the hrs.
2278     ShutdownHook.install(hrs.getConfiguration(), FileSystem.get(hrs
2279         .getConfiguration()), hrs, t);
2280     return t;
2281   }
2282 
2283   /**
2284    * Utility for constructing an instance of the passed HRegionServer class.
2285    *
2286    * @param regionServerClass
2287    * @param conf2
2288    * @return HRegionServer instance.
2289    */
2290   public static HRegionServer constructRegionServer(
2291       Class<? extends HRegionServer> regionServerClass,
2292       final Configuration conf2) {
2293     try {
2294       Constructor<? extends HRegionServer> c = regionServerClass
2295           .getConstructor(Configuration.class);
2296       return c.newInstance(conf2);
2297     } catch (Exception e) {
2298       throw new RuntimeException("Failed construction of " + "Regionserver: "
2299           + regionServerClass.toString(), e);
2300     }
2301   }
2302 
2303   /**
2304    * @see org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine
2305    */
2306   public static void main(String[] args) throws Exception {
2307 	VersionInfo.logVersion();
2308     Configuration conf = HBaseConfiguration.create();
2309     @SuppressWarnings("unchecked")
2310     Class<? extends HRegionServer> regionServerClass = (Class<? extends HRegionServer>) conf
2311         .getClass(HConstants.REGION_SERVER_IMPL, HRegionServer.class);
2312 
2313     new HRegionServerCommandLine(regionServerClass).doMain(args);
2314   }
2315 
2316   /**
2317    * Gets the online regions of the specified table.
2318    * This method looks at the in-memory onlineRegions.  It does not go to <code>hbase:meta</code>.
2319    * Only returns <em>online</em> regions.  If a region on this table has been
2320    * closed during a disable, etc., it will not be included in the returned list.
2321    * So, the returned list may not necessarily be ALL regions in this table, its
2322    * all the ONLINE regions in the table.
2323    * @param tableName
2324    * @return Online regions from <code>tableName</code>
2325    */
2326   @Override
2327   public List<HRegion> getOnlineRegions(TableName tableName) {
2328      List<HRegion> tableRegions = new ArrayList<HRegion>();
2329      synchronized (this.onlineRegions) {
2330        for (HRegion region: this.onlineRegions.values()) {
2331          HRegionInfo regionInfo = region.getRegionInfo();
2332          if(regionInfo.getTable().equals(tableName)) {
2333            tableRegions.add(region);
2334          }
2335        }
2336      }
2337      return tableRegions;
2338    }
2339 
2340   // used by org/apache/hbase/tmpl/regionserver/RSStatusTmpl.jamon (HBASE-4070).
2341   public String[] getCoprocessors() {
2342     TreeSet<String> coprocessors = new TreeSet<String>(
2343         this.hlog.getCoprocessorHost().getCoprocessors());
2344     Collection<HRegion> regions = getOnlineRegionsLocalContext();
2345     for (HRegion region: regions) {
2346       coprocessors.addAll(region.getCoprocessorHost().getCoprocessors());
2347     }
2348     return coprocessors.toArray(new String[coprocessors.size()]);
2349   }
2350 
2351   /**
2352    * Instantiated as a scanner lease. If the lease times out, the scanner is
2353    * closed
2354    */
2355   private class ScannerListener implements LeaseListener {
2356     private final String scannerName;
2357 
2358     ScannerListener(final String n) {
2359       this.scannerName = n;
2360     }
2361 
2362     @Override
2363     public void leaseExpired() {
2364       RegionScannerHolder rsh = scanners.remove(this.scannerName);
2365       if (rsh != null) {
2366         RegionScanner s = rsh.s;
2367         LOG.info("Scanner " + this.scannerName + " lease expired on region "
2368             + s.getRegionInfo().getRegionNameAsString());
2369         try {
2370           HRegion region = getRegion(s.getRegionInfo().getRegionName());
2371           if (region != null && region.getCoprocessorHost() != null) {
2372             region.getCoprocessorHost().preScannerClose(s);
2373           }
2374 
2375           s.close();
2376           if (region != null && region.getCoprocessorHost() != null) {
2377             region.getCoprocessorHost().postScannerClose(s);
2378           }
2379         } catch (IOException e) {
2380           LOG.error("Closing scanner for "
2381               + s.getRegionInfo().getRegionNameAsString(), e);
2382         }
2383       } else {
2384         LOG.info("Scanner " + this.scannerName + " lease expired");
2385       }
2386     }
2387   }
2388 
2389   /**
2390    * Called to verify that this server is up and running.
2391    *
2392    * @throws IOException
2393    */
2394   protected void checkOpen() throws IOException {
2395     if (this.stopped || this.abortRequested) {
2396       throw new RegionServerStoppedException("Server " + getServerName() +
2397         " not running" + (this.abortRequested ? ", aborting" : ""));
2398     }
2399     if (!fsOk) {
2400       throw new RegionServerStoppedException("File system not available");
2401     }
2402   }
2403 
2404 
2405   /**
2406    * Try to close the region, logs a warning on failure but continues.
2407    * @param region Region to close
2408    */
2409   private void closeRegionIgnoreErrors(HRegionInfo region, final boolean abort) {
2410     try {
2411       if (!closeRegion(region.getEncodedName(), abort, false, -1, null)) {
2412         LOG.warn("Failed to close " + region.getRegionNameAsString() +
2413             " - ignoring and continuing");
2414       }
2415     } catch (IOException e) {
2416       LOG.warn("Failed to close " + region.getRegionNameAsString() +
2417           " - ignoring and continuing", e);
2418     }
2419   }
2420 
2421   /**
2422    * Close asynchronously a region, can be called from the master or internally by the regionserver
2423    * when stopping. If called from the master, the region will update the znode status.
2424    *
2425    * <p>
2426    * If an opening was in progress, this method will cancel it, but will not start a new close. The
2427    * coprocessors are not called in this case. A NotServingRegionException exception is thrown.
2428    * </p>
2429 
2430    * <p>
2431    *   If a close was in progress, this new request will be ignored, and an exception thrown.
2432    * </p>
2433    *
2434    * @param encodedName Region to close
2435    * @param abort True if we are aborting
2436    * @param zk True if we are to update zk about the region close; if the close
2437    * was orchestrated by master, then update zk.  If the close is being run by
2438    * the regionserver because its going down, don't update zk.
2439    * @param versionOfClosingNode the version of znode to compare when RS transitions the znode from
2440    *   CLOSING state.
2441    * @return True if closed a region.
2442    * @throws NotServingRegionException if the region is not online
2443    * @throws RegionAlreadyInTransitionException if the region is already closing
2444    */
2445   protected boolean closeRegion(String encodedName, final boolean abort,
2446       final boolean zk, final int versionOfClosingNode, final ServerName sn)
2447       throws NotServingRegionException, RegionAlreadyInTransitionException {
2448     //Check for permissions to close.
2449     HRegion actualRegion = this.getFromOnlineRegions(encodedName);
2450     if ((actualRegion != null) && (actualRegion.getCoprocessorHost() != null)) {
2451       try {
2452         actualRegion.getCoprocessorHost().preClose(false);
2453       } catch (IOException exp) {
2454         LOG.warn("Unable to close region: the coprocessor launched an error ", exp);
2455         return false;
2456       }
2457     }
2458 
2459     final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(encodedName.getBytes(),
2460         Boolean.FALSE);
2461 
2462     if (Boolean.TRUE.equals(previous)) {
2463       LOG.info("Received CLOSE for the region:" + encodedName + " , which we are already " +
2464           "trying to OPEN. Cancelling OPENING.");
2465       if (!regionsInTransitionInRS.replace(encodedName.getBytes(), previous, Boolean.FALSE)){
2466         // The replace failed. That should be an exceptional case, but theoretically it can happen.
2467         // We're going to try to do a standard close then.
2468         LOG.warn("The opening for region " + encodedName + " was done before we could cancel it." +
2469             " Doing a standard close now");
2470         return closeRegion(encodedName, abort, zk, versionOfClosingNode, sn);
2471       }
2472       // Let's get the region from the online region list again
2473       actualRegion = this.getFromOnlineRegions(encodedName);
2474       if (actualRegion == null) { // If already online, we still need to close it.
2475         LOG.info("The opening previously in progress has been cancelled by a CLOSE request.");
2476         // The master deletes the znode when it receives this exception.
2477         throw new NotServingRegionException("The region " + encodedName +
2478           " was opening but not yet served. Opening is cancelled.");
2479       }
2480     } else if (Boolean.FALSE.equals(previous)) {
2481       LOG.info("Received CLOSE for the region: " + encodedName +
2482         " ,which we are already trying to CLOSE, but not completed yet");
2483       // The master will retry till the region is closed. We need to do this since
2484       // the region could fail to close somehow. If we mark the region closed in master
2485       // while it is not, there could be data loss.
2486       // If the region stuck in closing for a while, and master runs out of retries,
2487       // master will move the region to failed_to_close. Later on, if the region
2488       // is indeed closed, master can properly re-assign it.
2489       throw new RegionAlreadyInTransitionException("The region " + encodedName +
2490         " was already closing. New CLOSE request is ignored.");
2491     }
2492 
2493     if (actualRegion == null) {
2494       LOG.error("Received CLOSE for a region which is not online, and we're not opening.");
2495       this.regionsInTransitionInRS.remove(encodedName.getBytes());
2496       // The master deletes the znode when it receives this exception.
2497       throw new NotServingRegionException("The region " + encodedName +
2498           " is not online, and is not opening.");
2499     }
2500 
2501     CloseRegionHandler crh;
2502     final HRegionInfo hri = actualRegion.getRegionInfo();
2503     if (hri.isMetaRegion()) {
2504       crh = new CloseMetaHandler(this, this, hri, abort, zk, versionOfClosingNode);
2505     } else {
2506       crh = new CloseRegionHandler(this, this, hri, abort, zk, versionOfClosingNode, sn);
2507     }
2508     this.service.submit(crh);
2509     return true;
2510   }
2511 
2512    /**
2513    * @param regionName
2514    * @return HRegion for the passed binary <code>regionName</code> or null if
2515    *         named region is not member of the online regions.
2516    */
2517   public HRegion getOnlineRegion(final byte[] regionName) {
2518     String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
2519     return this.onlineRegions.get(encodedRegionName);
2520   }
2521 
2522   public InetSocketAddress[] getRegionBlockLocations(final String encodedRegionName) {
2523     return this.regionFavoredNodesMap.get(encodedRegionName);
2524   }
2525 
2526   @Override
2527   public HRegion getFromOnlineRegions(final String encodedRegionName) {
2528     return this.onlineRegions.get(encodedRegionName);
2529   }
2530 
2531 
2532   @Override
2533   public boolean removeFromOnlineRegions(final HRegion r, ServerName destination) {
2534     HRegion toReturn = this.onlineRegions.remove(r.getRegionInfo().getEncodedName());
2535 
2536     if (destination != null) {
2537       HLog wal = getWAL();
2538       long closeSeqNum = wal.getEarliestMemstoreSeqNum(r.getRegionInfo().getEncodedNameAsBytes());
2539       if (closeSeqNum == HConstants.NO_SEQNUM) {
2540         // No edits in WAL for this region; get the sequence number when the region was opened.
2541         closeSeqNum = r.getOpenSeqNum();
2542         if (closeSeqNum == HConstants.NO_SEQNUM) {
2543           closeSeqNum = 0;
2544         }
2545       }
2546       addToMovedRegions(r.getRegionInfo().getEncodedName(), destination, closeSeqNum);
2547     }
2548     this.regionFavoredNodesMap.remove(r.getRegionInfo().getEncodedName());
2549     return toReturn != null;
2550   }
2551 
2552   /**
2553    * Protected utility method for safely obtaining an HRegion handle.
2554    *
2555    * @param regionName
2556    *          Name of online {@link HRegion} to return
2557    * @return {@link HRegion} for <code>regionName</code>
2558    * @throws NotServingRegionException
2559    */
2560   protected HRegion getRegion(final byte[] regionName)
2561       throws NotServingRegionException {
2562     String encodedRegionName = HRegionInfo.encodeRegionName(regionName);
2563     return getRegionByEncodedName(regionName, encodedRegionName);
2564   }
2565 
2566   protected HRegion getRegionByEncodedName(String encodedRegionName)
2567       throws NotServingRegionException {
2568     return getRegionByEncodedName(null, encodedRegionName);
2569   }
2570 
2571   protected HRegion getRegionByEncodedName(byte[] regionName, String encodedRegionName)
2572     throws NotServingRegionException {
2573     HRegion region = this.onlineRegions.get(encodedRegionName);
2574     if (region == null) {
2575       MovedRegionInfo moveInfo = getMovedRegion(encodedRegionName);
2576       if (moveInfo != null) {
2577         throw new RegionMovedException(moveInfo.getServerName(), moveInfo.getSeqNum());
2578       }
2579       Boolean isOpening = this.regionsInTransitionInRS.get(Bytes.toBytes(encodedRegionName));
2580       String regionNameStr = regionName == null?
2581         encodedRegionName: Bytes.toStringBinary(regionName);
2582       if (isOpening != null && isOpening.booleanValue()) {
2583         throw new RegionOpeningException("Region " + regionNameStr + " is opening");
2584       }
2585       throw new NotServingRegionException("Region " + regionNameStr + " is not online");
2586     }
2587     return region;
2588   }
2589 
2590   /*
2591    * Cleanup after Throwable caught invoking method. Converts <code>t</code> to
2592    * IOE if it isn't already.
2593    *
2594    * @param t Throwable
2595    *
2596    * @return Throwable converted to an IOE; methods can only let out IOEs.
2597    */
2598   protected Throwable cleanup(final Throwable t) {
2599     return cleanup(t, null);
2600   }
2601 
2602   /*
2603    * Cleanup after Throwable caught invoking method. Converts <code>t</code> to
2604    * IOE if it isn't already.
2605    *
2606    * @param t Throwable
2607    *
2608    * @param msg Message to log in error. Can be null.
2609    *
2610    * @return Throwable converted to an IOE; methods can only let out IOEs.
2611    */
2612   protected Throwable cleanup(final Throwable t, final String msg) {
2613     // Don't log as error if NSRE; NSRE is 'normal' operation.
2614     if (t instanceof NotServingRegionException) {
2615       LOG.debug("NotServingRegionException; " + t.getMessage());
2616       return t;
2617     }
2618     if (msg == null) {
2619       LOG.error("", RemoteExceptionHandler.checkThrowable(t));
2620     } else {
2621       LOG.error(msg, RemoteExceptionHandler.checkThrowable(t));
2622     }
2623     if (!checkOOME(t)) {
2624       checkFileSystem();
2625     }
2626     return t;
2627   }
2628 
2629   /*
2630    * @param t
2631    *
2632    * @param msg Message to put in new IOE if passed <code>t</code> is not an IOE
2633    *
2634    * @return Make <code>t</code> an IOE if it isn't already.
2635    */
2636   protected IOException convertThrowableToIOE(final Throwable t, final String msg) {
2637     return (t instanceof IOException ? (IOException) t : msg == null
2638         || msg.length() == 0 ? new IOException(t) : new IOException(msg, t));
2639   }
2640 
2641   /*
2642    * Check if an OOME and, if so, abort immediately to avoid creating more objects.
2643    *
2644    * @param e
2645    *
2646    * @return True if we OOME'd and are aborting.
2647    */
2648   @Override
2649   public boolean checkOOME(final Throwable e) {
2650     boolean stop = false;
2651     try {
2652       if (e instanceof OutOfMemoryError
2653           || (e.getCause() != null && e.getCause() instanceof OutOfMemoryError)
2654           || (e.getMessage() != null && e.getMessage().contains(
2655               "java.lang.OutOfMemoryError"))) {
2656         stop = true;
2657         LOG.fatal(
2658           "Run out of memory; HRegionServer will abort itself immediately", e);
2659       }
2660     } finally {
2661       if (stop) {
2662         Runtime.getRuntime().halt(1);
2663       }
2664     }
2665     return stop;
2666   }
2667 
2668   /**
2669    * Checks to see if the file system is still accessible. If not, sets
2670    * abortRequested and stopRequested
2671    *
2672    * @return false if file system is not available
2673    */
2674   public boolean checkFileSystem() {
2675     if (this.fsOk && this.fs != null) {
2676       try {
2677         FSUtils.checkFileSystemAvailable(this.fs);
2678       } catch (IOException e) {
2679         abort("File System not available", e);
2680         this.fsOk = false;
2681       }
2682     }
2683     return this.fsOk;
2684   }
2685 
2686   protected long addScanner(RegionScanner s, HRegion r) throws LeaseStillHeldException {
2687     long scannerId = -1;
2688     while (true) {
2689       scannerId = Math.abs(rand.nextLong() << 24) ^ startcode;
2690       String scannerName = String.valueOf(scannerId);
2691       RegionScannerHolder existing =
2692         scanners.putIfAbsent(scannerName, new RegionScannerHolder(s, r));
2693       if (existing == null) {
2694         this.leases.createLease(scannerName, this.scannerLeaseTimeoutPeriod,
2695           new ScannerListener(scannerName));
2696         break;
2697       }
2698     }
2699     return scannerId;
2700   }
2701 
2702   /**
2703    * Generate a random positive long number
2704    *
2705    * @return a random positive long number
2706    */
2707   protected long nextLong() {
2708     long n = rand.nextLong();
2709     if (n == 0) {
2710       return nextLong();
2711     }
2712     if (n < 0) {
2713       n = -n;
2714     }
2715     return n;
2716   }
2717 
2718   // Start Client methods
2719 
2720   /**
2721    * Get data from a table.
2722    *
2723    * @param controller the RPC controller
2724    * @param request the get request
2725    * @throws ServiceException
2726    */
2727   @Override
2728   public GetResponse get(final RpcController controller,
2729       final GetRequest request) throws ServiceException {
2730     long before = EnvironmentEdgeManager.currentTimeMillis();
2731     try {
2732       requestCount.increment();
2733       HRegion region = getRegion(request.getRegion());
2734 
2735       GetResponse.Builder builder = GetResponse.newBuilder();
2736       ClientProtos.Get get = request.getGet();
2737       Boolean existence = null;
2738       Result r = null;
2739 
2740       if (get.hasClosestRowBefore() && get.getClosestRowBefore()) {
2741         if (get.getColumnCount() != 1) {
2742           throw new DoNotRetryIOException(
2743             "get ClosestRowBefore supports one and only one family now, not "
2744               + get.getColumnCount() + " families");
2745         }
2746         byte[] row = get.getRow().toByteArray();
2747         byte[] family = get.getColumn(0).getFamily().toByteArray();
2748         r = region.getClosestRowBefore(row, family);
2749       } else {
2750         Get clientGet = ProtobufUtil.toGet(get);
2751         if (get.getExistenceOnly() && region.getCoprocessorHost() != null) {
2752           existence = region.getCoprocessorHost().preExists(clientGet);
2753         }
2754         if (existence == null) {
2755           r = region.get(clientGet);
2756           if (get.getExistenceOnly()) {
2757             boolean exists = r.getExists();
2758             if (region.getCoprocessorHost() != null) {
2759               exists = region.getCoprocessorHost().postExists(clientGet, exists);
2760             }
2761             existence = exists;
2762           }
2763         }
2764       }
2765       if (existence != null){
2766         ClientProtos.Result pbr = ProtobufUtil.toResult(existence);
2767         builder.setResult(pbr);
2768       } else  if (r != null) {
2769         ClientProtos.Result pbr = ProtobufUtil.toResult(r);
2770         builder.setResult(pbr);
2771       }
2772       return builder.build();
2773     } catch (IOException ie) {
2774       throw new ServiceException(ie);
2775     } finally {
2776       metricsRegionServer.updateGet(EnvironmentEdgeManager.currentTimeMillis() - before);
2777     }
2778   }
2779 
2780 
2781   /**
2782    * Mutate data in a table.
2783    *
2784    * @param rpcc the RPC controller
2785    * @param request the mutate request
2786    * @throws ServiceException
2787    */
2788   @Override
2789   public MutateResponse mutate(final RpcController rpcc,
2790       final MutateRequest request) throws ServiceException {
2791     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
2792     // It is also the conduit via which we pass back data.
2793     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
2794     CellScanner cellScanner = controller != null? controller.cellScanner(): null;
2795     // Clear scanner so we are not holding on to reference across call.
2796     controller.setCellScanner(null);
2797     try {
2798       requestCount.increment();
2799       HRegion region = getRegion(request.getRegion());
2800       MutateResponse.Builder builder = MutateResponse.newBuilder();
2801       MutationProto mutation = request.getMutation();
2802       if (!region.getRegionInfo().isMetaTable()) {
2803         cacheFlusher.reclaimMemStoreMemory();
2804       }
2805       Result r = null;
2806       Boolean processed = null;
2807       MutationType type = mutation.getMutateType();
2808       switch (type) {
2809       case APPEND:
2810         r = append(region, mutation, cellScanner);
2811         break;
2812       case INCREMENT:
2813         r = increment(region, mutation, cellScanner);
2814         break;
2815       case PUT:
2816         Put put = ProtobufUtil.toPut(mutation, cellScanner);
2817         if (request.hasCondition()) {
2818           Condition condition = request.getCondition();
2819           byte[] row = condition.getRow().toByteArray();
2820           byte[] family = condition.getFamily().toByteArray();
2821           byte[] qualifier = condition.getQualifier().toByteArray();
2822           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2823           ByteArrayComparable comparator =
2824             ProtobufUtil.toComparator(condition.getComparator());
2825           if (region.getCoprocessorHost() != null) {
2826             processed = region.getCoprocessorHost().preCheckAndPut(
2827               row, family, qualifier, compareOp, comparator, put);
2828           }
2829           if (processed == null) {
2830             boolean result = region.checkAndMutate(row, family,
2831               qualifier, compareOp, comparator, put, true);
2832             if (region.getCoprocessorHost() != null) {
2833               result = region.getCoprocessorHost().postCheckAndPut(row, family,
2834                 qualifier, compareOp, comparator, put, result);
2835             }
2836             processed = result;
2837           }
2838         } else {
2839           region.put(put);
2840           processed = Boolean.TRUE;
2841         }
2842         break;
2843       case DELETE:
2844         Delete delete = ProtobufUtil.toDelete(mutation, cellScanner);
2845         if (request.hasCondition()) {
2846           Condition condition = request.getCondition();
2847           byte[] row = condition.getRow().toByteArray();
2848           byte[] family = condition.getFamily().toByteArray();
2849           byte[] qualifier = condition.getQualifier().toByteArray();
2850           CompareOp compareOp = CompareOp.valueOf(condition.getCompareType().name());
2851           ByteArrayComparable comparator =
2852             ProtobufUtil.toComparator(condition.getComparator());
2853           if (region.getCoprocessorHost() != null) {
2854             processed = region.getCoprocessorHost().preCheckAndDelete(
2855               row, family, qualifier, compareOp, comparator, delete);
2856           }
2857           if (processed == null) {
2858             boolean result = region.checkAndMutate(row, family,
2859               qualifier, compareOp, comparator, delete, true);
2860             if (region.getCoprocessorHost() != null) {
2861               result = region.getCoprocessorHost().postCheckAndDelete(row, family,
2862                 qualifier, compareOp, comparator, delete, result);
2863             }
2864             processed = result;
2865           }
2866         } else {
2867           region.delete(delete);
2868           processed = Boolean.TRUE;
2869         }
2870         break;
2871         default:
2872           throw new DoNotRetryIOException(
2873             "Unsupported mutate type: " + type.name());
2874       }
2875       if (processed != null) builder.setProcessed(processed.booleanValue());
2876       addResult(builder, r, controller);
2877       return builder.build();
2878     } catch (IOException ie) {
2879       checkFileSystem();
2880       throw new ServiceException(ie);
2881     }
2882   }
2883 
2884 
2885   /**
2886    * @return True if current call supports cellblocks
2887    */
2888   private boolean isClientCellBlockSupport() {
2889     RpcCallContext context = RpcServer.getCurrentCall();
2890     return context != null && context.isClientCellBlockSupport();
2891   }
2892 
2893   private void addResult(final MutateResponse.Builder builder,
2894       final Result result, final PayloadCarryingRpcController rpcc) {
2895     if (result == null) return;
2896     if (isClientCellBlockSupport()) {
2897       builder.setResult(ProtobufUtil.toResultNoData(result));
2898       rpcc.setCellScanner(result.cellScanner());
2899     } else {
2900       ClientProtos.Result pbr = ProtobufUtil.toResult(result);
2901       builder.setResult(pbr);
2902     }
2903   }
2904 
2905   //
2906   // remote scanner interface
2907   //
2908 
2909   /**
2910    * Scan data in a table.
2911    *
2912    * @param controller the RPC controller
2913    * @param request the scan request
2914    * @throws ServiceException
2915    */
2916   @Override
2917   public ScanResponse scan(final RpcController controller, final ScanRequest request)
2918   throws ServiceException {
2919     Leases.Lease lease = null;
2920     String scannerName = null;
2921     try {
2922       if (!request.hasScannerId() && !request.hasScan()) {
2923         throw new DoNotRetryIOException(
2924           "Missing required input: scannerId or scan");
2925       }
2926       long scannerId = -1;
2927       if (request.hasScannerId()) {
2928         scannerId = request.getScannerId();
2929         scannerName = String.valueOf(scannerId);
2930       }
2931       try {
2932         checkOpen();
2933       } catch (IOException e) {
2934         // If checkOpen failed, server not running or filesystem gone,
2935         // cancel this lease; filesystem is gone or we're closing or something.
2936         if (scannerName != null) {
2937           try {
2938             leases.cancelLease(scannerName);
2939           } catch (LeaseException le) {
2940             LOG.info("Server shutting down and client tried to access missing scanner " +
2941               scannerName);
2942           }
2943         }
2944         throw e;
2945       }
2946       requestCount.increment();
2947 
2948       int ttl = 0;
2949       HRegion region = null;
2950       RegionScanner scanner = null;
2951       RegionScannerHolder rsh = null;
2952       boolean moreResults = true;
2953       boolean closeScanner = false;
2954       ScanResponse.Builder builder = ScanResponse.newBuilder();
2955       if (request.hasCloseScanner()) {
2956         closeScanner = request.getCloseScanner();
2957       }
2958       int rows = 1;
2959       if (request.hasNumberOfRows()) {
2960         rows = request.getNumberOfRows();
2961       }
2962       if (request.hasScannerId()) {
2963         rsh = scanners.get(scannerName);
2964         if (rsh == null) {
2965           LOG.info("Client tried to access missing scanner " + scannerName);
2966           throw new UnknownScannerException(
2967             "Name: " + scannerName + ", already closed?");
2968         }
2969         scanner = rsh.s;
2970         HRegionInfo hri = scanner.getRegionInfo();
2971         region = getRegion(hri.getRegionName());
2972         if (region != rsh.r) { // Yes, should be the same instance
2973           throw new NotServingRegionException("Region was re-opened after the scanner"
2974             + scannerName + " was created: " + hri.getRegionNameAsString());
2975         }
2976       } else {
2977         region = getRegion(request.getRegion());
2978         ClientProtos.Scan protoScan = request.getScan();
2979         boolean isLoadingCfsOnDemandSet = protoScan.hasLoadColumnFamiliesOnDemand();
2980         Scan scan = ProtobufUtil.toScan(protoScan);
2981         // if the request doesn't set this, get the default region setting.
2982         if (!isLoadingCfsOnDemandSet) {
2983           scan.setLoadColumnFamiliesOnDemand(region.isLoadingCfsOnDemandDefault());
2984         }
2985         scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE);
2986         region.prepareScanner(scan);
2987         if (region.getCoprocessorHost() != null) {
2988           scanner = region.getCoprocessorHost().preScannerOpen(scan);
2989         }
2990         if (scanner == null) {
2991           scanner = region.getScanner(scan);
2992         }
2993         if (region.getCoprocessorHost() != null) {
2994           scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner);
2995         }
2996         scannerId = addScanner(scanner, region);
2997         scannerName = String.valueOf(scannerId);
2998         ttl = this.scannerLeaseTimeoutPeriod;
2999       }
3000 
3001       if (rows > 0) {
3002         // if nextCallSeq does not match throw Exception straight away. This needs to be
3003         // performed even before checking of Lease.
3004         // See HBASE-5974
3005         if (request.hasNextCallSeq()) {
3006           if (rsh == null) {
3007             rsh = scanners.get(scannerName);
3008           }
3009           if (rsh != null) {
3010             if (request.getNextCallSeq() != rsh.nextCallSeq) {
3011               throw new OutOfOrderScannerNextException("Expected nextCallSeq: " + rsh.nextCallSeq
3012                 + " But the nextCallSeq got from client: " + request.getNextCallSeq() +
3013                 "; request=" + TextFormat.shortDebugString(request));
3014             }
3015             // Increment the nextCallSeq value which is the next expected from client.
3016             rsh.nextCallSeq++;
3017           }
3018         }
3019         try {
3020           // Remove lease while its being processed in server; protects against case
3021           // where processing of request takes > lease expiration time.
3022           lease = leases.removeLease(scannerName);
3023           List<Result> results = new ArrayList<Result>(rows);
3024           long currentScanResultSize = 0;
3025 
3026           boolean done = false;
3027           // Call coprocessor. Get region info from scanner.
3028           if (region != null && region.getCoprocessorHost() != null) {
3029             Boolean bypass = region.getCoprocessorHost().preScannerNext(
3030               scanner, results, rows);
3031             if (!results.isEmpty()) {
3032               for (Result r : results) {
3033                 if (maxScannerResultSize < Long.MAX_VALUE){
3034                   for (Cell kv : r.rawCells()) {
3035                     // TODO
3036                     currentScanResultSize += KeyValueUtil.ensureKeyValue(kv).heapSize();
3037                   }
3038                 }
3039               }
3040             }
3041             if (bypass != null && bypass.booleanValue()) {
3042               done = true;
3043             }
3044           }
3045 
3046           if (!done) {
3047             long maxResultSize = scanner.getMaxResultSize();
3048             if (maxResultSize <= 0) {
3049               maxResultSize = maxScannerResultSize;
3050             }
3051             List<Cell> values = new ArrayList<Cell>();
3052             MultiVersionConsistencyControl.setThreadReadPoint(scanner.getMvccReadPoint());
3053             region.startRegionOperation(Operation.SCAN);
3054             try {
3055               int i = 0;
3056               synchronized(scanner) {
3057                 for (; i < rows
3058                     && currentScanResultSize < maxResultSize; i++) {
3059                   // Collect values to be returned here
3060                   boolean moreRows = scanner.nextRaw(values);
3061                   if (!values.isEmpty()) {
3062                     if (maxScannerResultSize < Long.MAX_VALUE){
3063                       for (Cell kv : values) {
3064                         currentScanResultSize += KeyValueUtil.ensureKeyValue(kv).heapSize();
3065                       }
3066                     }
3067                     results.add(Result.create(values));
3068                   }
3069                   if (!moreRows) {
3070                     break;
3071                   }
3072                   values.clear();
3073                 }
3074               }
3075               region.readRequestsCount.add(i);
3076             } finally {
3077               region.closeRegionOperation();
3078             }
3079 
3080             // coprocessor postNext hook
3081             if (region != null && region.getCoprocessorHost() != null) {
3082               region.getCoprocessorHost().postScannerNext(scanner, results, rows, true);
3083             }
3084           }
3085 
3086           // If the scanner's filter - if any - is done with the scan
3087           // and wants to tell the client to stop the scan. This is done by passing
3088           // a null result, and setting moreResults to false.
3089           if (scanner.isFilterDone() && results.isEmpty()) {
3090             moreResults = false;
3091             results = null;
3092           } else {
3093             addResults(builder, results, controller);
3094           }
3095         } finally {
3096           // We're done. On way out re-add the above removed lease.
3097           // Adding resets expiration time on lease.
3098           if (scanners.containsKey(scannerName)) {
3099             if (lease != null) leases.addLease(lease);
3100             ttl = this.scannerLeaseTimeoutPeriod;
3101           }
3102         }
3103       }
3104 
3105       if (!moreResults || closeScanner) {
3106         ttl = 0;
3107         moreResults = false;
3108         if (region != null && region.getCoprocessorHost() != null) {
3109           if (region.getCoprocessorHost().preScannerClose(scanner)) {
3110             return builder.build(); // bypass
3111           }
3112         }
3113         rsh = scanners.remove(scannerName);
3114         if (rsh != null) {
3115           scanner = rsh.s;
3116           scanner.close();
3117           leases.cancelLease(scannerName);
3118           if (region != null && region.getCoprocessorHost() != null) {
3119             region.getCoprocessorHost().postScannerClose(scanner);
3120           }
3121         }
3122       }
3123 
3124       if (ttl > 0) {
3125         builder.setTtl(ttl);
3126       }
3127       builder.setScannerId(scannerId);
3128       builder.setMoreResults(moreResults);
3129       return builder.build();
3130     } catch (IOException ie) {
3131       if (scannerName != null && ie instanceof NotServingRegionException) {
3132         scanners.remove(scannerName);
3133       }
3134       throw new ServiceException(ie);
3135     }
3136   }
3137 
3138   private void addResults(final ScanResponse.Builder builder, final List<Result> results,
3139       final RpcController controller) {
3140     if (results == null || results.isEmpty()) return;
3141     if (isClientCellBlockSupport()) {
3142       for (Result res : results) {
3143         builder.addCellsPerResult(res.size());
3144       }
3145       ((PayloadCarryingRpcController)controller).
3146         setCellScanner(CellUtil.createCellScanner(results));
3147     } else {
3148       for (Result res: results) {
3149         ClientProtos.Result pbr = ProtobufUtil.toResult(res);
3150         builder.addResults(pbr);
3151       }
3152     }
3153   }
3154 
3155   /**
3156    * Atomically bulk load several HFiles into an open region
3157    * @return true if successful, false is failed but recoverably (no action)
3158    * @throws IOException if failed unrecoverably
3159    */
3160   @Override
3161   public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller,
3162       final BulkLoadHFileRequest request) throws ServiceException {
3163     try {
3164       requestCount.increment();
3165       HRegion region = getRegion(request.getRegion());
3166       List<Pair<byte[], String>> familyPaths = new ArrayList<Pair<byte[], String>>();
3167       for (FamilyPath familyPath: request.getFamilyPathList()) {
3168         familyPaths.add(new Pair<byte[], String>(familyPath.getFamily().toByteArray(),
3169           familyPath.getPath()));
3170       }
3171       boolean bypass = false;
3172       if (region.getCoprocessorHost() != null) {
3173         bypass = region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
3174       }
3175       boolean loaded = false;
3176       if (!bypass) {
3177         loaded = region.bulkLoadHFiles(familyPaths, request.getAssignSeqNum());
3178       }
3179       if (region.getCoprocessorHost() != null) {
3180         loaded = region.getCoprocessorHost().postBulkLoadHFile(familyPaths, loaded);
3181       }
3182       BulkLoadHFileResponse.Builder builder = BulkLoadHFileResponse.newBuilder();
3183       builder.setLoaded(loaded);
3184       return builder.build();
3185     } catch (IOException ie) {
3186       throw new ServiceException(ie);
3187     }
3188   }
3189 
3190   @Override
3191   public CoprocessorServiceResponse execService(final RpcController controller,
3192       final CoprocessorServiceRequest request) throws ServiceException {
3193     try {
3194       requestCount.increment();
3195       HRegion region = getRegion(request.getRegion());
3196       // ignore the passed in controller (from the serialized call)
3197       ServerRpcController execController = new ServerRpcController();
3198       Message result = region.execService(execController, request.getCall());
3199       if (execController.getFailedOn() != null) {
3200         throw execController.getFailedOn();
3201       }
3202       CoprocessorServiceResponse.Builder builder =
3203           CoprocessorServiceResponse.newBuilder();
3204       builder.setRegion(RequestConverter.buildRegionSpecifier(
3205           RegionSpecifierType.REGION_NAME, region.getRegionName()));
3206       builder.setValue(
3207           builder.getValueBuilder().setName(result.getClass().getName())
3208               .setValue(result.toByteString()));
3209       return builder.build();
3210     } catch (IOException ie) {
3211       throw new ServiceException(ie);
3212     }
3213   }
3214 
3215   /**
3216    * Execute multiple actions on a table: get, mutate, and/or execCoprocessor
3217    *
3218    * @param rpcc the RPC controller
3219    * @param request the multi request
3220    * @throws ServiceException
3221    */
3222   @Override
3223   public MultiResponse multi(final RpcController rpcc, final MultiRequest request)
3224   throws ServiceException {
3225     
3226     // rpc controller is how we bring in data via the back door;  it is unprotobuf'ed data.
3227     // It is also the conduit via which we pass back data.
3228     PayloadCarryingRpcController controller = (PayloadCarryingRpcController)rpcc;
3229     CellScanner cellScanner = controller != null? controller.cellScanner(): null;
3230     if (controller != null) controller.setCellScanner(null);
3231     List<CellScannable> cellsToReturn = null;
3232     MultiResponse.Builder responseBuilder = MultiResponse.newBuilder();
3233     RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
3234 
3235     for (RegionAction regionAction : request.getRegionActionList()) {
3236        this.requestCount.add(regionAction.getActionCount());
3237       regionActionResultBuilder.clear();
3238        HRegion region;
3239        try {
3240          region = getRegion(regionAction.getRegion());
3241        } catch (IOException e) {
3242          regionActionResultBuilder.setException(ResponseConverter.buildException(e));
3243          responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
3244          continue;  // For this region it's a failure.
3245        }
3246 
3247        if (regionAction.hasAtomic() && regionAction.getAtomic()) {
3248          // How does this call happen?  It may need some work to play well w/ the surroundings.
3249          // Need to return an item per Action along w/ Action index.  TODO.
3250          try {
3251            mutateRows(region, regionAction.getActionList(), cellScanner);
3252          } catch (IOException e) {
3253            // As it's atomic, we may expect it's a global failure.
3254            regionActionResultBuilder.setException(ResponseConverter.buildException(e));
3255          }
3256        } else {
3257          // doNonAtomicRegionMutation manages the exception internally
3258          cellsToReturn = doNonAtomicRegionMutation(region, regionAction, cellScanner,
3259              regionActionResultBuilder, cellsToReturn);
3260        }
3261        responseBuilder.addRegionActionResult(regionActionResultBuilder.build());
3262      }
3263      // Load the controller with the Cells to return.
3264      if (cellsToReturn != null && !cellsToReturn.isEmpty() && controller != null) {
3265        controller.setCellScanner(CellUtil.createCellScanner(cellsToReturn));
3266      }
3267      return responseBuilder.build();
3268    }
3269 
3270    /**
3271     * Run through the regionMutation <code>rm</code> and per Mutation, do the work, and then when
3272     * done, add an instance of a {@link ResultOrException} that corresponds to each Mutation.
3273     * @param region
3274     * @param actions
3275     * @param cellScanner
3276     * @param builder
3277     * @param cellsToReturn  Could be null. May be allocated in this method.  This is what this
3278     * method returns as a 'result'.
3279     * @return Return the <code>cellScanner</code> passed
3280     */
3281    private List<CellScannable> doNonAtomicRegionMutation(final HRegion region,
3282        final RegionAction actions, final CellScanner cellScanner,
3283        final RegionActionResult.Builder builder, List<CellScannable> cellsToReturn) {
3284      // Gather up CONTIGUOUS Puts and Deletes in this mutations List.  Idea is that rather than do
3285      // one at a time, we instead pass them in batch.  Be aware that the corresponding
3286      // ResultOrException instance that matches each Put or Delete is then added down in the
3287      // doBatchOp call.  We should be staying aligned though the Put and Delete are deferred/batched
3288      List<ClientProtos.Action> mutations = null;
3289      for (ClientProtos.Action action: actions.getActionList()) {
3290        ClientProtos.ResultOrException.Builder resultOrExceptionBuilder = null;
3291        try {
3292          Result r = null;
3293          if (action.hasGet()) {
3294            Get get = ProtobufUtil.toGet(action.getGet());
3295            r = region.get(get);
3296          } else if (action.hasMutation()) {
3297            MutationType type = action.getMutation().getMutateType();
3298            if (type != MutationType.PUT && type != MutationType.DELETE && mutations != null &&
3299                !mutations.isEmpty()) {
3300              // Flush out any Puts or Deletes already collected.
3301              doBatchOp(builder, region, mutations, cellScanner);
3302              mutations.clear();
3303            }
3304            switch (type) {
3305            case APPEND:
3306              r = append(region, action.getMutation(), cellScanner);
3307              break;
3308            case INCREMENT:
3309              r = increment(region, action.getMutation(), cellScanner);
3310              break;
3311            case PUT:
3312            case DELETE:
3313              // Collect the individual mutations and apply in a batch
3314              if (mutations == null) {
3315                mutations = new ArrayList<ClientProtos.Action>(actions.getActionCount());
3316             }
3317              mutations.add(action);
3318              break;
3319            default:
3320              throw new DoNotRetryIOException("Unsupported mutate type: " + type.name());
3321           }
3322         } else {
3323            throw new HBaseIOException("Unexpected Action type");
3324         }
3325         if (r != null) {
3326           ClientProtos.Result pbResult = null;
3327           if (isClientCellBlockSupport()) {
3328              pbResult = ProtobufUtil.toResultNoData(r);
3329              //  Hard to guess the size here.  Just make a rough guess.
3330              if (cellsToReturn == null) cellsToReturn = new ArrayList<CellScannable>();
3331              cellsToReturn.add(r);
3332           } else {
3333             pbResult = ProtobufUtil.toResult(r);
3334           }
3335           resultOrExceptionBuilder =
3336             ClientProtos.ResultOrException.newBuilder().setResult(pbResult);
3337         }
3338         // Could get to here and there was no result and no exception.  Presumes we added
3339         // a Put or Delete to the collecting Mutations List for adding later.  In this
3340         // case the corresponding ResultOrException instance for the Put or Delete will be added
3341         // down in the doBatchOp method call rather than up here.
3342       } catch (IOException ie) {
3343         resultOrExceptionBuilder = ResultOrException.newBuilder().
3344           setException(ResponseConverter.buildException(ie));
3345       }
3346       if (resultOrExceptionBuilder != null) {
3347         // Propagate index.
3348         resultOrExceptionBuilder.setIndex(action.getIndex());
3349         builder.addResultOrException(resultOrExceptionBuilder.build());
3350       }
3351     }
3352     // Finish up any outstanding mutations
3353     if (mutations != null && !mutations.isEmpty()) {
3354       doBatchOp(builder, region, mutations, cellScanner);
3355     }
3356     return cellsToReturn;
3357   }
3358 
3359 // End Client methods
3360 // Start Admin methods
3361 
3362   @Override
3363   @QosPriority(priority=HConstants.HIGH_QOS)
3364   public GetRegionInfoResponse getRegionInfo(final RpcController controller,
3365       final GetRegionInfoRequest request) throws ServiceException {
3366     try {
3367       checkOpen();
3368       requestCount.increment();
3369       HRegion region = getRegion(request.getRegion());
3370       HRegionInfo info = region.getRegionInfo();
3371       GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
3372       builder.setRegionInfo(HRegionInfo.convert(info));
3373       if (request.hasCompactionState() && request.getCompactionState()) {
3374         builder.setCompactionState(region.getCompactionState());
3375       }
3376       builder.setIsRecovering(region.isRecovering());
3377       return builder.build();
3378     } catch (IOException ie) {
3379       throw new ServiceException(ie);
3380     }
3381   }
3382 
3383   @Override
3384   public GetStoreFileResponse getStoreFile(final RpcController controller,
3385       final GetStoreFileRequest request) throws ServiceException {
3386     try {
3387       HRegion region = getRegion(request.getRegion());
3388       requestCount.increment();
3389       Set<byte[]> columnFamilies;
3390       if (request.getFamilyCount() == 0) {
3391         columnFamilies = region.getStores().keySet();
3392       } else {
3393         columnFamilies = new TreeSet<byte[]>(Bytes.BYTES_RAWCOMPARATOR);
3394         for (ByteString cf: request.getFamilyList()) {
3395           columnFamilies.add(cf.toByteArray());
3396         }
3397       }
3398       int nCF = columnFamilies.size();
3399       List<String>  fileList = region.getStoreFileList(
3400         columnFamilies.toArray(new byte[nCF][]));
3401       GetStoreFileResponse.Builder builder = GetStoreFileResponse.newBuilder();
3402       builder.addAllStoreFile(fileList);
3403       return builder.build();
3404     } catch (IOException ie) {
3405       throw new ServiceException(ie);
3406     }
3407   }
3408 
3409   @Override
3410   @QosPriority(priority=HConstants.HIGH_QOS)
3411   public GetOnlineRegionResponse getOnlineRegion(final RpcController controller,
3412       final GetOnlineRegionRequest request) throws ServiceException {
3413     try {
3414       checkOpen();
3415       requestCount.increment();
3416       List<HRegionInfo> list = new ArrayList<HRegionInfo>(onlineRegions.size());
3417       for (HRegion region: this.onlineRegions.values()) {
3418         list.add(region.getRegionInfo());
3419       }
3420       Collections.sort(list);
3421       return ResponseConverter.buildGetOnlineRegionResponse(list);
3422     } catch (IOException ie) {
3423       throw new ServiceException(ie);
3424     }
3425   }
3426 
3427   // Region open/close direct RPCs
3428 
3429   /**
3430    * Open asynchronously a region or a set of regions on the region server.
3431    *
3432    * The opening is coordinated by ZooKeeper, and this method requires the znode to be created
3433    *  before being called. As a consequence, this method should be called only from the master.
3434    * <p>
3435    * Different manages states for the region are:<ul>
3436    *  <li>region not opened: the region opening will start asynchronously.</li>
3437    *  <li>a close is already in progress: this is considered as an error.</li>
3438    *  <li>an open is already in progress: this new open request will be ignored. This is important
3439    *  because the Master can do multiple requests if it crashes.</li>
3440    *  <li>the region is already opened:  this new open request will be ignored./li>
3441    *  </ul>
3442    * </p>
3443    * <p>
3444    * Bulk assign: If there are more than 1 region to open, it will be considered as a bulk assign.
3445    * For a single region opening, errors are sent through a ServiceException. For bulk assign,
3446    * errors are put in the response as FAILED_OPENING.
3447    * </p>
3448    * @param controller the RPC controller
3449    * @param request the request
3450    * @throws ServiceException
3451    */
3452   @Override
3453   @QosPriority(priority=HConstants.HIGH_QOS)
3454   public OpenRegionResponse openRegion(final RpcController controller,
3455       final OpenRegionRequest request) throws ServiceException {
3456     try {
3457       checkOpen();
3458     } catch (IOException ie) {
3459       throw new ServiceException(ie);
3460     }
3461     requestCount.increment();
3462     OpenRegionResponse.Builder builder = OpenRegionResponse.newBuilder();
3463     final int regionCount = request.getOpenInfoCount();
3464     final Map<TableName, HTableDescriptor> htds =
3465         new HashMap<TableName, HTableDescriptor>(regionCount);
3466     final boolean isBulkAssign = regionCount > 1;
3467     for (RegionOpenInfo regionOpenInfo : request.getOpenInfoList()) {
3468       final HRegionInfo region = HRegionInfo.convert(regionOpenInfo.getRegion());
3469 
3470       int versionOfOfflineNode = -1;
3471       if (regionOpenInfo.hasVersionOfOfflineNode()) {
3472         versionOfOfflineNode = regionOpenInfo.getVersionOfOfflineNode();
3473       }
3474       HTableDescriptor htd;
3475       try {
3476         final HRegion onlineRegion = getFromOnlineRegions(region.getEncodedName());
3477         if (onlineRegion != null) {
3478           //Check if the region can actually be opened.
3479           if (onlineRegion.getCoprocessorHost() != null) {
3480             onlineRegion.getCoprocessorHost().preOpen();
3481           }
3482           // See HBASE-5094. Cross check with hbase:meta if still this RS is owning
3483           // the region.
3484           Pair<HRegionInfo, ServerName> p = MetaReader.getRegion(
3485               this.catalogTracker, region.getRegionName());
3486           if (this.getServerName().equals(p.getSecond())) {
3487             Boolean closing = regionsInTransitionInRS.get(region.getEncodedNameAsBytes());
3488             // Map regionsInTransitionInRSOnly has an entry for a region only if the region
3489             // is in transition on this RS, so here closing can be null. If not null, it can
3490             // be true or false. True means the region is opening on this RS; while false
3491             // means the region is closing. Only return ALREADY_OPENED if not closing (i.e.
3492             // not in transition any more, or still transition to open.
3493             if (!Boolean.FALSE.equals(closing)
3494                 && getFromOnlineRegions(region.getEncodedName()) != null) {
3495               LOG.warn("Attempted open of " + region.getEncodedName()
3496                 + " but already online on this server");
3497               builder.addOpeningState(RegionOpeningState.ALREADY_OPENED);
3498               continue;
3499             }
3500           } else {
3501             LOG.warn("The region " + region.getEncodedName() + " is online on this server" +
3502                 " but hbase:meta does not have this server - continue opening.");
3503             removeFromOnlineRegions(onlineRegion, null);
3504           }
3505         }
3506         LOG.info("Open " + region.getRegionNameAsString());
3507         htd = htds.get(region.getTable());
3508         if (htd == null) {
3509           htd = this.tableDescriptors.get(region.getTable());
3510           htds.put(region.getTable(), htd);
3511         }
3512 
3513         final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(
3514             region.getEncodedNameAsBytes(), Boolean.TRUE);
3515 
3516         if (Boolean.FALSE.equals(previous)) {
3517           // There is a close in progress. We need to mark this open as failed in ZK.
3518           OpenRegionHandler.
3519               tryTransitionFromOfflineToFailedOpen(this, region, versionOfOfflineNode);
3520 
3521           throw new RegionAlreadyInTransitionException("Received OPEN for the region:" +
3522               region.getRegionNameAsString() + " , which we are already trying to CLOSE ");
3523         }
3524 
3525         if (Boolean.TRUE.equals(previous)) {
3526           // An open is in progress. This is supported, but let's log this.
3527           LOG.info("Receiving OPEN for the region:" +
3528               region.getRegionNameAsString() + " , which we are already trying to OPEN" +
3529               " - ignoring this new request for this region.");
3530         }
3531 
3532         // We are opening this region. If it moves back and forth for whatever reason, we don't
3533         // want to keep returning the stale moved record while we are opening/if we close again.
3534         removeFromMovedRegions(region.getEncodedName());
3535 
3536         if (previous == null) {
3537           // check if the region to be opened is marked in recovering state in ZK
3538           if (this.distributedLogReplay
3539               && SplitLogManager.isRegionMarkedRecoveringInZK(this.getZooKeeper(),
3540             region.getEncodedName())) {
3541             this.recoveringRegions.put(region.getEncodedName(), null);
3542           }
3543           // If there is no action in progress, we can submit a specific handler.
3544           // Need to pass the expected version in the constructor.
3545           if (region.isMetaRegion()) {
3546             this.service.submit(new OpenMetaHandler(this, this, region, htd,
3547                 versionOfOfflineNode));
3548           } else {
3549             updateRegionFavoredNodesMapping(region.getEncodedName(),
3550                 regionOpenInfo.getFavoredNodesList());
3551             this.service.submit(new OpenRegionHandler(this, this, region, htd,
3552                 versionOfOfflineNode));
3553           }
3554         }
3555 
3556         builder.addOpeningState(RegionOpeningState.OPENED);
3557 
3558       } catch (KeeperException zooKeeperEx) {
3559         LOG.error("Can't retrieve recovering state from zookeeper", zooKeeperEx);
3560         throw new ServiceException(zooKeeperEx);
3561       } catch (IOException ie) {
3562         LOG.warn("Failed opening region " + region.getRegionNameAsString(), ie);
3563         if (isBulkAssign) {
3564           builder.addOpeningState(RegionOpeningState.FAILED_OPENING);
3565         } else {
3566           throw new ServiceException(ie);
3567         }
3568       }
3569     }
3570 
3571     return builder.build();
3572   }
3573 
3574   @Override
3575   public void updateRegionFavoredNodesMapping(String encodedRegionName,
3576       List<org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName> favoredNodes) {
3577     InetSocketAddress[] addr = new InetSocketAddress[favoredNodes.size()];
3578     // Refer to the comment on the declaration of regionFavoredNodesMap on why
3579     // it is a map of region name to InetSocketAddress[]
3580     for (int i = 0; i < favoredNodes.size(); i++) {
3581       addr[i] = InetSocketAddress.createUnresolved(favoredNodes.get(i).getHostName(),
3582           favoredNodes.get(i).getPort());
3583     }
3584     regionFavoredNodesMap.put(encodedRegionName, addr);
3585   }
3586 
3587   /**
3588    * Return the favored nodes for a region given its encoded name. Look at the
3589    * comment around {@link #regionFavoredNodesMap} on why it is InetSocketAddress[]
3590    * @param encodedRegionName
3591    * @return array of favored locations
3592    */
3593   @Override
3594   public InetSocketAddress[] getFavoredNodesForRegion(String encodedRegionName) {
3595     return regionFavoredNodesMap.get(encodedRegionName);
3596   }
3597 
3598   /**
3599    * Close a region on the region server.
3600    *
3601    * @param controller the RPC controller
3602    * @param request the request
3603    * @throws ServiceException
3604    */
3605   @Override
3606   @QosPriority(priority=HConstants.HIGH_QOS)
3607   public CloseRegionResponse closeRegion(final RpcController controller,
3608       final CloseRegionRequest request) throws ServiceException {
3609     int versionOfClosingNode = -1;
3610     if (request.hasVersionOfClosingNode()) {
3611       versionOfClosingNode = request.getVersionOfClosingNode();
3612     }
3613     boolean zk = request.getTransitionInZK();
3614     final ServerName sn = (request.hasDestinationServer() ?
3615       ProtobufUtil.toServerName(request.getDestinationServer()) : null);
3616 
3617     try {
3618       checkOpen();
3619       final String encodedRegionName = ProtobufUtil.getRegionEncodedName(request.getRegion());
3620 
3621       // Can be null if we're calling close on a region that's not online
3622       final HRegion region = this.getFromOnlineRegions(encodedRegionName);
3623       if ((region  != null) && (region .getCoprocessorHost() != null)) {
3624         region.getCoprocessorHost().preClose(false);
3625       }
3626 
3627       requestCount.increment();
3628       LOG.info("Close " + encodedRegionName + ", via zk=" + (zk ? "yes" : "no") +
3629         ", znode version=" + versionOfClosingNode + ", on " + sn);
3630 
3631       boolean closed = closeRegion(encodedRegionName, false, zk, versionOfClosingNode, sn);
3632       CloseRegionResponse.Builder builder = CloseRegionResponse.newBuilder().setClosed(closed);
3633       return builder.build();
3634     } catch (IOException ie) {
3635       throw new ServiceException(ie);
3636     }
3637   }
3638 
3639   /**
3640    * Flush a region on the region server.
3641    *
3642    * @param controller the RPC controller
3643    * @param request the request
3644    * @throws ServiceException
3645    */
3646   @Override
3647   @QosPriority(priority=HConstants.HIGH_QOS)
3648   public FlushRegionResponse flushRegion(final RpcController controller,
3649       final FlushRegionRequest request) throws ServiceException {
3650     try {
3651       checkOpen();
3652       requestCount.increment();
3653       HRegion region = getRegion(request.getRegion());
3654       LOG.info("Flushing " + region.getRegionNameAsString());
3655       boolean shouldFlush = true;
3656       if (request.hasIfOlderThanTs()) {
3657         shouldFlush = region.getLastFlushTime() < request.getIfOlderThanTs();
3658       }
3659       FlushRegionResponse.Builder builder = FlushRegionResponse.newBuilder();
3660       if (shouldFlush) {
3661         boolean result = region.flushcache();
3662         if (result) {
3663           this.compactSplitThread.requestSystemCompaction(region,
3664               "Compaction through user triggered flush");
3665         }
3666         builder.setFlushed(result);
3667       }
3668       builder.setLastFlushTime(region.getLastFlushTime());
3669       return builder.build();
3670     } catch (IOException ie) {
3671       throw new ServiceException(ie);
3672     }
3673   }
3674 
3675   /**
3676    * Split a region on the region server.
3677    *
3678    * @param controller the RPC controller
3679    * @param request the request
3680    * @throws ServiceException
3681    */
3682   @Override
3683   @QosPriority(priority=HConstants.HIGH_QOS)
3684   public SplitRegionResponse splitRegion(final RpcController controller,
3685       final SplitRegionRequest request) throws ServiceException {
3686     try {
3687       checkOpen();
3688       requestCount.increment();
3689       HRegion region = getRegion(request.getRegion());
3690       region.startRegionOperation(Operation.SPLIT_REGION);
3691       LOG.info("Splitting " + region.getRegionNameAsString());
3692       region.flushcache();
3693       byte[] splitPoint = null;
3694       if (request.hasSplitPoint()) {
3695         splitPoint = request.getSplitPoint().toByteArray();
3696       }
3697       region.forceSplit(splitPoint);
3698       compactSplitThread.requestSplit(region, region.checkSplit());
3699       return SplitRegionResponse.newBuilder().build();
3700     } catch (IOException ie) {
3701       throw new ServiceException(ie);
3702     }
3703   }
3704 
3705   /**
3706    * Merge regions on the region server.
3707    *
3708    * @param controller the RPC controller
3709    * @param request the request
3710    * @return merge regions response
3711    * @throws ServiceException
3712    */
3713   @Override
3714   @QosPriority(priority = HConstants.HIGH_QOS)
3715   public MergeRegionsResponse mergeRegions(final RpcController controller,
3716       final MergeRegionsRequest request) throws ServiceException {
3717     try {
3718       checkOpen();
3719       requestCount.increment();
3720       HRegion regionA = getRegion(request.getRegionA());
3721       HRegion regionB = getRegion(request.getRegionB());
3722       boolean forcible = request.getForcible();
3723       regionA.startRegionOperation(Operation.MERGE_REGION);
3724       regionB.startRegionOperation(Operation.MERGE_REGION);
3725       LOG.info("Receiving merging request for  " + regionA + ", " + regionB
3726           + ",forcible=" + forcible);
3727       regionA.flushcache();
3728       regionB.flushcache();
3729       compactSplitThread.requestRegionsMerge(regionA, regionB, forcible);
3730       return MergeRegionsResponse.newBuilder().build();
3731     } catch (IOException ie) {
3732       throw new ServiceException(ie);
3733     }
3734   }
3735 
3736   /**
3737    * Compact a region on the region server.
3738    *
3739    * @param controller the RPC controller
3740    * @param request the request
3741    * @throws ServiceException
3742    */
3743   @Override
3744   @QosPriority(priority=HConstants.HIGH_QOS)
3745   public CompactRegionResponse compactRegion(final RpcController controller,
3746       final CompactRegionRequest request) throws ServiceException {
3747     try {
3748       checkOpen();
3749       requestCount.increment();
3750       HRegion region = getRegion(request.getRegion());
3751       region.startRegionOperation(Operation.COMPACT_REGION);
3752       LOG.info("Compacting " + region.getRegionNameAsString());
3753       boolean major = false;
3754       byte [] family = null;
3755       Store store = null;
3756       if (request.hasFamily()) {
3757         family = request.getFamily().toByteArray();
3758         store = region.getStore(family);
3759         if (store == null) {
3760           throw new ServiceException(new IOException("column family " + Bytes.toString(family) +
3761             " does not exist in region " + region.getRegionNameAsString()));
3762         }
3763       }
3764       if (request.hasMajor()) {
3765         major = request.getMajor();
3766       }
3767       if (major) {
3768         if (family != null) {
3769           store.triggerMajorCompaction();
3770         } else {
3771           region.triggerMajorCompaction();
3772         }
3773       }
3774 
3775       String familyLogMsg = (family != null)?" for column family: " + Bytes.toString(family):"";
3776       LOG.trace("User-triggered compaction requested for region " +
3777         region.getRegionNameAsString() + familyLogMsg);
3778       String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
3779       if(family != null) {
3780         compactSplitThread.requestCompaction(region, store, log,
3781           Store.PRIORITY_USER, null);
3782       } else {
3783         compactSplitThread.requestCompaction(region, log,
3784           Store.PRIORITY_USER, null);
3785       }
3786       return CompactRegionResponse.newBuilder().build();
3787     } catch (IOException ie) {
3788       throw new ServiceException(ie);
3789     }
3790   }
3791 
3792   /**
3793    * Replicate WAL entries on the region server.
3794    *
3795    * @param controller the RPC controller
3796    * @param request the request
3797    * @throws ServiceException
3798    */
3799   @Override
3800   @QosPriority(priority=HConstants.REPLICATION_QOS)
3801   public ReplicateWALEntryResponse replicateWALEntry(final RpcController controller,
3802       final ReplicateWALEntryRequest request)
3803   throws ServiceException {
3804     try {
3805       if (replicationSinkHandler != null) {
3806         checkOpen();
3807         requestCount.increment();
3808         this.replicationSinkHandler.replicateLogEntries(request.getEntryList(),
3809           ((PayloadCarryingRpcController)controller).cellScanner());
3810       }
3811       return ReplicateWALEntryResponse.newBuilder().build();
3812     } catch (IOException ie) {
3813       throw new ServiceException(ie);
3814     }
3815   }
3816 
3817   /**
3818    * Replay the given changes when distributedLogReplay WAL edits from a failed RS. The guarantee is
3819    * that the given mutations will be durable on the receiving RS if this method returns without any
3820    * exception.
3821    * @param controller the RPC controller
3822    * @param request the request
3823    * @throws ServiceException
3824    */
3825   @Override
3826   @QosPriority(priority = HConstants.REPLAY_QOS)
3827   public ReplicateWALEntryResponse replay(final RpcController controller,
3828       final ReplicateWALEntryRequest request) throws ServiceException {
3829     long before = EnvironmentEdgeManager.currentTimeMillis();
3830     CellScanner cells = ((PayloadCarryingRpcController) controller).cellScanner();
3831     try {
3832       checkOpen();
3833       List<WALEntry> entries = request.getEntryList();
3834       if (entries == null || entries.isEmpty()) {
3835         // empty input
3836         return ReplicateWALEntryResponse.newBuilder().build();
3837       }
3838       HRegion region = this.getRegionByEncodedName(
3839         entries.get(0).getKey().getEncodedRegionName().toStringUtf8());
3840       RegionCoprocessorHost coprocessorHost = region.getCoprocessorHost();
3841       List<Pair<HLogKey, WALEdit>> walEntries = new ArrayList<Pair<HLogKey, WALEdit>>();
3842       List<Pair<MutationType, Mutation>> mutations = new ArrayList<Pair<MutationType, Mutation>>();
3843       for (WALEntry entry : entries) {
3844         Pair<HLogKey, WALEdit> walEntry = (coprocessorHost == null) ? null :
3845           new Pair<HLogKey, WALEdit>();
3846         List<Pair<MutationType, Mutation>> edits = HLogSplitter.getMutationsFromWALEntry(entry,
3847           cells, walEntry);
3848         if (coprocessorHost != null) {
3849           // Start coprocessor replay here. The coprocessor is for each WALEdit instead of a
3850           // KeyValue.
3851           if (coprocessorHost.preWALRestore(region.getRegionInfo(), walEntry.getFirst(),
3852             walEntry.getSecond())) {
3853             // if bypass this log entry, ignore it ...
3854             continue;
3855           }
3856           walEntries.add(walEntry);
3857         }
3858         mutations.addAll(edits);
3859       }
3860 
3861       if (!mutations.isEmpty()) {
3862         OperationStatus[] result = doBatchOp(region, mutations, true);
3863         // check if it's a partial success
3864         for (int i = 0; result != null && i < result.length; i++) {
3865           if (result[i] != OperationStatus.SUCCESS) {
3866             throw new IOException(result[i].getExceptionMsg());
3867           }
3868         }
3869       }
3870       if (coprocessorHost != null) {
3871         for (Pair<HLogKey, WALEdit> wal : walEntries) {
3872           coprocessorHost.postWALRestore(region.getRegionInfo(), wal.getFirst(),
3873             wal.getSecond());
3874         }
3875       }
3876       return ReplicateWALEntryResponse.newBuilder().build();
3877     } catch (IOException ie) {
3878       throw new ServiceException(ie);
3879     } finally {
3880       metricsRegionServer.updateReplay(EnvironmentEdgeManager.currentTimeMillis() - before);
3881     }
3882   }
3883 
3884   /**
3885    * Roll the WAL writer of the region server.
3886    * @param controller the RPC controller
3887    * @param request the request
3888    * @throws ServiceException
3889    */
3890   @Override
3891   public RollWALWriterResponse rollWALWriter(final RpcController controller,
3892       final RollWALWriterRequest request) throws ServiceException {
3893     try {
3894       requestCount.increment();
3895       HLog wal = this.getWAL();
3896       byte[][] regionsToFlush = wal.rollWriter(true);
3897       RollWALWriterResponse.Builder builder = RollWALWriterResponse.newBuilder();
3898       if (regionsToFlush != null) {
3899         for (byte[] region: regionsToFlush) {
3900           builder.addRegionToFlush(ZeroCopyLiteralByteString.wrap(region));
3901         }
3902       }
3903       return builder.build();
3904     } catch (IOException ie) {
3905       throw new ServiceException(ie);
3906     }
3907   }
3908 
3909   /**
3910    * Stop the region server.
3911    *
3912    * @param controller the RPC controller
3913    * @param request the request
3914    * @throws ServiceException
3915    */
3916   @Override
3917   public StopServerResponse stopServer(final RpcController controller,
3918       final StopServerRequest request) throws ServiceException {
3919     requestCount.increment();
3920     String reason = request.getReason();
3921     stop(reason);
3922     return StopServerResponse.newBuilder().build();
3923   }
3924 
3925   /**
3926    * Get some information of the region server.
3927    *
3928    * @param controller the RPC controller
3929    * @param request the request
3930    * @throws ServiceException
3931    */
3932   @Override
3933   public GetServerInfoResponse getServerInfo(final RpcController controller,
3934       final GetServerInfoRequest request) throws ServiceException {
3935     ServerName serverName = getServerName();
3936     requestCount.increment();
3937     return ResponseConverter.buildGetServerInfoResponse(serverName, webuiport);
3938   }
3939 
3940 // End Admin methods
3941 
3942   /**
3943    * Find the HRegion based on a region specifier
3944    *
3945    * @param regionSpecifier the region specifier
3946    * @return the corresponding region
3947    * @throws IOException if the specifier is not null,
3948    *    but failed to find the region
3949    */
3950   protected HRegion getRegion(
3951       final RegionSpecifier regionSpecifier) throws IOException {
3952     return getRegionByEncodedName(regionSpecifier.getValue().toByteArray(),
3953         ProtobufUtil.getRegionEncodedName(regionSpecifier));
3954   }
3955 
3956   /**
3957    * Execute an append mutation.
3958    *
3959    * @param region
3960    * @param m
3961    * @param cellScanner
3962    * @return result to return to client if default operation should be
3963    * bypassed as indicated by RegionObserver, null otherwise
3964    * @throws IOException
3965    */
3966   protected Result append(final HRegion region,
3967       final MutationProto m, final CellScanner cellScanner) throws IOException {
3968     long before = EnvironmentEdgeManager.currentTimeMillis();
3969     Append append = ProtobufUtil.toAppend(m, cellScanner);
3970     Result r = null;
3971     if (region.getCoprocessorHost() != null) {
3972       r = region.getCoprocessorHost().preAppend(append);
3973     }
3974     if (r == null) {
3975       r = region.append(append);
3976       if (region.getCoprocessorHost() != null) {
3977         region.getCoprocessorHost().postAppend(append, r);
3978       }
3979     }
3980     metricsRegionServer.updateAppend(EnvironmentEdgeManager.currentTimeMillis() - before);
3981     return r;
3982   }
3983 
3984   /**
3985    * Execute an increment mutation.
3986    *
3987    * @param region
3988    * @param mutation
3989    * @return the Result
3990    * @throws IOException
3991    */
3992   protected Result increment(final HRegion region, final MutationProto mutation,
3993       final CellScanner cells)
3994   throws IOException {
3995     long before = EnvironmentEdgeManager.currentTimeMillis();
3996     Increment increment = ProtobufUtil.toIncrement(mutation, cells);
3997     Result r = null;
3998     if (region.getCoprocessorHost() != null) {
3999       r = region.getCoprocessorHost().preIncrement(increment);
4000     }
4001     if (r == null) {
4002       r = region.increment(increment);
4003       if (region.getCoprocessorHost() != null) {
4004         r = region.getCoprocessorHost().postIncrement(increment, r);
4005       }
4006     }
4007     metricsRegionServer.updateIncrement(EnvironmentEdgeManager.currentTimeMillis() - before);
4008     return r;
4009   }
4010 
4011   /**
4012    * Execute a list of Put/Delete mutations.
4013    *
4014    * @param builder
4015    * @param region
4016    * @param mutations
4017    */
4018   protected void doBatchOp(final RegionActionResult.Builder builder, final HRegion region,
4019       final List<ClientProtos.Action> mutations, final CellScanner cells) {
4020     Mutation[] mArray = new Mutation[mutations.size()];
4021     long before = EnvironmentEdgeManager.currentTimeMillis();
4022     boolean batchContainsPuts = false, batchContainsDelete = false;
4023     try {
4024       int i = 0;
4025       for (ClientProtos.Action action: mutations) {
4026         MutationProto m = action.getMutation();
4027         Mutation mutation;
4028         if (m.getMutateType() == MutationType.PUT) {
4029           mutation = ProtobufUtil.toPut(m, cells);
4030           batchContainsPuts = true;
4031         } else {
4032           mutation = ProtobufUtil.toDelete(m, cells);
4033           batchContainsDelete = true;
4034         }
4035         mArray[i++] = mutation;
4036       }
4037 
4038       requestCount.add(mutations.size());
4039       if (!region.getRegionInfo().isMetaTable()) {
4040         cacheFlusher.reclaimMemStoreMemory();
4041       }
4042 
4043       OperationStatus codes[] = region.batchMutate(mArray, false);
4044       for (i = 0; i < codes.length; i++) {
4045         int index = mutations.get(i).getIndex();
4046         Exception e = null;
4047         switch (codes[i].getOperationStatusCode()) {
4048           case BAD_FAMILY:
4049             e = new NoSuchColumnFamilyException(codes[i].getExceptionMsg());
4050             builder.addResultOrException(getResultOrException(e, index));
4051             break;
4052 
4053           case SANITY_CHECK_FAILURE:
4054             e = new FailedSanityCheckException(codes[i].getExceptionMsg());
4055             builder.addResultOrException(getResultOrException(e, index));
4056             break;
4057 
4058           default:
4059             e = new DoNotRetryIOException(codes[i].getExceptionMsg());
4060             builder.addResultOrException(getResultOrException(e, index));
4061             break;
4062 
4063           case SUCCESS:
4064             builder.addResultOrException(getResultOrException(ClientProtos.Result.getDefaultInstance(), index));
4065             break;
4066         }
4067       }
4068     } catch (IOException ie) {
4069       for (int i = 0; i < mutations.size(); i++) {
4070         builder.addResultOrException(getResultOrException(ie, mutations.get(i).getIndex()));
4071       }
4072     }
4073     long after = EnvironmentEdgeManager.currentTimeMillis();
4074     if (batchContainsPuts) {
4075       metricsRegionServer.updatePut(after - before);
4076     }
4077     if (batchContainsDelete) {
4078       metricsRegionServer.updateDelete(after - before);
4079     }
4080   }
4081   private static ResultOrException getResultOrException(final ClientProtos.Result r,
4082       final int index) {
4083     return getResultOrException(ResponseConverter.buildActionResult(r), index);
4084   }
4085   private static ResultOrException getResultOrException(final Exception e, final int index) {
4086     return getResultOrException(ResponseConverter.buildActionResult(e), index);
4087   }
4088 
4089   private static ResultOrException getResultOrException(final ResultOrException.Builder builder,
4090       final int index) {
4091     return builder.setIndex(index).build();
4092   }
4093 
4094   /**
4095    * Execute a list of Put/Delete mutations. The function returns OperationStatus instead of
4096    * constructing MultiResponse to save a possible loop if caller doesn't need MultiResponse.
4097    * @param region
4098    * @param mutations
4099    * @param isReplay
4100    * @return an array of OperationStatus which internally contains the OperationStatusCode and the
4101    *         exceptionMessage if any
4102    * @throws IOException
4103    */
4104   protected OperationStatus [] doBatchOp(final HRegion region,
4105       final List<Pair<MutationType, Mutation>> mutations, boolean isReplay)
4106   throws IOException {
4107     Mutation[] mArray = new Mutation[mutations.size()];
4108     long before = EnvironmentEdgeManager.currentTimeMillis();
4109     boolean batchContainsPuts = false, batchContainsDelete = false;
4110     try {
4111       int i = 0;
4112       for (Pair<MutationType, Mutation> m : mutations) {
4113         if (m.getFirst() == MutationType.PUT) {
4114           batchContainsPuts = true;
4115         } else {
4116           batchContainsDelete = true;
4117         }
4118         mArray[i++] = m.getSecond();
4119       }
4120       requestCount.add(mutations.size());
4121       if (!region.getRegionInfo().isMetaTable()) {
4122         cacheFlusher.reclaimMemStoreMemory();
4123       }
4124       return region.batchMutate(mArray, isReplay);
4125     } finally {
4126       long after = EnvironmentEdgeManager.currentTimeMillis();
4127       if (batchContainsPuts) {
4128         metricsRegionServer.updatePut(after - before);
4129       }
4130       if (batchContainsDelete) {
4131         metricsRegionServer.updateDelete(after - before);
4132       }
4133     }
4134   }
4135 
4136   /**
4137    * Mutate a list of rows atomically.
4138    *
4139    * @param region
4140    * @param actions
4141  * @param cellScanner if non-null, the mutation data -- the Cell content.
4142    * @throws IOException
4143    */
4144   protected void mutateRows(final HRegion region, final List<ClientProtos.Action> actions,
4145       final CellScanner cellScanner)
4146   throws IOException {
4147     if (!region.getRegionInfo().isMetaTable()) {
4148       cacheFlusher.reclaimMemStoreMemory();
4149     }
4150     RowMutations rm = null;
4151     for (ClientProtos.Action action: actions) {
4152       if (action.hasGet()) {
4153         throw new DoNotRetryIOException("Atomic put and/or delete only, not a Get=" +
4154           action.getGet());
4155       }
4156       MutationType type = action.getMutation().getMutateType();
4157       if (rm == null) {
4158         rm = new RowMutations(action.getMutation().getRow().toByteArray());
4159       }
4160       switch (type) {
4161       case PUT:
4162         rm.add(ProtobufUtil.toPut(action.getMutation(), cellScanner));
4163         break;
4164       case DELETE:
4165         rm.add(ProtobufUtil.toDelete(action.getMutation(), cellScanner));
4166         break;
4167         default:
4168           throw new DoNotRetryIOException("Atomic put and/or delete only, not " + type.name());
4169       }
4170     }
4171     region.mutateRow(rm);
4172   }
4173 
4174   private static class MovedRegionInfo {
4175     private final ServerName serverName;
4176     private final long seqNum;
4177     private final long ts;
4178 
4179     public MovedRegionInfo(ServerName serverName, long closeSeqNum) {
4180       this.serverName = serverName;
4181       this.seqNum = closeSeqNum;
4182       ts = EnvironmentEdgeManager.currentTimeMillis();
4183      }
4184 
4185     public ServerName getServerName() {
4186       return serverName;
4187     }
4188 
4189     public long getSeqNum() {
4190       return seqNum;
4191     }
4192 
4193     public long getMoveTime() {
4194       return ts;
4195     }
4196   }
4197 
4198   // This map will contains all the regions that we closed for a move.
4199   //  We add the time it was moved as we don't want to keep too old information
4200   protected Map<String, MovedRegionInfo> movedRegions =
4201       new ConcurrentHashMap<String, MovedRegionInfo>(3000);
4202 
4203   // We need a timeout. If not there is a risk of giving a wrong information: this would double
4204   //  the number of network calls instead of reducing them.
4205   private static final int TIMEOUT_REGION_MOVED = (2 * 60 * 1000);
4206 
4207   protected void addToMovedRegions(String encodedName, ServerName destination, long closeSeqNum) {
4208     if (ServerName.isSameHostnameAndPort(destination, this.getServerName())) {
4209       LOG.warn("Not adding moved region record: " + encodedName + " to self.");
4210       return;
4211     }
4212     LOG.info("Adding moved region record: " + encodedName + " to "
4213         + destination.getServerName() + ":" + destination.getPort()
4214         + " as of " + closeSeqNum);
4215     movedRegions.put(encodedName, new MovedRegionInfo(destination, closeSeqNum));
4216   }
4217 
4218   private void removeFromMovedRegions(String encodedName) {
4219     movedRegions.remove(encodedName);
4220   }
4221 
4222   private MovedRegionInfo getMovedRegion(final String encodedRegionName) {
4223     MovedRegionInfo dest = movedRegions.get(encodedRegionName);
4224 
4225     long now = EnvironmentEdgeManager.currentTimeMillis();
4226     if (dest != null) {
4227       if (dest.getMoveTime() > (now - TIMEOUT_REGION_MOVED)) {
4228         return dest;
4229       } else {
4230         movedRegions.remove(encodedRegionName);
4231       }
4232     }
4233 
4234     return null;
4235   }
4236 
4237   /**
4238    * Remove the expired entries from the moved regions list.
4239    */
4240   protected void cleanMovedRegions() {
4241     final long cutOff = System.currentTimeMillis() - TIMEOUT_REGION_MOVED;
4242     Iterator<Entry<String, MovedRegionInfo>> it = movedRegions.entrySet().iterator();
4243 
4244     while (it.hasNext()){
4245       Map.Entry<String, MovedRegionInfo> e = it.next();
4246       if (e.getValue().getMoveTime() < cutOff) {
4247         it.remove();
4248       }
4249     }
4250   }
4251 
4252   /**
4253    * Creates a Chore thread to clean the moved region cache.
4254    */
4255   protected static class MovedRegionsCleaner extends Chore implements Stoppable {
4256     private HRegionServer regionServer;
4257     Stoppable stoppable;
4258 
4259     private MovedRegionsCleaner(
4260       HRegionServer regionServer, Stoppable stoppable){
4261       super("MovedRegionsCleaner for region "+regionServer, TIMEOUT_REGION_MOVED, stoppable);
4262       this.regionServer = regionServer;
4263       this.stoppable = stoppable;
4264     }
4265 
4266     static MovedRegionsCleaner createAndStart(HRegionServer rs){
4267       Stoppable stoppable = new Stoppable() {
4268         private volatile boolean isStopped = false;
4269         @Override public void stop(String why) { isStopped = true;}
4270         @Override public boolean isStopped() {return isStopped;}
4271       };
4272 
4273       return new MovedRegionsCleaner(rs, stoppable);
4274     }
4275 
4276     @Override
4277     protected void chore() {
4278       regionServer.cleanMovedRegions();
4279     }
4280 
4281     @Override
4282     public void stop(String why) {
4283       stoppable.stop(why);
4284     }
4285 
4286     @Override
4287     public boolean isStopped() {
4288       return stoppable.isStopped();
4289     }
4290   }
4291 
4292   private String getMyEphemeralNodePath() {
4293     return ZKUtil.joinZNode(this.zooKeeper.rsZNode, getServerName().toString());
4294   }
4295 
4296   /**
4297    * Holder class which holds the RegionScanner and nextCallSeq together.
4298    */
4299   private static class RegionScannerHolder {
4300     private RegionScanner s;
4301     private long nextCallSeq = 0L;
4302     private HRegion r;
4303 
4304     public RegionScannerHolder(RegionScanner s, HRegion r) {
4305       this.s = s;
4306       this.r = r;
4307     }
4308   }
4309 
4310   private boolean isHealthCheckerConfigured() {
4311     String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
4312     return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
4313   }
4314 
4315   /**
4316    * @return the underlying {@link CompactSplitThread} for the servers
4317    */
4318   public CompactSplitThread getCompactSplitThread() {
4319     return this.compactSplitThread;
4320   }
4321 
4322   /**
4323    * A helper function to store the last flushed sequence Id with the previous failed RS for a
4324    * recovering region. The Id is used to skip wal edits which are flushed. Since the flushed
4325    * sequence id is only valid for each RS, we associate the Id with corresponding failed RS.
4326    * @throws KeeperException
4327    * @throws IOException
4328    */
4329   private void updateRecoveringRegionLastFlushedSequenceId(HRegion r) throws KeeperException,
4330       IOException {
4331     if (!r.isRecovering()) {
4332       // return immdiately for non-recovering regions
4333       return;
4334     }
4335 
4336     HRegionInfo region = r.getRegionInfo();
4337     ZooKeeperWatcher zkw = getZooKeeper();
4338     String previousRSName = this.getLastFailedRSFromZK(region.getEncodedName());
4339     Map<byte[], Long> maxSeqIdInStores = r.getMaxStoreSeqIdForLogReplay();
4340     long minSeqIdForLogReplay = -1;
4341     for (byte[] columnFamily : maxSeqIdInStores.keySet()) {
4342       Long storeSeqIdForReplay = maxSeqIdInStores.get(columnFamily);
4343       if (minSeqIdForLogReplay == -1 || storeSeqIdForReplay < minSeqIdForLogReplay) {
4344         minSeqIdForLogReplay = storeSeqIdForReplay;
4345       }
4346     }
4347     long lastRecordedFlushedSequenceId = -1;
4348     String nodePath = ZKUtil.joinZNode(this.zooKeeper.recoveringRegionsZNode,
4349       region.getEncodedName());
4350     // recovering-region level
4351     byte[] data = ZKUtil.getData(zkw, nodePath);
4352     if (data != null) {
4353       lastRecordedFlushedSequenceId = SplitLogManager.parseLastFlushedSequenceIdFrom(data);
4354     }
4355     if (data == null || lastRecordedFlushedSequenceId < minSeqIdForLogReplay) {
4356       ZKUtil.setData(zkw, nodePath, ZKUtil.positionToByteArray(minSeqIdForLogReplay));
4357     }
4358     if (previousRSName != null) {
4359       // one level deeper for the failed RS
4360       nodePath = ZKUtil.joinZNode(nodePath, previousRSName);
4361       ZKUtil.setData(zkw, nodePath,
4362         ZKUtil.regionSequenceIdsToByteArray(minSeqIdForLogReplay, maxSeqIdInStores));
4363       LOG.debug("Update last flushed sequence id of region " + region.getEncodedName() + " for "
4364           + previousRSName);
4365     } else {
4366       LOG.warn("Can't find failed region server for recovering region " + region.getEncodedName());
4367     }
4368   }
4369 
4370   /**
4371    * Return the last failed RS name under /hbase/recovering-regions/encodedRegionName
4372    * @param encodedRegionName
4373    * @throws KeeperException
4374    */
4375   private String getLastFailedRSFromZK(String encodedRegionName) throws KeeperException {
4376     String result = null;
4377     long maxZxid = 0;
4378     ZooKeeperWatcher zkw = this.getZooKeeper();
4379     String nodePath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, encodedRegionName);
4380     List<String> failedServers = ZKUtil.listChildrenNoWatch(zkw, nodePath);
4381     if (failedServers == null || failedServers.isEmpty()) {
4382       return result;
4383     }
4384     for (String failedServer : failedServers) {
4385       String rsPath = ZKUtil.joinZNode(nodePath, failedServer);
4386       Stat stat = new Stat();
4387       ZKUtil.getDataNoWatch(zkw, rsPath, stat);
4388       if (maxZxid < stat.getCzxid()) {
4389         maxZxid = stat.getCzxid();
4390         result = failedServer;
4391       }
4392     }
4393     return result;
4394   }
4395 
4396   @Override
4397   public UpdateFavoredNodesResponse updateFavoredNodes(RpcController controller,
4398       UpdateFavoredNodesRequest request) throws ServiceException {
4399     List<UpdateFavoredNodesRequest.RegionUpdateInfo> openInfoList = request.getUpdateInfoList();
4400     UpdateFavoredNodesResponse.Builder respBuilder = UpdateFavoredNodesResponse.newBuilder();
4401     for (UpdateFavoredNodesRequest.RegionUpdateInfo regionUpdateInfo : openInfoList) {
4402       HRegionInfo hri = HRegionInfo.convert(regionUpdateInfo.getRegion());
4403       updateRegionFavoredNodesMapping(hri.getEncodedName(),
4404           regionUpdateInfo.getFavoredNodesList());
4405     }
4406     respBuilder.setResponse(openInfoList.size());
4407     return respBuilder.build();
4408   }
4409 }